From 972051a61eb8dd9e1c60c5de9ad5e3aeef860eb6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 20 Sep 2019 10:16:23 -0700 Subject: [PATCH 001/150] removing 3 and 4 years from validity range options --- .../app/angular/certificates/certificate/tracking.tpl.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index 19d8f37f..7ac2107f 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -133,7 +133,7 @@
@@ -141,8 +141,6 @@ - -
From a13c45e9cce2a6bf9de53d2516f3aec32f54ddc0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 20 Sep 2019 13:49:38 -0700 Subject: [PATCH 002/150] updating dependencies, and fixing the deprecated arrow.replaces to shift --- lemur/certificates/service.py | 4 +- lemur/common/missing.py | 4 +- lemur/pending_certificates/schemas.py | 4 +- lemur/pending_certificates/service.py | 2 +- lemur/plugins/lemur_digicert/plugin.py | 10 ++-- lemur/plugins/lemur_verisign/plugin.py | 14 ++--- lemur/tests/test_missing.py | 4 +- package.json | 4 +- requirements-dev.txt | 30 ++++++---- requirements-docs.txt | 75 ++++++++++++----------- requirements-tests.txt | 83 ++++++++++++++------------ requirements.in | 2 +- requirements.txt | 67 +++++++++++---------- 13 files changed, 159 insertions(+), 144 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 5a65c383..1a0cdc5a 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -419,7 +419,7 @@ def render(args): ) if time_range: - to = arrow.now().replace(weeks=+time_range).format("YYYY-MM-DD") + to = arrow.now().shift(weeks=+time_range).format("YYYY-MM-DD") now = arrow.now().format("YYYY-MM-DD") query = query.filter(Certificate.not_after <= to).filter( Certificate.not_after >= now @@ -561,7 +561,7 @@ def stats(**kwargs): """ if kwargs.get("metric") == "not_after": start = arrow.utcnow() - end = start.replace(weeks=+32) + end = start.shift(weeks=+32) items = ( database.db.session.query(Certificate.issuer, func.count(Certificate.id)) .group_by(Certificate.issuer) diff --git a/lemur/common/missing.py b/lemur/common/missing.py index 2f5156df..f991d2e3 100644 --- a/lemur/common/missing.py +++ b/lemur/common/missing.py @@ -15,11 +15,11 @@ def convert_validity_years(data): now = arrow.utcnow() data["validity_start"] = now.isoformat() - end = now.replace(years=+int(data["validity_years"])) + end = now.shift(years=+int(data["validity_years"])) if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True): if is_weekend(end): - end = end.replace(days=-2) + end = end.shift(days=-2) data["validity_end"] = end.isoformat() return data diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 68f22b4a..989178e9 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -46,10 +46,10 @@ class PendingCertificateOutputSchema(LemurOutputSchema): # Note aliasing is the first step in deprecating these fields. notify = fields.Boolean() - active = fields.Boolean(attribute="notify") + active = fields.Boolean(attribute="notify", dump_only=True) cn = fields.String() - common_name = fields.String(attribute="cn") + common_name = fields.String(attribute="cn", dump_only=True) owner = fields.Email() diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 935ea689..8b4d033c 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -244,7 +244,7 @@ def render(args): ) if time_range: - to = arrow.now().replace(weeks=+time_range).format("YYYY-MM-DD") + to = arrow.now().shift(weeks=+time_range).format("YYYY-MM-DD") now = arrow.now().format("YYYY-MM-DD") query = query.filter(PendingCertificate.not_after <= to).filter( PendingCertificate.not_after >= now diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index c5b01cc4..d2648bc1 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -72,11 +72,11 @@ def determine_validity_years(end_date): """ now = arrow.utcnow() - if end_date < now.replace(years=+1): + if end_date < now.shift(years=+1): return 1 - elif end_date < now.replace(years=+2): + elif end_date < now.shift(years=+2): return 2 - elif end_date < now.replace(years=+3): + elif end_date < now.shift(years=+3): return 3 raise Exception( @@ -148,12 +148,12 @@ def map_cis_fields(options, csr): """ if not options.get("validity_years"): if not options.get("validity_end"): - options["validity_end"] = arrow.utcnow().replace( + options["validity_end"] = arrow.utcnow().shift( years=current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) ) options["validity_years"] = determine_validity_years(options["validity_end"]) else: - options["validity_end"] = arrow.utcnow().replace( + options["validity_end"] = arrow.utcnow().shift( years=options["validity_years"] ) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 65bd1cac..7bf517b7 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -111,16 +111,14 @@ def process_options(options): data["subject_alt_names"] = ",".join(get_additional_names(options)) - if options.get("validity_end") > arrow.utcnow().replace(years=2): + if options.get("validity_end") > arrow.utcnow().shift(years=2): raise Exception( "Verisign issued certificates cannot exceed two years in validity" ) if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date - if options.get("validity_end") < arrow.utcnow().replace(years=2).replace( - days=-1 - ): + if options.get("validity_end") < arrow.utcnow().shift(years=2, days=-1): period = get_default_issuance(options) data["specificEndDate"] = options["validity_end"].format("MM/DD/YYYY") data["validityPeriod"] = period @@ -149,9 +147,9 @@ def get_default_issuance(options): """ now = arrow.utcnow() - if options["validity_end"] < now.replace(years=+1): + if options["validity_end"] < now.shift(years=+1): validity_period = "1Y" - elif options["validity_end"] < now.replace(years=+2): + elif options["validity_end"] < now.shift(years=+2): validity_period = "2Y" else: raise Exception( @@ -261,7 +259,7 @@ class VerisignIssuerPlugin(IssuerPlugin): url = current_app.config.get("VERISIGN_URL") + "/reportingws" end = arrow.now() - start = end.replace(days=-7) + start = end.shift(days=-7) data = { "reportType": "detail", @@ -299,7 +297,7 @@ class VerisignSourcePlugin(SourcePlugin): def get_certificates(self): url = current_app.config.get("VERISIGN_URL") + "/reportingws" end = arrow.now() - start = end.replace(years=-5) + start = end.shift(years=-5) data = { "reportType": "detail", "startDate": start.format("MM/DD/YYYY"), diff --git a/lemur/tests/test_missing.py b/lemur/tests/test_missing.py index be615ced..59bac2d6 100644 --- a/lemur/tests/test_missing.py +++ b/lemur/tests/test_missing.py @@ -10,11 +10,11 @@ def test_convert_validity_years(session): data = convert_validity_years(dict(validity_years=2)) assert data["validity_start"] == arrow.utcnow().isoformat() - assert data["validity_end"] == arrow.utcnow().replace(years=+2).isoformat() + assert data["validity_end"] == arrow.utcnow().shift(years=+2).isoformat() with freeze_time("2015-01-10"): data = convert_validity_years(dict(validity_years=1)) assert ( data["validity_end"] - == arrow.utcnow().replace(years=+1, days=-2).isoformat() + == arrow.utcnow().shift(years=+1, days=-2).isoformat() ) diff --git a/package.json b/package.json index fe1267a6..9b899176 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ }, "dependencies": { "bower": "^1.8.2", - "browser-sync": "^2.3.1", + "browser-sync": "^2.26.7", "del": "^2.2.2", "gulp-autoprefixer": "^3.1.1", "gulp-cache": "^0.4.5", @@ -25,7 +25,7 @@ "gulp-minify-css": "^1.2.4", "gulp-minify-html": "~1.0.6", "gulp-ng-annotate": "~2.0.0", - "gulp-ng-html2js": "~0.2.2", + "gulp-ng-html2js": "^0.2.3", "gulp-notify": "^2.2.0", "gulp-plumber": "^1.1.0", "gulp-print": "^2.0.1", diff --git a/requirements-dev.txt b/requirements-dev.txt index 030c3f93..6dff5655 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,31 +6,35 @@ # aspy.yaml==1.3.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2019.3.9 # via requests -cfgv==2.0.0 # via pre-commit +certifi==2019.9.11 # via requests +cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests -docutils==0.14 # via readme-renderer +docutils==0.15.2 # via readme-renderer flake8==3.5.0 -identify==1.4.3 # via pre-commit +identify==1.4.7 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.17 # via pre-commit -invoke==1.2.0 +importlib-metadata==0.23 # via pre-commit +invoke==1.3.0 mccabe==0.6.1 # via flake8 +more-itertools==7.2.0 # via zipp nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.16.1 +pre-commit==1.18.3 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.4.2 # via readme-renderer -pyyaml==5.1 +pyyaml==5.1.2 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.32.1 # via twine -twine==1.13.0 -urllib3==1.25.3 # via requests -virtualenv==16.6.0 # via pre-commit +tqdm==4.36.1 # via twine +twine==1.15.0 +urllib3==1.25.5 # via requests +virtualenv==16.7.5 # via pre-commit webencodings==0.5.1 # via bleach -zipp==0.5.1 # via importlib-metadata +zipp==0.6.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.2.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index c0fe427e..05cfb49c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,23 +4,23 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.34.2 +acme==0.38.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.10 -amqp==2.5.0 -aniso8601==6.0.0 -arrow==0.14.2 +alembic==1.2.0 +amqp==2.5.1 +aniso8601==8.0.0 +arrow==0.15.2 asn1crypto==0.24.0 asyncpool==1.0 babel==2.7.0 # via sphinx -bcrypt==3.1.6 -billiard==3.6.0.0 +bcrypt==3.1.7 +billiard==3.6.1.0 blinker==1.4 -boto3==1.9.160 -botocore==1.12.160 +boto3==1.9.232 +botocore==1.12.232 celery[redis]==4.3.0 -certifi==2019.3.9 +certifi==2019.9.11 certsrv==2.1.1 cffi==1.12.3 chardet==3.0.4 @@ -29,10 +29,10 @@ cloudflare==2.3.0 cryptography==2.7 dnspython3==1.15.0 dnspython==1.15.0 -docutils==0.14 +docutils==0.15.2 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.7 +flask-cors==3.0.8 flask-mail==0.9.1 flask-migrate==2.5.2 flask-principal==0.4.0 @@ -40,10 +40,10 @@ flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.0 -flask==1.0.3 +flask==1.1.1 future==0.17.1 gunicorn==19.9.0 -hvac==0.9.1 +hvac==0.9.5 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 @@ -51,47 +51,47 @@ itsdangerous==1.1.0 javaobj-py3==0.3.0 jinja2==2.10.1 jmespath==0.9.4 -josepy==1.1.0 +josepy==1.2.0 jsonlines==1.2.0 kombu==4.5.0 lockfile==0.12.2 logmatic-python==0.1.7 -mako==1.0.11 +mako==1.1.0 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.3 -marshmallow==2.19.2 +marshmallow-sqlalchemy==0.19.0 +marshmallow==2.20.4 mock==3.0.5 ndg-httpsclient==0.5.1 -packaging==19.0 # via sphinx -paramiko==2.4.2 -pem==19.1.0 -psycopg2==2.8.2 -pyasn1-modules==0.2.5 -pyasn1==0.4.5 +packaging==19.2 # via sphinx +paramiko==2.6.0 +pem==19.2.0 +psycopg2==2.8.3 +pyasn1-modules==0.2.6 +pyasn1==0.4.7 pycparser==2.19 -pycryptodomex==3.8.2 +pycryptodomex==3.9.0 pygments==2.4.2 # via sphinx pyjks==19.0.0 pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==19.0.0 -pyparsing==2.4.0 # via packaging +pyparsing==2.4.2 # via packaging pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.11 -pytz==2019.1 -pyyaml==5.1 +pytz==2019.2 +pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.2.1 +redis==3.3.8 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 -s3transfer==0.2.0 +s3transfer==0.2.1 six==1.12.0 -snowballstemmer==1.2.1 # via sphinx +snowballstemmer==1.9.1 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==2.1.0 +sphinx==2.2.0 sphinxcontrib-applehelp==1.0.1 # via sphinx sphinxcontrib-devhelp==1.0.1 # via sphinx sphinxcontrib-htmlhelp==1.0.2 # via sphinx @@ -99,11 +99,14 @@ sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.4 +sqlalchemy-utils==0.34.2 +sqlalchemy==1.3.8 tabulate==0.8.3 twofish==0.3.0 -urllib3==1.25.3 +urllib3==1.25.5 vine==1.3.0 -werkzeug==0.15.4 +werkzeug==0.16.0 xmltodict==0.12.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.2.0 # via acme, josepy, sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 77bc92af..242e7e5c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,75 +7,82 @@ appdirs==1.4.3 # via black asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==19.1.0 # via black, pytest -aws-sam-translator==1.11.0 # via cfn-lint +attrs==19.1.0 # via black, jsonschema, pytest +aws-sam-translator==1.14.0 # via cfn-lint aws-xray-sdk==2.4.2 # via moto -bandit==1.6.0 +bandit==1.6.2 black==19.3b0 -boto3==1.9.160 # via aws-sam-translator, moto +boto3==1.9.232 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.160 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.3.9 # via requests +botocore==1.12.232 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2019.9.11 # via requests cffi==1.12.3 # via cryptography -cfn-lint==0.21.4 # via moto +cfn-lint==0.24.1 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask -coverage==4.5.3 -cryptography==2.7 # via moto -docker==4.0.1 # via moto -docutils==0.14 # via botocore -ecdsa==0.13.2 # via python-jose +coverage==4.5.4 +cryptography==2.7 # via moto, sshpubkeys +datetime==4.3 # via moto +docker==4.0.2 # via moto +docutils==0.15.2 # via botocore +ecdsa==0.13.2 # via python-jose, sshpubkeys factory-boy==2.12.0 -faker==1.0.7 -flask==1.0.3 # via pytest-flask +faker==2.0.2 +flask==1.1.1 # via pytest-flask freezegun==0.3.12 future==0.17.1 # via aws-xray-sdk, python-jose gitdb2==2.0.5 # via gitpython -gitpython==2.1.11 # via bandit +gitpython==3.0.2 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.17 # via pluggy, pytest +importlib-metadata==0.23 # via pluggy, pytest itsdangerous==1.1.0 # via flask jinja2==2.10.1 # via flask, moto jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.2 # via moto -jsonpatch==1.23 # via cfn-lint +jsonpatch==1.24 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==2.6.0 # via aws-sam-translator, cfn-lint +jsonschema==3.0.2 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto -more-itertools==7.0.0 # via pytest -moto==1.3.8 +more-itertools==7.2.0 # via pytest, zipp +moto==1.3.13 nose==1.3.7 -packaging==19.0 # via pytest -pbr==5.2.1 # via stevedore -pluggy==0.12.0 # via pytest +packaging==19.2 # via pytest +pbr==5.4.3 # via stevedore +pluggy==0.13.0 # via pytest py==1.8.0 # via pytest -pyasn1==0.4.5 # via rsa +pyasn1==0.4.7 # via rsa pycparser==2.19 # via cffi pyflakes==2.1.1 -pyparsing==2.4.0 # via packaging +pyparsing==2.4.2 # via packaging +pyrsistent==0.15.4 # via jsonschema pytest-flask==0.15.0 pytest-mock==1.10.4 -pytest==4.6.2 +pytest==5.1.2 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto -pytz==2019.1 # via moto -pyyaml==5.1 -requests-mock==1.6.0 -requests==2.22.0 # via cfn-lint, docker, moto, requests-mock, responses +pytz==2019.2 # via datetime, moto +pyyaml==5.1.2 +requests-mock==1.7.0 +requests==2.22.0 # via docker, moto, requests-mock, responses responses==0.10.6 # via moto rsa==4.0 # via python-jose -s3transfer==0.2.0 # via boto3 -six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, mock, moto, packaging, pytest, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +s3transfer==0.2.1 # via boto3 +six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap2==2.0.5 # via gitdb2 -stevedore==1.30.1 # via bandit -text-unidecode==1.2 # via faker +sshpubkeys==3.1.0 # via moto +stevedore==1.31.0 # via bandit +text-unidecode==1.3 # via faker toml==0.10.0 # via black -urllib3==1.25.3 # via botocore, requests +urllib3==1.25.5 # via botocore, requests wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker -werkzeug==0.15.4 # via flask, moto, pytest-flask -wrapt==1.11.1 # via aws-xray-sdk +werkzeug==0.16.0 # via flask, moto, pytest-flask +wrapt==1.11.2 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==0.5.1 # via importlib-metadata +zipp==0.6.0 # via importlib-metadata +zope.interface==4.6.0 # via datetime + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface diff --git a/requirements.in b/requirements.in index d766b7a9..c7c79137 100644 --- a/requirements.in +++ b/requirements.in @@ -32,7 +32,7 @@ kombu<4.6.0 # Bug with inspecting active tasks: https://github.com/celery/kombu/ lockfile logmatic-python marshmallow-sqlalchemy -marshmallow +marshmallow<2.20.5 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 ndg-httpsclient paramiko # required for the SFTP destination plugin pem diff --git a/requirements.txt b/requirements.txt index c19c7b6e..db7e46a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,21 +4,21 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.34.2 +acme==0.38.0 alembic-autogenerate-enums==0.0.2 -alembic==1.0.10 # via flask-migrate -amqp==2.5.0 # via kombu -aniso8601==6.0.0 # via flask-restful -arrow==0.14.2 +alembic==1.2.0 # via flask-migrate +amqp==2.5.1 # via kombu +aniso8601==8.0.0 # via flask-restful +arrow==0.15.2 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 -bcrypt==3.1.6 # via flask-bcrypt, paramiko -billiard==3.6.0.0 # via celery +bcrypt==3.1.7 # via flask-bcrypt, paramiko +billiard==3.6.1.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.160 -botocore==1.12.160 +boto3==1.9.232 +botocore==1.12.232 celery[redis]==4.3.0 -certifi==2019.3.9 +certifi==2019.9.11 certsrv==2.1.1 cffi==1.12.3 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -27,10 +27,10 @@ cloudflare==2.3.0 cryptography==2.7 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 -docutils==0.14 # via botocore +docutils==0.15.2 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.7 +flask-cors==3.0.8 flask-mail==0.9.1 flask-migrate==2.5.2 flask-principal==0.4.0 @@ -38,34 +38,34 @@ flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.0 -flask==1.0.3 +flask==1.1.1 future==0.17.1 gunicorn==19.9.0 -hvac==0.9.1 +hvac==0.9.5 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask javaobj-py3==0.3.0 # via pyjks jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore -josepy==1.1.0 # via acme +josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.5.0 lockfile==0.12.2 logmatic-python==0.1.7 -mako==1.0.11 # via alembic +mako==1.1.0 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.3 -marshmallow==2.19.2 +marshmallow-sqlalchemy==0.19.0 +marshmallow==2.20.4 mock==3.0.5 # via acme ndg-httpsclient==0.5.1 -paramiko==2.4.2 -pem==19.1.0 -psycopg2==2.8.2 -pyasn1-modules==0.2.5 # via pyjks, python-ldap -pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, pyjks, python-ldap +paramiko==2.6.0 +pem==19.2.0 +psycopg2==2.8.3 +pyasn1-modules==0.2.6 # via pyjks, python-ldap +pyasn1==0.4.7 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi -pycryptodomex==3.8.2 # via pyjks +pycryptodomex==3.9.0 # via pyjks pyjks==19.0.0 pyjwt==1.7.1 pynacl==1.3.0 # via paramiko @@ -75,20 +75,23 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 -pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1 +pytz==2019.2 # via acme, celery, flask-restful, pyrfc3339 +pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.2.1 +redis==3.3.8 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 -s3transfer==0.2.0 # via boto3 +s3transfer==0.2.1 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.4 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.34.2 +sqlalchemy==1.3.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 twofish==0.3.0 # via pyjks -urllib3==1.25.3 # via botocore, requests +urllib3==1.25.5 # via botocore, requests vine==1.3.0 # via amqp, celery -werkzeug==0.15.4 # via flask +werkzeug==0.16.0 # via flask xmltodict==0.12.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.2.0 # via acme, josepy From 96b2149433fb79f56567a4785f32b0d472173bb2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 20 Sep 2019 15:22:45 -0700 Subject: [PATCH 003/150] removing unintended commit --- lemur/pending_certificates/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 989178e9..68f22b4a 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -46,10 +46,10 @@ class PendingCertificateOutputSchema(LemurOutputSchema): # Note aliasing is the first step in deprecating these fields. notify = fields.Boolean() - active = fields.Boolean(attribute="notify", dump_only=True) + active = fields.Boolean(attribute="notify") cn = fields.String() - common_name = fields.String(attribute="cn", dump_only=True) + common_name = fields.String(attribute="cn") owner = fields.Email() From 86f661a8afab8fd93146c6af649ae82dd2388422 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 23 Sep 2019 12:36:08 -0700 Subject: [PATCH 004/150] With NLBs the DNS formatting has changed, which resulted in Lemur not getting the region correctly parsed --- lemur/plugins/lemur_aws/plugin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 4414a62c..d4a4a6d4 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -40,7 +40,11 @@ from lemur.plugins.lemur_aws import iam, s3, elb, ec2 def get_region_from_dns(dns): - return dns.split(".")[-4] + # XXX.REGION.elb.amazonaws.com + if dns.endswith(".elb.amazonaws.com"): + return dns.split(".")[-4] + else: # NLBs have a different pattern on the dns XXXX.elb.REGION.amazonaws.com + return dns.split(".")[-3] def format_elb_cipher_policy_v2(policy): From 477db836f4ba7bbf3f029a2e1ebb86be23f1290f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 23 Sep 2019 12:52:17 -0700 Subject: [PATCH 005/150] lint --- lemur/plugins/lemur_aws/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index d4a4a6d4..cf6c8643 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -40,10 +40,11 @@ from lemur.plugins.lemur_aws import iam, s3, elb, ec2 def get_region_from_dns(dns): - # XXX.REGION.elb.amazonaws.com + # XXX.REGION.elb.amazonaws.com if dns.endswith(".elb.amazonaws.com"): return dns.split(".")[-4] - else: # NLBs have a different pattern on the dns XXXX.elb.REGION.amazonaws.com + else: + # NLBs have a different pattern on the dns XXXX.elb.REGION.amazonaws.com return dns.split(".")[-3] From f0652ca6a9ff2c10e065e604fe0a03fee1251a40 Mon Sep 17 00:00:00 2001 From: pmelse Date: Thu, 10 Oct 2019 15:49:31 -0400 Subject: [PATCH 006/150] bug fix for overwriting certificates --- lemur/plugins/lemur_sftp/plugin.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index de8df427..9cc8140e 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -170,8 +170,17 @@ class SFTPDestinationPlugin(DestinationPlugin): current_app.logger.debug( "Uploading {0} to {1}".format(filename, dst_path_cn) ) - with sftp.open(dst_path_cn + "/" + filename, "w") as f: - f.write(data) + try: + with sftp.open(dst_path_cn + "/" + filename, "w") as f: + f.write(data) + except (PermissionError) as permerror: + if permerror.errno == 13: + current_app.logger.debug( + "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn) + ) + sftp.chmod(dst_path_cn + "/" + filename, 0o600) + with sftp.open(dst_path_cn + "/" + filename, "w") as f: + f.write(data) # read only for owner, -r-------- sftp.chmod(dst_path_cn + "/" + filename, 0o400) From 6f96a8f5b0ccd6aa16d1a3a606bb4c25f4c7ab46 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 15 Oct 2019 15:47:21 -0700 Subject: [PATCH 007/150] updating requirements --- requirements-dev.txt | 6 +++--- requirements-docs.txt | 36 +++++++++++++++++----------------- requirements-tests.txt | 44 +++++++++++++++++++++--------------------- requirements.txt | 34 ++++++++++++++++---------------- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6dff5655..4e940357 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -30,11 +30,11 @@ requests==2.22.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.36.1 # via twine -twine==1.15.0 -urllib3==1.25.5 # via requests +twine==2.0.0 +urllib3==1.25.6 # via requests virtualenv==16.7.5 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via twine +# setuptools==41.4.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index 05cfb49c..260c8608 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,25 +4,25 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.38.0 +acme==0.39.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.2.0 -amqp==2.5.1 +alembic==1.2.1 +amqp==2.5.2 aniso8601==8.0.0 arrow==0.15.2 -asn1crypto==0.24.0 +asn1crypto==1.1.0 asyncpool==1.0 babel==2.7.0 # via sphinx bcrypt==3.1.7 billiard==3.6.1.0 blinker==1.4 -boto3==1.9.232 -botocore==1.12.232 +boto3==1.9.250 +botocore==1.12.250 celery[redis]==4.3.0 certifi==2019.9.11 certsrv==2.1.1 -cffi==1.12.3 +cffi==1.13.0 chardet==3.0.4 click==7.0 cloudflare==2.3.0 @@ -39,9 +39,9 @@ flask-principal==0.4.0 flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 -flask-sqlalchemy==2.4.0 +flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.17.1 +future==0.18.0 gunicorn==19.9.0 hvac==0.9.5 idna==2.8 @@ -49,7 +49,7 @@ imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 javaobj-py3==0.3.0 -jinja2==2.10.1 +jinja2==2.10.3 jmespath==0.9.4 josepy==1.2.0 jsonlines==1.2.0 @@ -66,7 +66,7 @@ packaging==19.2 # via sphinx paramiko==2.6.0 pem==19.2.0 psycopg2==2.8.3 -pyasn1-modules==0.2.6 +pyasn1-modules==0.2.7 pyasn1==0.4.7 pycparser==2.19 pycryptodomex==3.9.0 @@ -80,16 +80,16 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.11 -pytz==2019.2 +pytz==2019.3 pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.3.8 +redis==3.3.11 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 six==1.12.0 -snowballstemmer==1.9.1 # via sphinx +snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 sphinx==2.2.0 sphinxcontrib-applehelp==1.0.1 # via sphinx @@ -100,13 +100,13 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.8 -tabulate==0.8.3 +sqlalchemy==1.3.10 +tabulate==0.8.5 twofish==0.3.0 -urllib3==1.25.5 +urllib3==1.25.6 vine==1.3.0 werkzeug==0.16.0 xmltodict==0.12.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via acme, josepy, sphinx +# setuptools==41.4.0 # via acme, josepy, sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 29d272a0..e6dc53c5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,45 +5,45 @@ # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # appdirs==1.4.3 # via black -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.1.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==19.1.0 # via black, jsonschema, pytest -aws-sam-translator==1.14.0 # via cfn-lint +attrs==19.3.0 # via black, jsonschema, pytest +aws-sam-translator==1.15.1 # via cfn-lint aws-xray-sdk==2.4.2 # via moto bandit==1.6.2 black==19.3b0 -boto3==1.9.232 # via aws-sam-translator, moto +boto3==1.9.250 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.232 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.12.250 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2019.9.11 # via requests -cffi==1.12.3 # via cryptography -cfn-lint==0.24.1 # via moto +cffi==1.13.0 # via cryptography +cfn-lint==0.24.4 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask coverage==4.5.4 cryptography==2.7 # via moto, sshpubkeys datetime==4.3 # via moto -docker==4.0.2 # via moto +docker==4.1.0 # via moto docutils==0.15.2 # via botocore -ecdsa==0.13.2 # via python-jose, sshpubkeys +ecdsa==0.13.3 # via python-jose, sshpubkeys factory-boy==2.12.0 -faker==2.0.2 +faker==2.0.3 fakeredis==1.0.5 flask==1.1.1 # via pytest-flask freezegun==0.3.12 -future==0.17.1 # via aws-xray-sdk, python-jose -gitdb2==2.0.5 # via gitpython -gitpython==3.0.2 # via bandit +future==0.18.0 # via aws-xray-sdk, python-jose +gitdb2==2.0.6 # via gitpython +gitpython==3.0.3 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.23 # via pluggy, pytest +importlib-metadata==0.23 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.10.1 # via flask, moto +jinja2==2.10.3 # via flask, moto jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.2 # via moto jsonpatch==1.24 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==3.0.2 # via aws-sam-translator, cfn-lint +jsonschema==3.1.1 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto more-itertools==7.2.0 # via pytest, zipp @@ -59,13 +59,13 @@ pyflakes==2.1.1 pyparsing==2.4.2 # via packaging pyrsistent==0.15.4 # via jsonschema pytest-flask==0.15.0 -pytest-mock==1.10.4 -pytest==5.1.2 +pytest-mock==1.11.1 +pytest==5.2.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto -pytz==2019.2 # via datetime, moto +pytz==2019.3 # via datetime, moto pyyaml==5.1.2 -redis==3.3.8 # via fakeredis +redis==3.3.11 # via fakeredis requests-mock==1.7.0 requests==2.22.0 # via docker, moto, requests-mock, responses responses==0.10.6 # via moto @@ -78,7 +78,7 @@ sshpubkeys==3.1.0 # via moto stevedore==1.31.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -urllib3==1.25.5 # via botocore, requests +urllib3==1.25.6 # via botocore, requests wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker werkzeug==0.16.0 # via flask, moto, pytest-flask @@ -88,4 +88,4 @@ zipp==0.6.0 # via importlib-metadata zope.interface==4.6.0 # via datetime # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface +# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface diff --git a/requirements.txt b/requirements.txt index db7e46a7..305fe7e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,23 +4,23 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.38.0 +acme==0.39.0 alembic-autogenerate-enums==0.0.2 -alembic==1.2.0 # via flask-migrate -amqp==2.5.1 # via kombu +alembic==1.2.1 # via flask-migrate +amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful arrow==0.15.2 -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.1.0 # via cryptography asyncpool==1.0 bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.1.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.232 -botocore==1.12.232 +boto3==1.9.250 +botocore==1.12.250 celery[redis]==4.3.0 certifi==2019.9.11 certsrv==2.1.1 -cffi==1.12.3 # via bcrypt, cryptography, pynacl +cffi==1.13.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.3.0 @@ -37,16 +37,16 @@ flask-principal==0.4.0 flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 -flask-sqlalchemy==2.4.0 +flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.17.1 +future==0.18.0 gunicorn==19.9.0 hvac==0.9.5 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask javaobj-py3==0.3.0 # via pyjks -jinja2==2.10.1 +jinja2==2.10.3 jmespath==0.9.4 # via boto3, botocore josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare @@ -62,7 +62,7 @@ ndg-httpsclient==0.5.1 paramiko==2.6.0 pem==19.2.0 psycopg2==2.8.3 -pyasn1-modules==0.2.6 # via pyjks, python-ldap +pyasn1-modules==0.2.7 # via pyjks, python-ldap pyasn1==0.4.7 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi pycryptodomex==3.9.0 # via pyjks @@ -75,23 +75,23 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 -pytz==2019.2 # via acme, celery, flask-restful, pyrfc3339 +pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.3.8 +redis==3.3.11 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 # via boto3 six==1.12.0 sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.3 +sqlalchemy==1.3.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.5 twofish==0.3.0 # via pyjks -urllib3==1.25.5 # via botocore, requests +urllib3==1.25.6 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.16.0 # via flask xmltodict==0.12.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via acme, josepy +# setuptools==41.4.0 # via acme, josepy From a076497cf0c8bb33be81ade490e7bb5b258eec5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2019 22:49:30 +0000 Subject: [PATCH 008/150] Bump ecdsa from 0.13.2 to 0.13.3 Bumps [ecdsa](https://github.com/warner/python-ecdsa) from 0.13.2 to 0.13.3. - [Release notes](https://github.com/warner/python-ecdsa/releases) - [Changelog](https://github.com/warner/python-ecdsa/blob/master/NEWS) - [Commits](https://github.com/warner/python-ecdsa/compare/python-ecdsa-0.13.2...python-ecdsa-0.13.3) Signed-off-by: dependabot[bot] --- requirements-tests.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 29d272a0..8f646bc0 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ cryptography==2.7 # via moto, sshpubkeys datetime==4.3 # via moto docker==4.0.2 # via moto docutils==0.15.2 # via botocore -ecdsa==0.13.2 # via python-jose, sshpubkeys +ecdsa==0.13.3 # via python-jose, sshpubkeys factory-boy==2.12.0 faker==2.0.2 fakeredis==1.0.5 @@ -88,4 +88,4 @@ zipp==0.6.0 # via importlib-metadata zope.interface==4.6.0 # via datetime # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface +# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface From b5ab87877b34e760de6c1d0abef166301bd9618a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 17 Oct 2019 10:16:33 -0700 Subject: [PATCH 009/150] adding retry to acme setup client, since it can experience timeouts or other types of Connection Errors --- lemur/plugins/lemur_acme/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f31ffdcb..e38870d8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns +from retrying import retry class AuthorizationRecord(object): @@ -197,6 +198,7 @@ class AcmeHandler(object): ) return pem_certificate, pem_certificate_chain + @retry(stop_max_attempt_number=5, wait_fixed=5000) def setup_acme_client(self, authority): if not authority.options: raise InvalidAuthority("Invalid authority. Options not set") From 10b600424efbabcdbe2727e1a94d3ba15778ae71 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 08:45:32 -0700 Subject: [PATCH 010/150] refactoring searching for cert --- lemur/sources/service.py | 47 +++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index d5bd7426..070e1a47 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -124,40 +124,47 @@ def sync_endpoints(source): return new, updated +def find_cert(certificate): + updated_by_hash = 0 + exists = False + + if certificate.get("search", None): + conditions = certificate.pop("search") + exists = certificate_service.get_by_attributes(conditions) + + if not exists and certificate.get("name"): + result = certificate_service.get_by_name(certificate["name"]) + if result: + exists = [result] + + if not exists and certificate.get("serial"): + exists = certificate_service.get_by_serial(certificate["serial"]) + + if not exists: + cert = parse_certificate(certificate["body"]) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) + updated_by_hash += 1 + + exists = [x for x in exists if x] + return exists, updated_by_hash + # TODO this is very slow as we don't batch update certificates def sync_certificates(source, user): - new, updated = 0, 0 + new, updated, updated_by_hash = 0, 0, 0 current_app.logger.debug("Retrieving certificates from {0}".format(source.label)) s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) for certificate in certificates: - exists = False - - if certificate.get("search", None): - conditions = certificate.pop("search") - exists = certificate_service.get_by_attributes(conditions) - - if not exists and certificate.get("name"): - result = certificate_service.get_by_name(certificate["name"]) - if result: - exists = [result] - - if not exists and certificate.get("serial"): - exists = certificate_service.get_by_serial(certificate["serial"]) - - if not exists: - cert = parse_certificate(certificate["body"]) - matching_serials = certificate_service.get_by_serial(serial(cert)) - exists = find_matching_certificates_by_hash(cert, matching_serials) + exists, updated_by_hash = find_cert(certificate) if not certificate.get("owner"): certificate["owner"] = user.email certificate["creator"] = user - exists = [x for x in exists if x] if not exists: certificate_create(certificate, source) From d43e859c34ca61caca375485a5c0a912655d5474 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 08:46:01 -0700 Subject: [PATCH 011/150] describing the cert for each endpoint, for better cert search --- lemur/plugins/lemur_aws/plugin.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index cf6c8643..a03e92a8 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,7 +32,9 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ +from acme.errors import ClientError from flask import current_app +from lemur.extensions import sentry, metrics from lemur.plugins import lemur_aws as aws from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin @@ -110,6 +112,8 @@ def get_elb_endpoints(account_number, region, elb_dict): listener["Listener"]["SSLCertificateId"] ), ) + endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], + account_number=account_number) if listener["PolicyNames"]: policy = elb.describe_load_balancer_policies( @@ -127,6 +131,28 @@ def get_elb_endpoints(account_number, region, elb_dict): return endpoints +def get_elb_certificate_by_name(certificate_name, account_number): + # certificate name may contain path, in which case we remove it + if "/" in certificate_name: + certificate_name = certificate_name.split('/')[1] + try: + cert = iam.get_certificate(certificate_name, account_number=account_number) + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) + except ClientError: + current_app.logger.warning( + "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) + sentry.captureException() + metrics.send( + "get_elb_certificate_failed", "counter", 1, + metric_tags={"certificate_name": certificate_name, "account_number": account_number} + ) + return None + + def get_elb_endpoints_v2(account_number, region, elb_dict): """ Retrieves endpoint information from elbv2 response data. @@ -153,6 +179,8 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): port=listener["Port"], certificate_name=iam.get_name_from_arn(certificate["CertificateArn"]), ) + endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], + account_number=account_number) if listener["SslPolicy"]: policy = elb.describe_ssl_policies_v2( From f075c5af3d7e6c8d5353186770b3b7bc05453b50 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 08:48:11 -0700 Subject: [PATCH 012/150] in case no cert match via name-search, search via the cert itself (serial number, hash comparison) --- lemur/sources/service.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 070e1a47..23f2af72 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -66,7 +66,7 @@ def sync_update_destination(certificate, source): def sync_endpoints(source): - new, updated = 0, 0 + new, updated, updated_by_hash = 0, 0, 0 current_app.logger.debug("Retrieving endpoints from {0}".format(source.label)) s = plugins.get(source.plugin_name) @@ -89,6 +89,29 @@ def sync_endpoints(source): endpoint["certificate"] = certificate_service.get_by_name(certificate_name) + # if get cert by name failed, we attempt a search via serial number and hash comparison + # and link the endpoint certificate to Lemur certificate + if not endpoint["certificate"]: + certificate_attached_to_endpoint = endpoint.pop("certificate") + if certificate_attached_to_endpoint: + lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint) + updated_by_hash += updated_by_hash_tmp + + if lemur_matching_cert: + endpoint["certificate"] = lemur_matching_cert[0] + + if len(lemur_matching_cert) > 1: + current_app.logger.error( + "Too Many Certificates Found. Name: {0} Endpoint: {1}".format( + certificate_name, endpoint["name"] + ) + ) + metrics.send("endpoint.certificate.conflict", + "counter", 1, + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], + "acct": s.get_option("accountNumber", source.options)}) + + # this indicates the we were not able to describe the endpoint cert if not endpoint["certificate"]: current_app.logger.error( "Certificate Not Found. Name: {0} Endpoint: {1}".format( @@ -97,7 +120,8 @@ def sync_endpoints(source): ) metrics.send("endpoint.certificate.not.found", "counter", 1, - metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], + "acct": s.get_option("accountNumber", source.options)}) continue policy = endpoint.pop("policy") @@ -122,7 +146,8 @@ def sync_endpoints(source): endpoint_service.update(exists.id, **endpoint) updated += 1 - return new, updated + return new, updated, updated_by_hash + def find_cert(certificate): updated_by_hash = 0 @@ -159,7 +184,7 @@ def sync_certificates(source, user): certificates = s.get_certificates(source.options) for certificate in certificates: - exists, updated_by_hash = find_cert(certificate) + exists, updated_by_hash = find_cert(certificate) if not certificate.get("owner"): certificate["owner"] = user.email @@ -179,12 +204,12 @@ def sync_certificates(source, user): certificate_update(e, source) updated += 1 - return new, updated + return new, updated, updated_by_hash def sync(source, user): - new_certs, updated_certs = sync_certificates(source, user) - new_endpoints, updated_endpoints = sync_endpoints(source) + new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user) + new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source) source.last_run = arrow.utcnow() database.update(source) From 8aea257e6abb3f2d940ebf230fa81075c2425547 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 09:24:49 -0700 Subject: [PATCH 013/150] optimizing the call to describe cert to only the few certs with the naming issue --- lemur/plugins/lemur_aws/plugin.py | 48 ++++++++++++++----------------- lemur/sources/service.py | 16 +++++++++-- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index a03e92a8..46c65c4f 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -112,8 +112,6 @@ def get_elb_endpoints(account_number, region, elb_dict): listener["Listener"]["SSLCertificateId"] ), ) - endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], - account_number=account_number) if listener["PolicyNames"]: policy = elb.describe_load_balancer_policies( @@ -131,28 +129,6 @@ def get_elb_endpoints(account_number, region, elb_dict): return endpoints -def get_elb_certificate_by_name(certificate_name, account_number): - # certificate name may contain path, in which case we remove it - if "/" in certificate_name: - certificate_name = certificate_name.split('/')[1] - try: - cert = iam.get_certificate(certificate_name, account_number=account_number) - return dict( - body=cert["CertificateBody"], - chain=cert.get("CertificateChain"), - name=cert["ServerCertificateMetadata"]["ServerCertificateName"], - ) - except ClientError: - current_app.logger.warning( - "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) - sentry.captureException() - metrics.send( - "get_elb_certificate_failed", "counter", 1, - metric_tags={"certificate_name": certificate_name, "account_number": account_number} - ) - return None - - def get_elb_endpoints_v2(account_number, region, elb_dict): """ Retrieves endpoint information from elbv2 response data. @@ -179,8 +155,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): port=listener["Port"], certificate_name=iam.get_name_from_arn(certificate["CertificateArn"]), ) - endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], - account_number=account_number) if listener["SslPolicy"]: policy = elb.describe_ssl_policies_v2( @@ -299,6 +273,28 @@ class AWSSourcePlugin(SourcePlugin): account_number = self.get_option("accountNumber", options) iam.delete_cert(certificate.name, account_number=account_number) + def get_certificate_by_name(self, certificate_name, options): + account_number = self.get_option("accountNumber", options) + # certificate name may contain path, in which case we remove it + if "/" in certificate_name: + certificate_name = certificate_name.split('/')[1] + try: + cert = iam.get_certificate(certificate_name, account_number=account_number) + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) + except ClientError: + current_app.logger.warning( + "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) + sentry.captureException() + metrics.send( + "get_elb_certificate_failed", "counter", 1, + metric_tags={"certificate_name": certificate_name, "account_number": account_number} + ) + return None + class AWSDestinationPlugin(DestinationPlugin): title = "AWS" diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 23f2af72..498adfeb 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -15,7 +15,7 @@ from lemur.sources.models import Source from lemur.certificates.models import Certificate from lemur.certificates import service as certificate_service from lemur.endpoints import service as endpoint_service -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema @@ -92,7 +92,18 @@ def sync_endpoints(source): # if get cert by name failed, we attempt a search via serial number and hash comparison # and link the endpoint certificate to Lemur certificate if not endpoint["certificate"]: - certificate_attached_to_endpoint = endpoint.pop("certificate") + certificate_attached_to_endpoint = None + try: + certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options) + except NotImplementedError: + current_app.logger.warning( + "Unable to describe server certificate for endpoints in source {0}:" + " plugin has not implemented 'get_certificate_by_name'".format( + source.label + ) + ) + sentry.captureException() + if certificate_attached_to_endpoint: lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint) updated_by_hash += updated_by_hash_tmp @@ -111,7 +122,6 @@ def sync_endpoints(source): metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) - # this indicates the we were not able to describe the endpoint cert if not endpoint["certificate"]: current_app.logger.error( "Certificate Not Found. Name: {0} Endpoint: {1}".format( From 1768aad9e2ee95ed28ecfa9837b7db3597ff8551 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 10:17:58 -0700 Subject: [PATCH 014/150] capturing no such entity exception. --- lemur/plugins/lemur_aws/iam.py | 10 ++++++---- lemur/plugins/lemur_aws/plugin.py | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 67c35262..13590ddd 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -10,7 +10,7 @@ import botocore from retrying import retry -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.plugins.lemur_aws.sts import sts_client @@ -122,9 +122,11 @@ def get_certificate(name, **kwargs): """ client = kwargs.pop("client") metrics.send("get_certificate", "counter", 1, metric_tags={"name": name}) - return client.get_server_certificate(ServerCertificateName=name)[ - "ServerCertificate" - ] + try: + return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"] + except client.exceptions.NoSuchEntityException: + sentry.captureException() + return None @sts_client("iam") diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 46c65c4f..86cd7912 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -280,11 +280,12 @@ class AWSSourcePlugin(SourcePlugin): certificate_name = certificate_name.split('/')[1] try: cert = iam.get_certificate(certificate_name, account_number=account_number) - return dict( - body=cert["CertificateBody"], - chain=cert.get("CertificateChain"), - name=cert["ServerCertificateMetadata"]["ServerCertificateName"], - ) + if cert: + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) except ClientError: current_app.logger.warning( "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) From 9037f8843072ed3ab1695d4bca681d38e01f46de Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 11:02:41 -0700 Subject: [PATCH 015/150] just in case the path varies --- lemur/plugins/lemur_aws/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 86cd7912..98b01672 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -277,7 +277,7 @@ class AWSSourcePlugin(SourcePlugin): account_number = self.get_option("accountNumber", options) # certificate name may contain path, in which case we remove it if "/" in certificate_name: - certificate_name = certificate_name.split('/')[1] + certificate_name = certificate_name.split('/')[-1] try: cert = iam.get_certificate(certificate_name, account_number=account_number) if cert: From 14e13b512e70f1819f88964291744ab690417aff Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 11:03:28 -0700 Subject: [PATCH 016/150] providing a count for conflicts --- lemur/sources/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 498adfeb..8ba4ea0d 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -113,12 +113,12 @@ def sync_endpoints(source): if len(lemur_matching_cert) > 1: current_app.logger.error( - "Too Many Certificates Found. Name: {0} Endpoint: {1}".format( - certificate_name, endpoint["name"] + "Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format( + len(lemur_matching_cert), certificate_name, endpoint["name"] ) ) metrics.send("endpoint.certificate.conflict", - "counter", 1, + "gauge", len(lemur_matching_cert), metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) From 06f4aed6939f8b7081b30002f705a5be5d2cdc62 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 11:20:52 -0700 Subject: [PATCH 017/150] keeping track of certs found by hash --- lemur/sources/service.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 8ba4ea0d..f69f70f5 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -221,6 +221,14 @@ def sync(source, user): new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user) new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source) + metrics.send("sync.updated_certs_by_hash", + "gauge", updated_certs_by_hash, + metric_tags={"source": source.label}) + + metrics.send("sync.updated_endpoints_by_hash", + "gauge", updated_endpoints_by_hash, + metric_tags={"source": source.label}) + source.last_run = arrow.utcnow() database.update(source) From 0d983bd2b5f2e0ea7fec565b70bfa8f423358236 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 15:39:36 -0700 Subject: [PATCH 018/150] missed edge case --- lemur/sources/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f69f70f5..f4783313 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -78,7 +78,7 @@ def sync_endpoints(source): source.label ) ) - return new, updated + return new, updated, updated_by_hash for endpoint in endpoints: exists = endpoint_service.get_by_dnsname_and_port( From f803fab41300ba2acba3634b5d5795072112d7b0 Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 6 Nov 2019 10:14:49 -0800 Subject: [PATCH 019/150] add plugin to send atlas metric via redis --- lemur/plugins/lemur_atlas_redis/__init__.py | 4 + lemur/plugins/lemur_atlas_redis/plugin.py | 97 +++++++++++++++++++++ setup.py | 1 + 3 files changed, 102 insertions(+) create mode 100644 lemur/plugins/lemur_atlas_redis/__init__.py create mode 100644 lemur/plugins/lemur_atlas_redis/plugin.py diff --git a/lemur/plugins/lemur_atlas_redis/__init__.py b/lemur/plugins/lemur_atlas_redis/__init__.py new file mode 100644 index 00000000..f8afd7e3 --- /dev/null +++ b/lemur/plugins/lemur_atlas_redis/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__("pkg_resources").get_distribution(__name__).version +except Exception as e: + VERSION = "unknown" diff --git a/lemur/plugins/lemur_atlas_redis/plugin.py b/lemur/plugins/lemur_atlas_redis/plugin.py new file mode 100644 index 00000000..9a36743f --- /dev/null +++ b/lemur/plugins/lemur_atlas_redis/plugin.py @@ -0,0 +1,97 @@ +""" +.. module: lemur.plugins.lemur_atlas_redis.plugin + :platform: Unix + :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Jay Zarfoss +""" +from time import time +from redis import Redis +import json +from datetime import datetime + +from flask import current_app +from lemur.plugins import lemur_atlas as atlas +from lemur.plugins.bases.metric import MetricPlugin + + +def millis_since_epoch(): + """ + current time since epoch in milliseconds + """ + epoch = datetime.utcfromtimestamp(0) + delta = datetime.now() - epoch + return int(delta.total_seconds() * 1000.0) + + +class AtlasMetricRedisPlugin(MetricPlugin): + title = "AtlasRedis" + slug = "atlas-metric-Redius" + description = "Adds support for sending key metrics to Atlas via local Redis" + version = atlas.VERSION + + author = "Jay Zarfoss" + author_url = "https://github.com/netflix/lemur" + + options = [ + { + "name": "redis_host", + "type": "str", + "required": False, + "help_message": "If no host is provided localhost is assumed", + "default": "localhost", + }, + {"name": "redis_port", "type": "int", "required": False, "default": 28527}, + ] + + metric_data = {} + redis_host = None + redis_port = None + + def submit( + self, metric_name, metric_type, metric_value, metric_tags=None, options=None + ): + if not options: + options = self.options + + valid_types = ["COUNTER", "GAUGE", "TIMER"] + if metric_type.upper() not in valid_types: + raise Exception( + "Invalid Metric Type for Atlas: '{metric}' choose from: {options}".format( + metric=metric_type, options=",".join(valid_types) + ) + ) + + if metric_tags: + if not isinstance(metric_tags, dict): + raise Exception( + "Invalid Metric Tags for Atlas: Tags must be in dict format" + ) + + self.metric_data["timestamp"] = millis_since_epoch() + self.metric_data["type"] = metric_type.upper() + self.metric_data["name"] = str(metric_name) + self.metric_data["tags"] = metric_tags + + if ( + metric_value == "NaN" + or isinstance(metric_value, int) + or isinstance(metric_value, float) + ): + self.metric_data["value"] = metric_value + else: + raise Exception("Invalid Metric Value for Atlas: Metric must be a number") + + self.redis_host = self.get_option("redis_host", options) + self.redis_port = self.get_option("redis_port", options) + + try: + r = Redis(host=self.redis_host, port=self.redis_port, socket_timeout=0.1) + r.rpush('atlas-agent', json.dumps(self.metric_data)) + except Exception: + current_app.logger.warning( + "AtlasMetricsRedis: could not post atlas metrics to AtlasRedis {host}:{port}".format( + host=self.redis_host, port=self.redis_port + ) + ) diff --git a/setup.py b/setup.py index a01c110f..1c61e9f9 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,7 @@ setup( 'java_keystore_export = lemur.plugins.lemur_jks.plugin:JavaKeystoreExportPlugin', 'openssl_export = lemur.plugins.lemur_openssl.plugin:OpenSSLExportPlugin', 'atlas_metric = lemur.plugins.lemur_atlas.plugin:AtlasMetricPlugin', + 'atlas_metric_redis = lemur.plugins.lemur_atlas.plugin:AtlasMetricRedisPlugin', 'kubernetes_destination = lemur.plugins.lemur_kubernetes.plugin:KubernetesDestinationPlugin', 'cryptography_issuer = lemur.plugins.lemur_cryptography.plugin:CryptographyIssuerPlugin', 'cfssl_issuer = lemur.plugins.lemur_cfssl.plugin:CfsslIssuerPlugin', From 113c9dd65744a4783e4ec2498797f2fe5e341061 Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 6 Nov 2019 10:42:59 -0800 Subject: [PATCH 020/150] atlas redis plugin typo cleanup and better exception handling --- lemur/plugins/lemur_atlas_redis/plugin.py | 10 +++++----- setup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_atlas_redis/plugin.py b/lemur/plugins/lemur_atlas_redis/plugin.py index 9a36743f..e69ae672 100644 --- a/lemur/plugins/lemur_atlas_redis/plugin.py +++ b/lemur/plugins/lemur_atlas_redis/plugin.py @@ -6,7 +6,7 @@ .. moduleauthor:: Jay Zarfoss """ -from time import time + from redis import Redis import json from datetime import datetime @@ -27,7 +27,7 @@ def millis_since_epoch(): class AtlasMetricRedisPlugin(MetricPlugin): title = "AtlasRedis" - slug = "atlas-metric-Redius" + slug = "atlas-metric-redis" description = "Adds support for sending key metrics to Atlas via local Redis" version = atlas.VERSION @@ -89,9 +89,9 @@ class AtlasMetricRedisPlugin(MetricPlugin): try: r = Redis(host=self.redis_host, port=self.redis_port, socket_timeout=0.1) r.rpush('atlas-agent', json.dumps(self.metric_data)) - except Exception: + except Exception as e: current_app.logger.warning( - "AtlasMetricsRedis: could not post atlas metrics to AtlasRedis {host}:{port}".format( - host=self.redis_host, port=self.redis_port + "AtlasMetricsRedis: exception [{exception}] could not post atlas metrics to AtlasRedis [{host}:{port}], metric [{metricdata}]".format( + exception=e, host=self.redis_host, port=self.redis_port, metricdata=json.dumps(self.metric_data) ) ) diff --git a/setup.py b/setup.py index 1c61e9f9..90c0b2f8 100644 --- a/setup.py +++ b/setup.py @@ -147,7 +147,7 @@ setup( 'java_keystore_export = lemur.plugins.lemur_jks.plugin:JavaKeystoreExportPlugin', 'openssl_export = lemur.plugins.lemur_openssl.plugin:OpenSSLExportPlugin', 'atlas_metric = lemur.plugins.lemur_atlas.plugin:AtlasMetricPlugin', - 'atlas_metric_redis = lemur.plugins.lemur_atlas.plugin:AtlasMetricRedisPlugin', + 'atlas_metric_redis = lemur.plugins.lemur_atlas_redis.plugin:AtlasMetricRedisPlugin', 'kubernetes_destination = lemur.plugins.lemur_kubernetes.plugin:KubernetesDestinationPlugin', 'cryptography_issuer = lemur.plugins.lemur_cryptography.plugin:CryptographyIssuerPlugin', 'cfssl_issuer = lemur.plugins.lemur_cfssl.plugin:CfsslIssuerPlugin', From 00a0a27826f81ea7ca5859191dfaa8b7ef35efef Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 20 Nov 2019 09:44:31 -0800 Subject: [PATCH 021/150] used fixedName variable to transport db lookup optimization --- lemur/certificates/service.py | 3 +++ lemur/static/app/angular/certificates/view/view.js | 2 +- lemur/static/app/angular/certificates/view/view.tpl.html | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1a0cdc5a..0e91b563 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -393,6 +393,9 @@ def render(args): Certificate.cn.ilike(term), ) ) + elif "fixedName" in terms: + # only what matches the fixed name directly if a fixedname is provided + query = query.filter(Certificate.name == terms[1]) else: query = database.filter(query, Certificate, terms) diff --git a/lemur/static/app/angular/certificates/view/view.js b/lemur/static/app/angular/certificates/view/view.js index e4ae0314..0ee0d2c2 100644 --- a/lemur/static/app/angular/certificates/view/view.js +++ b/lemur/static/app/angular/certificates/view/view.js @@ -11,7 +11,7 @@ angular.module('lemur') controller: 'CertificatesViewController' }) .state('certificate', { - url: '/certificates/:name', + url: '/certificates/:fixedName', // use "fixedName" if in URL to indicate 'like' query can be avoided templateUrl: '/angular/certificates/view/view.tpl.html', controller: 'CertificatesViewController' }); diff --git a/lemur/static/app/angular/certificates/view/view.tpl.html b/lemur/static/app/angular/certificates/view/view.tpl.html index 9d5c7772..3f952aa2 100644 --- a/lemur/static/app/angular/certificates/view/view.tpl.html +++ b/lemur/static/app/angular/certificates/view/view.tpl.html @@ -52,7 +52,7 @@
- Permalink + Permalink From f188aea2c22c4342cb6d2610586be67590b43f86 Mon Sep 17 00:00:00 2001 From: Niels Bischof Date: Mon, 2 Dec 2019 06:22:09 -0500 Subject: [PATCH 022/150] type on quickstart/index.rst --- docs/quickstart/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 280bb612..01a5c7ca 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -31,7 +31,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git). -.. note:: Installing node from a package manager may creat the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following +.. note:: Installing node from a package manager may create the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following sudo ln -s /user/bin/nodejs /usr/bin/node Now, install Python ``virtualenv`` package: From 189e8b2725792e6f58c9d8399ebb0c50942b5d69 Mon Sep 17 00:00:00 2001 From: Ilya Labun Date: Fri, 13 Dec 2019 14:33:39 +0100 Subject: [PATCH 023/150] Eliminate subqueries when showing certificates list --- lemur/certificates/schemas.py | 25 +++++++++++++++++++ lemur/certificates/views.py | 3 ++- lemur/common/schema.py | 7 +++++- .../certificates/certificate/certificate.js | 10 +++++++- .../app/angular/certificates/view/view.js | 2 ++ .../angular/certificates/view/view.tpl.html | 2 +- 6 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 7f3c2ac0..c987e5fa 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -6,6 +6,8 @@ .. moduleauthor:: Kevin Glisson """ from flask import current_app +from flask_restful import inputs +from flask_restful.reqparse import RequestParser from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError @@ -285,6 +287,16 @@ class CertificateOutputSchema(LemurOutputSchema): rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema) +class CertificateShortOutputSchema(LemurOutputSchema): + id = fields.Integer() + name = fields.String() + owner = fields.Email() + notify = fields.Boolean() + authority = fields.Nested(AuthorityNestedOutputSchema) + issuer = fields.String() + cn = fields.String() + + class CertificateUploadInputSchema(CertificateCreationSchema): name = fields.String() authority = fields.Nested(AssociatedAuthoritySchema, required=False) @@ -363,9 +375,22 @@ class CertificateRevokeSchema(LemurInputSchema): comments = fields.String() +certificates_list_request_parser = RequestParser() +certificates_list_request_parser.add_argument("short", type=inputs.boolean, default=False, location="args") + + +def certificates_list_output_schema_factory(): + args = certificates_list_request_parser.parse_args() + if args["short"]: + return certificates_short_output_schema + else: + return certificates_output_schema + + certificate_input_schema = CertificateInputSchema() certificate_output_schema = CertificateOutputSchema() certificates_output_schema = CertificateOutputSchema(many=True) +certificates_short_output_schema = CertificateShortOutputSchema(many=True) certificate_upload_input_schema = CertificateUploadInputSchema() certificate_export_input_schema = CertificateExportInputSchema() certificate_edit_input_schema = CertificateEditInputSchema() diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 1a003e78..51f7f615 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -27,6 +27,7 @@ from lemur.certificates.schemas import ( certificates_output_schema, certificate_export_input_schema, certificate_edit_input_schema, + certificates_list_output_schema_factory, ) from lemur.roles import service as role_service @@ -250,7 +251,7 @@ class CertificatesList(AuthenticatedResource): self.reqparse = reqparse.RequestParser() super(CertificatesList, self).__init__() - @validate_schema(None, certificates_output_schema) + @validate_schema(None, certificates_list_output_schema_factory) def get(self): """ .. http:get:: /certificates diff --git a/lemur/common/schema.py b/lemur/common/schema.py index bfa0a091..ee1db464 100644 --- a/lemur/common/schema.py +++ b/lemur/common/schema.py @@ -169,7 +169,12 @@ def validate_schema(input_schema, output_schema): if not resp: return dict(message="No data found"), 404 - return unwrap_pagination(resp, output_schema), 200 + if callable(output_schema): + output_schema_to_use = output_schema() + else: + output_schema_to_use = output_schema + + return unwrap_pagination(resp, output_schema_to_use), 200 return decorated_function diff --git a/lemur/static/app/angular/certificates/certificate/certificate.js b/lemur/static/app/angular/certificates/certificate/certificate.js index 273fc9d5..21f61f22 100644 --- a/lemur/static/app/angular/certificates/certificate/certificate.js +++ b/lemur/static/app/angular/certificates/certificate/certificate.js @@ -371,4 +371,12 @@ angular.module('lemur') }); }); }; -}); +}) +.controller('CertificateInfoController', function ($scope, CertificateApi) { + $scope.fetchFullCertificate = function (certId) { + CertificateApi.get(certId).then(function (certificate) { + $scope.certificate = certificate; + }); + }; +}) +; diff --git a/lemur/static/app/angular/certificates/view/view.js b/lemur/static/app/angular/certificates/view/view.js index 0ee0d2c2..72a31618 100644 --- a/lemur/static/app/angular/certificates/view/view.js +++ b/lemur/static/app/angular/certificates/view/view.js @@ -28,6 +28,7 @@ angular.module('lemur') sorting: { id: 'desc' // initial sorting }, + short: true, filter: $scope.filter }, { total: 0, // length of data @@ -54,6 +55,7 @@ angular.module('lemur') sorting: { id: 'desc' // initial sorting }, + short: true, filter: $scope.filter }, { getData: function ($defer, params) { diff --git a/lemur/static/app/angular/certificates/view/view.tpl.html b/lemur/static/app/angular/certificates/view/view.tpl.html index 3f952aa2..7b0919f8 100644 --- a/lemur/static/app/angular/certificates/view/view.tpl.html +++ b/lemur/static/app/angular/certificates/view/view.tpl.html @@ -71,7 +71,7 @@
- + From 9fb4be12737989c3c0983838f3ab4057e479f0a5 Mon Sep 17 00:00:00 2001 From: pmelse Date: Fri, 27 Dec 2019 13:25:03 -0500 Subject: [PATCH 024/150] remove trailing whitespace --- lemur/plugins/lemur_sftp/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 9cc8140e..66784048 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -174,7 +174,7 @@ class SFTPDestinationPlugin(DestinationPlugin): with sftp.open(dst_path_cn + "/" + filename, "w") as f: f.write(data) except (PermissionError) as permerror: - if permerror.errno == 13: + if permerror.errno == 13: current_app.logger.debug( "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn) ) From 1ccc15859fdc8b263aee5fd4315f79a1c601609e Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 3 Jan 2020 14:14:01 -0800 Subject: [PATCH 025/150] updating requirements. removing this pin, since the issue is now resolved. kombu<4.6.0 # Bug with inspecting active tasks: https://github.com/celery/kombu/issues/1051 --- requirements-dev.txt | 27 +++++++------- requirements-docs.txt | 74 +++++++++++++++++++------------------- requirements-tests.txt | 81 +++++++++++++++++++++--------------------- requirements.in | 1 - requirements.txt | 64 +++++++++++++++++---------------- 5 files changed, 125 insertions(+), 122 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e940357..d1423888 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,35 +6,36 @@ # aspy.yaml==1.3.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2019.9.11 # via requests +certifi==2019.11.28 # via requests cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests docutils==0.15.2 # via readme-renderer flake8==3.5.0 -identify==1.4.7 # via pre-commit +identify==1.4.9 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.23 # via pre-commit +importlib-metadata==1.3.0 # via keyring, pre-commit, twine invoke==1.3.0 +keyring==21.0.0 # via twine mccabe==0.6.1 # via flake8 -more-itertools==7.2.0 # via zipp +more-itertools==8.0.2 # via zipp nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.18.3 +pre-commit==1.21.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 -pygments==2.4.2 # via readme-renderer -pyyaml==5.1.2 +pygments==2.5.2 # via readme-renderer +pyyaml==5.2 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine -six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer +six==1.13.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.36.1 # via twine -twine==2.0.0 -urllib3==1.25.6 # via requests -virtualenv==16.7.5 # via pre-commit +tqdm==4.41.1 # via twine +twine==3.1.1 +urllib3==1.25.7 # via requests +virtualenv==16.7.9 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.4.0 # via twine +# setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 260c8608..893965ca 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,29 +4,28 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.39.0 +acme==1.0.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.2.1 +alembic==1.3.2 amqp==2.5.2 aniso8601==8.0.0 -arrow==0.15.2 -asn1crypto==1.1.0 +arrow==0.15.5 asyncpool==1.0 -babel==2.7.0 # via sphinx +babel==2.8.0 # via sphinx bcrypt==3.1.7 billiard==3.6.1.0 blinker==1.4 -boto3==1.9.250 -botocore==1.12.250 -celery[redis]==4.3.0 -certifi==2019.9.11 +boto3==1.10.46 +botocore==1.13.46 +celery[redis]==4.4.0 +certifi==2019.11.28 certsrv==2.1.1 -cffi==1.13.0 +cffi==1.13.2 chardet==3.0.4 click==7.0 -cloudflare==2.3.0 -cryptography==2.7 +cloudflare==2.3.1 +cryptography==2.8 dnspython3==1.15.0 dnspython==1.15.0 docutils==0.15.2 @@ -41,57 +40,59 @@ flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.18.0 -gunicorn==19.9.0 -hvac==0.9.5 +future==0.18.2 +gunicorn==20.0.4 +hvac==0.9.6 idna==2.8 -imagesize==1.1.0 # via sphinx +imagesize==1.2.0 # via sphinx +importlib-metadata==1.3.0 inflection==0.3.1 itsdangerous==1.1.0 -javaobj-py3==0.3.0 +javaobj-py3==0.4.0.1 jinja2==2.10.3 jmespath==0.9.4 josepy==1.2.0 jsonlines==1.2.0 -kombu==4.5.0 +kombu==4.6.7 lockfile==0.12.2 logmatic-python==0.1.7 mako==1.1.0 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.19.0 +marshmallow-sqlalchemy==0.21.0 marshmallow==2.20.4 mock==3.0.5 +more-itertools==8.0.2 ndg-httpsclient==0.5.1 packaging==19.2 # via sphinx -paramiko==2.6.0 -pem==19.2.0 -psycopg2==2.8.3 +paramiko==2.7.1 +pem==19.3.0 +psycopg2==2.8.4 pyasn1-modules==0.2.7 -pyasn1==0.4.7 +pyasn1==0.4.8 pycparser==2.19 -pycryptodomex==3.9.0 -pygments==2.4.2 # via sphinx +pycryptodomex==3.9.4 +pygments==2.5.2 # via sphinx pyjks==19.0.0 pyjwt==1.7.1 pynacl==1.3.0 -pyopenssl==19.0.0 -pyparsing==2.4.2 # via packaging +pyopenssl==19.1.0 +pyparsing==2.4.6 # via packaging pyrfc3339==1.1 -python-dateutil==2.8.0 +python-dateutil==2.8.1 python-editor==1.0.4 python-json-logger==0.1.11 pytz==2019.3 -pyyaml==5.1.2 +pyyaml==5.2 raven[flask]==6.10.0 redis==3.3.11 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 -six==1.12.0 +six==1.13.0 snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==2.2.0 +sphinx==2.3.1 sphinxcontrib-applehelp==1.0.1 # via sphinx sphinxcontrib-devhelp==1.0.1 # via sphinx sphinxcontrib-htmlhelp==1.0.2 # via sphinx @@ -99,14 +100,15 @@ sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.10 -tabulate==0.8.5 +sqlalchemy-utils==0.36.1 +sqlalchemy==1.3.12 +tabulate==0.8.6 twofish==0.3.0 -urllib3==1.25.6 +urllib3==1.25.7 vine==1.3.0 werkzeug==0.16.0 xmltodict==0.12.0 +zipp==0.6.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.4.0 # via acme, josepy, sphinx +# setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index e6dc53c5..293bd350 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,37 +5,34 @@ # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # appdirs==1.4.3 # via black -asn1crypto==1.1.0 # via cryptography -atomicwrites==1.3.0 # via pytest attrs==19.3.0 # via black, jsonschema, pytest -aws-sam-translator==1.15.1 # via cfn-lint -aws-xray-sdk==2.4.2 # via moto +aws-sam-translator==1.19.1 # via cfn-lint +aws-xray-sdk==2.4.3 # via moto bandit==1.6.2 -black==19.3b0 -boto3==1.9.250 # via aws-sam-translator, moto +black==19.10b0 +boto3==1.10.46 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.250 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.9.11 # via requests -cffi==1.13.0 # via cryptography -cfn-lint==0.24.4 # via moto +botocore==1.13.46 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2019.11.28 # via requests +cffi==1.13.2 # via cryptography +cfn-lint==0.26.2 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask -coverage==4.5.4 -cryptography==2.7 # via moto, sshpubkeys -datetime==4.3 # via moto +coverage==5.0.1 +cryptography==2.8 # via moto, sshpubkeys docker==4.1.0 # via moto docutils==0.15.2 # via botocore -ecdsa==0.13.3 # via python-jose, sshpubkeys +ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 -faker==2.0.3 -fakeredis==1.0.5 +faker==3.0.0 +fakeredis==1.1.0 flask==1.1.1 # via pytest-flask freezegun==0.3.12 -future==0.18.0 # via aws-xray-sdk, python-jose +future==0.18.2 # via aws-xray-sdk gitdb2==2.0.6 # via gitpython -gitpython==3.0.3 # via bandit +gitpython==3.0.5 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.23 # via jsonschema, pluggy, pytest +importlib-metadata==1.3.0 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask jinja2==2.10.3 # via flask, moto jmespath==0.9.4 # via boto3, botocore @@ -43,49 +40,51 @@ jsondiff==1.1.2 # via moto jsonpatch==1.24 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==3.1.1 # via aws-sam-translator, cfn-lint +jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto -more-itertools==7.2.0 # via pytest, zipp -moto==1.3.13 +more-itertools==8.0.2 # via pytest, zipp +moto==1.3.14 nose==1.3.7 packaging==19.2 # via pytest -pbr==5.4.3 # via stevedore -pluggy==0.13.0 # via pytest -py==1.8.0 # via pytest -pyasn1==0.4.7 # via rsa +pathspec==0.7.0 # via black +pbr==5.4.4 # via stevedore +pluggy==0.13.1 # via pytest +py==1.8.1 # via pytest +pyasn1==0.4.8 # via python-jose, rsa pycparser==2.19 # via cffi pyflakes==2.1.1 -pyparsing==2.4.2 # via packaging -pyrsistent==0.15.4 # via jsonschema +pyparsing==2.4.6 # via packaging +pyrsistent==0.15.6 # via jsonschema pytest-flask==0.15.0 -pytest-mock==1.11.1 -pytest==5.2.1 -python-dateutil==2.8.0 # via botocore, faker, freezegun, moto -python-jose==3.0.1 # via moto -pytz==2019.3 # via datetime, moto -pyyaml==5.1.2 +pytest-mock==1.13.0 +pytest==5.3.2 +python-dateutil==2.8.1 # via botocore, faker, freezegun, moto +python-jose==3.1.0 # via moto +pytz==2019.3 # via moto +pyyaml==5.2 redis==3.3.11 # via fakeredis +regex==2019.12.20 # via black requests-mock==1.7.0 requests==2.22.0 # via docker, moto, requests-mock, responses -responses==0.10.6 # via moto +responses==0.10.9 # via moto rsa==4.0 # via python-jose s3transfer==0.2.1 # via boto3 -six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, fakeredis, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.13.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, faker, fakeredis, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap2==2.0.5 # via gitdb2 sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto stevedore==1.31.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -urllib3==1.25.6 # via botocore, requests -wcwidth==0.1.7 # via pytest -websocket-client==0.56.0 # via docker +typed-ast==1.4.0 # via black +urllib3==1.25.7 # via botocore, requests +wcwidth==0.1.8 # via pytest +websocket-client==0.57.0 # via docker werkzeug==0.16.0 # via flask, moto, pytest-flask wrapt==1.11.2 # via aws-xray-sdk xmltodict==0.12.0 # via moto zipp==0.6.0 # via importlib-metadata -zope.interface==4.6.0 # via datetime # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface +# setuptools diff --git a/requirements.in b/requirements.in index c7c79137..ed2093c9 100644 --- a/requirements.in +++ b/requirements.in @@ -28,7 +28,6 @@ gunicorn hvac # required for the vault destination plugin inflection jinja2 -kombu<4.6.0 # Bug with inspecting active tasks: https://github.com/celery/kombu/issues/1051 lockfile logmatic-python marshmallow-sqlalchemy diff --git a/requirements.txt b/requirements.txt index 305fe7e1..639c9377 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,27 +4,26 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.39.0 +acme==1.0.0 alembic-autogenerate-enums==0.0.2 -alembic==1.2.1 # via flask-migrate +alembic==1.3.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.2 -asn1crypto==1.1.0 # via cryptography +arrow==0.15.5 asyncpool==1.0 bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.1.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.250 -botocore==1.12.250 -celery[redis]==4.3.0 -certifi==2019.9.11 +boto3==1.10.46 +botocore==1.13.46 +celery[redis]==4.4.0 +certifi==2019.11.28 certsrv==2.1.1 -cffi==1.13.0 # via bcrypt, cryptography, pynacl +cffi==1.13.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask -cloudflare==2.3.0 -cryptography==2.7 +cloudflare==2.3.1 +cryptography==2.8 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore @@ -39,59 +38,62 @@ flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.18.0 -gunicorn==19.9.0 -hvac==0.9.5 +future==0.18.2 +gunicorn==20.0.4 +hvac==0.9.6 idna==2.8 # via requests +importlib-metadata==1.3.0 # via kombu inflection==0.3.1 itsdangerous==1.1.0 # via flask -javaobj-py3==0.3.0 # via pyjks +javaobj-py3==0.4.0.1 # via pyjks jinja2==2.10.3 jmespath==0.9.4 # via boto3, botocore josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.5.0 +kombu==4.6.7 # via celery lockfile==0.12.2 logmatic-python==0.1.7 mako==1.1.0 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.19.0 +marshmallow-sqlalchemy==0.21.0 marshmallow==2.20.4 mock==3.0.5 # via acme +more-itertools==8.0.2 # via zipp ndg-httpsclient==0.5.1 -paramiko==2.6.0 -pem==19.2.0 -psycopg2==2.8.3 +paramiko==2.7.1 +pem==19.3.0 +psycopg2==2.8.4 pyasn1-modules==0.2.7 # via pyjks, python-ldap -pyasn1==0.4.7 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap +pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi -pycryptodomex==3.9.0 # via pyjks +pycryptodomex==3.9.4 # via pyjks pyjks==19.0.0 pyjwt==1.7.1 pynacl==1.3.0 # via paramiko -pyopenssl==19.0.0 +pyopenssl==19.1.0 pyrfc3339==1.1 # via acme -python-dateutil==2.8.0 # via alembic, arrow, botocore +python-dateutil==2.8.1 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1.2 +pyyaml==5.2 raven[flask]==6.10.0 redis==3.3.11 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 # via boto3 -six==1.12.0 -sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.5 +six==1.13.0 +sqlalchemy-utils==0.36.1 +sqlalchemy==1.3.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.6 twofish==0.3.0 # via pyjks -urllib3==1.25.6 # via botocore, requests +urllib3==1.25.7 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.16.0 # via flask xmltodict==0.12.0 +zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.4.0 # via acme, josepy +# setuptools From 1537d591a8353f2bc14e9a652a2b557eba59e323 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 8 Jan 2020 14:42:16 -0800 Subject: [PATCH 026/150] Improved messaging to point out to the Auto Rotate option for certificate issuance and renewal. --- lemur/plugins/lemur_email/templates/expiration.html | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_email/templates/expiration.html b/lemur/plugins/lemur_email/templates/expiration.html index 3c500c38..f5185acd 100644 --- a/lemur/plugins/lemur_email/templates/expiration.html +++ b/lemur/plugins/lemur_email/templates/expiration.html @@ -106,7 +106,13 @@ - If the above certificates are still in use. You should re-issue and deploy new certificates as soon as possible. + Your action is required if the above certificates are still needed for your service. +

+ If your endpoints are still in use, you can access your certificate in Lemur, and enable Auto Rotate under the Action->Edit menu. + Lemur will take care of re-issuance and rotation of the certificate on the listed endpoints within one day. +

+ If your certificate is deployed with your service, you should re-issue and manually deploy a new certificate as soon as possible. + From 8be8c95b170b93da3316a20798d0993b21ca655b Mon Sep 17 00:00:00 2001 From: jenkins-x-bot Date: Thu, 9 Jan 2020 15:16:19 +0200 Subject: [PATCH 027/150] handled cfssl-key type error --- lemur/plugins/lemur_cfssl/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index ae16d168..02f3159d 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -56,7 +56,7 @@ class CfsslIssuerPlugin(IssuerPlugin): try: hex_key = current_app.config.get("CFSSL_KEY") key = bytes.fromhex(hex_key) - except (ValueError, NameError): + except (ValueError, NameError, TypeError): # unable to find CFSSL_KEY in config, continue using normal sign method pass else: From 409b499217c50f6f558c103be07d90ca30c1cfc2 Mon Sep 17 00:00:00 2001 From: jenkins-x-bot Date: Sun, 12 Jan 2020 01:25:22 +0200 Subject: [PATCH 028/150] added kubernetes auth for vault --- lemur/plugins/lemur_vault_dest/plugin.py | 60 ++++++++++++++++++------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index e1715592..47206708 100755 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -50,11 +50,19 @@ class VaultSourcePlugin(SourcePlugin): "helpMessage": "Version of the Vault KV API to use", }, { - "name": "vaultAuthTokenFile", + "name": "authenticationMethod", + "type": "select", + "value": "token", + "available": ["token", "kubernetes"], + "required": True, + "helpMessage": "Authentication method to use", + }, + { + "name": "tokenFile/VaultRole", "type": "str", "required": True, - "validation": "(/[^/]+)+", - "helpMessage": "Must be a valid file path!", + "validation": "^([a-zA-Z0-9/._-]+/?)+$", + "helpMessage": "Must be vaild file path for token based auth and valid role if k8s based auth", }, { "name": "vaultMount", @@ -85,7 +93,8 @@ class VaultSourcePlugin(SourcePlugin): cert = [] body = "" url = self.get_option("vaultUrl", options) - token_file = self.get_option("vaultAuthTokenFile", options) + auth_method = self.get_option("authenticationMethod", options) + auth_key = self.get_option("tokenFile/vaultRole", options) mount = self.get_option("vaultMount", options) path = self.get_option("vaultPath", options) obj_name = self.get_option("objectName", options) @@ -93,10 +102,17 @@ class VaultSourcePlugin(SourcePlugin): cert_filter = "-----BEGIN CERTIFICATE-----" cert_delimiter = "-----END CERTIFICATE-----" - with open(token_file, "r") as tfile: - token = tfile.readline().rstrip("\n") + client = hvac.Client(url=url) + if auth_method == 'token': + with open(auth_key, "r") as tfile: + token = tfile.readline().rstrip("\n") + client.token = token + + if auth_method == 'kubernetes': + f = open('/var/run/secrets/kubernetes.io/serviceaccount/token') + jwt = f.read() + client.auth_kubernetes(auth_key, jwt) - client = hvac.Client(url=url, token=token) client.secrets.kv.default_kv_version = api_version path = "{0}/{1}".format(path, obj_name) @@ -160,11 +176,19 @@ class VaultDestinationPlugin(DestinationPlugin): "helpMessage": "Version of the Vault KV API to use", }, { - "name": "vaultAuthTokenFile", + "name": "authenticationMethod", + "type": "select", + "value": "token", + "available": ["token", "kubernetes"], + "required": True, + "helpMessage": "Authentication method to use", + }, + { + "name": "tokenFile/VaultRole", "type": "str", "required": True, - "validation": "(/[^/]+)+", - "helpMessage": "Must be a valid file path!", + "validation": "^([a-zA-Z0-9/._-]+/?)+$", + "helpMessage": "Must be vaild file path for token based auth and valid role if k8s based auth", }, { "name": "vaultMount", @@ -219,7 +243,8 @@ class VaultDestinationPlugin(DestinationPlugin): cname = common_name(parse_certificate(body)) url = self.get_option("vaultUrl", options) - token_file = self.get_option("vaultAuthTokenFile", options) + auth_method = self.get_option("authenticationMethod", options) + auth_key = self.get_option("tokenFile/vaultRole", options) mount = self.get_option("vaultMount", options) path = self.get_option("vaultPath", options) bundle = self.get_option("bundleChain", options) @@ -245,10 +270,17 @@ class VaultDestinationPlugin(DestinationPlugin): exc_info=True, ) - with open(token_file, "r") as tfile: - token = tfile.readline().rstrip("\n") + client = hvac.Client(url=url) + if auth_method == 'token': + with open(auth_key, "r") as tfile: + token = tfile.readline().rstrip("\n") + client.token = token + + if auth_method == 'kubernetes': + f = open('/var/run/secrets/kubernetes.io/serviceaccount/token') + jwt = f.read() + client.auth_kubernetes(auth_key, jwt) - client = hvac.Client(url=url, token=token) client.secrets.kv.default_kv_version = api_version if obj_name: From cad56c813ee653a5712e9860f19c354a8767d99d Mon Sep 17 00:00:00 2001 From: jenkins-x-bot Date: Sun, 12 Jan 2020 01:51:48 +0200 Subject: [PATCH 029/150] fixed lint error --- lemur/plugins/lemur_vault_dest/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 47206708..cb821a36 100755 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -107,7 +107,7 @@ class VaultSourcePlugin(SourcePlugin): with open(auth_key, "r") as tfile: token = tfile.readline().rstrip("\n") client.token = token - + if auth_method == 'kubernetes': f = open('/var/run/secrets/kubernetes.io/serviceaccount/token') jwt = f.read() @@ -275,7 +275,7 @@ class VaultDestinationPlugin(DestinationPlugin): with open(auth_key, "r") as tfile: token = tfile.readline().rstrip("\n") client.token = token - + if auth_method == 'kubernetes': f = open('/var/run/secrets/kubernetes.io/serviceaccount/token') jwt = f.read() From 78f9c490ddedfd3b9a1efd8dfe9c8c9b92f752fc Mon Sep 17 00:00:00 2001 From: Ilya Labun Date: Mon, 13 Jan 2020 15:26:35 +0100 Subject: [PATCH 030/150] Fix Dockercompose for tests --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b9439be7..fc83a034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get update RUN apt-get install -y make software-properties-common curl RUN curl -sL https://deb.nodesource.com/setup_7.x | bash - RUN apt-get update -RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev +RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev RUN pip install -U setuptools RUN pip install coveralls bandit WORKDIR /app From 8d957f22af40b741829290cee66a45083cb6a8f0 Mon Sep 17 00:00:00 2001 From: jenkins-x-bot Date: Mon, 13 Jan 2020 22:46:34 +0200 Subject: [PATCH 031/150] changed file handling --- lemur/plugins/lemur_vault_dest/plugin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index cb821a36..d401387b 100755 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -109,8 +109,9 @@ class VaultSourcePlugin(SourcePlugin): client.token = token if auth_method == 'kubernetes': - f = open('/var/run/secrets/kubernetes.io/serviceaccount/token') - jwt = f.read() + token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' + with open(token_path, 'r') as f: + jwt = f.read() client.auth_kubernetes(auth_key, jwt) client.secrets.kv.default_kv_version = api_version @@ -277,8 +278,9 @@ class VaultDestinationPlugin(DestinationPlugin): client.token = token if auth_method == 'kubernetes': - f = open('/var/run/secrets/kubernetes.io/serviceaccount/token') - jwt = f.read() + token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' + with open(token_path, 'r') as f: + jwt = f.read() client.auth_kubernetes(auth_key, jwt) client.secrets.kv.default_kv_version = api_version From cd7d9aee55839c8806c72ed403a03c41ef4cbdec Mon Sep 17 00:00:00 2001 From: jenkins-x-bot Date: Mon, 13 Jan 2020 23:09:58 +0200 Subject: [PATCH 032/150] fixed lint error --- lemur/plugins/lemur_vault_dest/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index d401387b..41b9c252 100755 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -109,7 +109,7 @@ class VaultSourcePlugin(SourcePlugin): client.token = token if auth_method == 'kubernetes': - token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' + token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' with open(token_path, 'r') as f: jwt = f.read() client.auth_kubernetes(auth_key, jwt) From 58d8a145c30e117218edef6363f1c48d84004dce Mon Sep 17 00:00:00 2001 From: pmelse Date: Mon, 13 Jan 2020 22:13:30 -0500 Subject: [PATCH 033/150] update for #2857 workaround update for #2857 workaround --- docs/quickstart/index.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 01a5c7ca..82bfc357 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -180,6 +180,13 @@ Lemur provides a helpful command that will initialize your database for you. It In addition to creating a new user, Lemur also creates a few default email notifications. These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``. They basically guarantee that every certificate within Lemur will send one expiration notification to the security team. +Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user. + +.. code-block:: bash + sudo -u postgres -i + psql + postgres=# ALTER USER lemur WITH SUPERUSER + Additional notifications can be created through the UI or API. See :ref:`Creating Notifications ` and :ref:`Command Line Interface ` for details. **Make note of the password used as this will be used during first login to the Lemur UI.** @@ -189,10 +196,16 @@ Additional notifications can be created through the UI or API. See :ref:`Creati cd /www/lemur/lemur lemur init +.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now. + +.. code-block:: bash + sudo -u postgres -i + psql + postgres=# ALTER USER lemur WITH NOSUPERUSER + .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. - Set Up a Reverse Proxy --------------------- From 1ed6ae539deea5e6d3513c30d982c52f5d6207db Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 15 Jan 2020 16:19:48 -0800 Subject: [PATCH 034/150] # possibility to default to a SIGNING_ALGORITHM for a given profile --- lemur/plugins/lemur_digicert/plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 5f52611f..88ea5b6b 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -171,6 +171,9 @@ def map_cis_fields(options, csr): "units": [options["organizational_unit"]], }, } + # possibility to default to a SIGNING_ALGORITHM for a given profile + if current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name): + data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name) return data From d6f41b6a99d8fedfc0b7505a95a921a2add22466 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 16 Jan 2020 13:45:13 -0800 Subject: [PATCH 035/150] improving string formatting to avoid dangling white spaces and new lines --- lemur/plugins/lemur_aws/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 98b01672..6669f641 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -212,7 +212,7 @@ class AWSSourcePlugin(SourcePlugin): if not regions: regions = ec2.get_regions(account_number=account_number) else: - regions = regions.split(",") + regions = "".join(regions.split()).split(",") for region in regions: elbs = elb.get_all_elbs(account_number=account_number, region=region) From 3080a9527c8ea539da8f767d0a735e6b53e8e0fe Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 17 Jan 2020 18:29:37 -0800 Subject: [PATCH 036/150] adding PowerDNS get_zones functionality and unit tests --- lemur/plugins/lemur_acme/plugin.py | 4 +- lemur/plugins/lemur_acme/powerdns.py | 108 +++++++++++++++ lemur/plugins/lemur_acme/tests/test_acme.py | 12 +- .../plugins/lemur_acme/tests/test_powerdns.py | 124 ++++++++++++++++++ 4 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 lemur/plugins/lemur_acme/powerdns.py create mode 100644 lemur/plugins/lemur_acme/tests/test_powerdns.py diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e38870d8..93628905 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -31,7 +31,7 @@ from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProv from lemur.extensions import metrics, sentry from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin -from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns +from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns from retrying import retry @@ -377,6 +377,7 @@ class AcmeHandler(object): "dyn": dyn, "route53": route53, "ultradns": ultradns, + # "powerdns": powerdns, } provider = provider_types.get(type) if not provider: @@ -436,6 +437,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "dyn": dyn, "route53": route53, "ultradns": ultradns, + # "powerdns": powerdns, } provider = provider_types.get(type) if not provider: diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py new file mode 100644 index 00000000..2f80e54f --- /dev/null +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -0,0 +1,108 @@ +import time +import requests +import json +import sys + +import dns +import dns.exception +import dns.name +import dns.query +import dns.resolver + +from flask import current_app +from lemur.extensions import metrics, sentry + +class Zone: + """ This class implements a PowerDNS zone in JSON. """ + + def __init__(self, _data): + self._data = _data + + @property + def id(self): + """ Zone id, has a trailing "." at the end, which we manually remove. """ + return self._data["id"][:-1] + + @property + def name(self): + """ Zone name, has a trailing "." at the end, which we manually remove. """ + return self._data["name"][:-1] + + @property + def kind(self): + """ Indicates whether the zone is setup as a PRIMARY or SECONDARY """ + return self._data["kind"] + + +def _generate_header(): + """Function to generate the header for a request using the PowerDNS API Key""" + + api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME", "") + api_key = current_app.config.get("ACME_POWERDNS_APIKEY", "") + return {api_key_name: api_key} + + +def _get(path, params=None): + """ + Function to execute a GET request on the given URL (base_uri + path) with given params + Returns JSON response object + """ + base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") + resp = requests.get( + f"{base_uri}{path}", + headers=_generate_header(), + params=params, + verify=True, + ) + resp.raise_for_status() + return resp.json() + + +def get_zones(account_number): + """Get zones from the PowerDNS""" + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") + path = f"/api/v1/servers/{server_id}/zones" + zones = [] + for elem in _get(path): + zone = Zone(elem) + if zone.kind == 'Master': + zones.append(zone.name) + return zones + +def create_txt_record(domain, token, account_number): + """ + Create a TXT record for the given domain. + + The part of the domain that matches with the zone becomes the zone name. + The remainder becomes the owner name (referred to as node name here) + Example: Let's say we have a zone named "exmaple.com" in PowerDNS and we + get a request to create a cert for lemur.example.com + Domain - _acme-challenge.lemur.example.com + Matching zone - example.com + Owner name - _acme-challenge.lemur + """ + pass + +def wait_for_dns_change(change_id, account_number=None): + """ + Waits and checks if the DNS changes have propagated or not. + + First check the domains authoritative server. Once this succeeds, + we ask a public DNS server (Google <8.8.8.8> in our case). + """ + pass + +def delete_txt_record(change_id, account_number, domain, token): + """ + Delete the TXT record that was created in the create_txt_record() function. + + UltraDNS handles records differently compared to Dyn. It creates an RRSet + which is a set of records of the same type and owner. This means + that while deleting the record, we cannot delete any individual record from + the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are + being created for the same domain at the same time, the challenge TXT records + that are created will be added under the same RRSet. If the RRSet had more + than 1 record, then we create a new RRSet on UltraDNS minus the record that + has to be deleted. + """ + pass diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 2f9dd719..04997ace 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -364,7 +364,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.ultradns.requests") @patch("lemur.plugins.lemur_acme.ultradns.current_app") - def test_get_ultradns_token(self, mock_current_app, mock_requests): + def test_ultradns_get_token(self, mock_current_app, mock_requests): # ret_val = json.dumps({"access_token": "access"}) the_response = Response() the_response._content = b'{"access_token": "access"}' @@ -374,7 +374,7 @@ class TestAcme(unittest.TestCase): self.assertTrue(len(result) > 0) @patch("lemur.plugins.lemur_acme.ultradns.current_app") - def test_create_txt_record(self, mock_current_app): + def test_ultradns_create_txt_record(self, mock_current_app): domain = "_acme_challenge.test.example.com" zone = "test.example.com" token = "ABCDEFGHIJ" @@ -395,7 +395,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.ultradns.current_app") @patch("lemur.extensions.metrics") - def test_delete_txt_record(self, mock_metrics, mock_current_app): + def test_ultradns_delete_txt_record(self, mock_metrics, mock_current_app): domain = "_acme_challenge.test.example.com" zone = "test.example.com" token = "ABCDEFGHIJ" @@ -418,7 +418,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.ultradns.current_app") @patch("lemur.extensions.metrics") - def test_wait_for_dns_change(self, mock_metrics, mock_current_app): + def test_ultradns_wait_for_dns_change(self, mock_metrics, mock_current_app): ultradns._has_dns_propagated = Mock(return_value=True) nameserver = "1.1.1.1" ultradns.get_authoritative_nameserver = Mock(return_value=nameserver) @@ -437,7 +437,7 @@ class TestAcme(unittest.TestCase): } mock_current_app.logger.debug.assert_called_with(log_data) - def test_get_zone_name(self): + def test_ultradns_get_zone_name(self): zones = ['example.com', 'test.example.com'] zone = "test.example.com" domain = "_acme-challenge.test.example.com" @@ -446,7 +446,7 @@ class TestAcme(unittest.TestCase): result = ultradns.get_zone_name(domain, account_number) self.assertEqual(result, zone) - def test_get_zones(self): + def test_ultradns_get_zones(self): account_number = "1234567890" path = "a/b/c" zones = ['example.com', 'test.example.com'] diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py new file mode 100644 index 00000000..f39ef3c5 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -0,0 +1,124 @@ +import unittest +from requests.models import Response + +from mock import MagicMock, Mock, patch + +from lemur.plugins.lemur_acme import plugin, powerdns + +class TestPowerdns(unittest.TestCase): + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + def setUp(self, mock_dns_provider_service): + self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() + self.acme = plugin.AcmeHandler() + mock_dns_provider = Mock() + mock_dns_provider.name = "powerdns" + mock_dns_provider.credentials = "{}" + mock_dns_provider.provider_type = "powerdns" + self.acme.dns_providers_for_domain = { + "www.test.com": [mock_dns_provider], + "test.fakedomain.net": [mock_dns_provider], + } + + @patch("lemur.plugins.lemur_acme.powerdns.requests") + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_powerdns_get_token(self, mock_current_app, mock_requests): + # ret_val = json.dumps({"access_token": "access"}) + the_response = Response() + the_response._content = b'{"access_token": "access"}' + mock_requests.post = Mock(return_value=the_response) + mock_current_app.config.get = Mock(return_value="Test") + result = powerdns.get_powerdns_token() + self.assertTrue(len(result) > 0) + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_powerdns_create_txt_record(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + powerdns.get_zone_name = Mock(return_value=zone) + mock_current_app.logger.debug = Mock() + powerdns._post = Mock() + log_data = { + "function": "create_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record created" + } + result = powerdns.create_txt_record(domain, token, account_number) + mock_current_app.logger.debug.assert_called_with(log_data) + self.assertEqual(result, change_id) + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + @patch("lemur.extensions.metrics") + def test_powerdns_delete_txt_record(self, mock_metrics, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + mock_current_app.logger.debug = Mock() + powerdns.get_zone_name = Mock(return_value=zone) + powerdns._post = Mock() + powerdns._get = Mock() + powerdns._get.return_value = {'zoneName': 'test.example.com.com', + 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', + 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], + 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, + 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} + powerdns._delete = Mock() + mock_metrics.send = Mock() + powerdns.delete_txt_record(change_id, account_number, domain, token) + mock_current_app.logger.debug.assert_not_called() + mock_metrics.send.assert_not_called() + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + @patch("lemur.extensions.metrics") + def test_powerdns_wait_for_dns_change(self, mock_metrics, mock_current_app): + powerdns._has_dns_propagated = Mock(return_value=True) + nameserver = "1.1.1.1" + powerdns.get_authoritative_nameserver = Mock(return_value=nameserver) + mock_metrics.send = Mock() + domain = "_acme-challenge.test.example.com" + token = "ABCDEFGHIJ" + change_id = (domain, token) + mock_current_app.logger.debug = Mock() + powerdns.wait_for_dns_change(change_id) + # mock_metrics.send.assert_not_called() + log_data = { + "function": "wait_for_dns_change", + "fqdn": domain, + "status": True, + "message": "Record status on Public DNS" + } + mock_current_app.logger.debug.assert_called_with(log_data) + + def test_powerdns_get_zone_name(self): + zones = ['example.com', 'test.example.com'] + zone = "test.example.com" + domain = "_acme-challenge.test.example.com" + account_number = "1234567890" + powerdns.get_zones = Mock(return_value=zones) + result = powerdns.get_zone_name(domain, account_number) + self.assertEqual(result, zone) + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_powerdns_get_zones(self, mock_current_app): + account_number = "1234567890" + path = "a/b/c" + zones = ['example.com', 'test.example.com'] + get_response = [{'account': '', 'dnssec': 'False', 'id': 'example.com.', 'kind': 'Master', 'last_check': 0, 'masters': [], + 'name': 'example.com.', 'notified_serial': '2019111907', 'serial': '2019111907', + 'url': '/api/v1/servers/localhost/zones/example.com.'}, + {'account': '', 'dnssec': 'False', 'id': 'bad.example.com.', 'kind': 'Secondary', 'last_check': 0, 'masters': [], + 'name': 'bad.example.com.', 'notified_serial': '2018053104', 'serial': '2018053104', + 'url': '/api/v1/servers/localhost/zones/bad.example.com.'}, + {'account': '', 'dnssec': 'False', 'id': 'test.example.com.', 'kind': 'Master', 'last_check': 0, + 'masters': [], 'name': 'test.example.com.', 'notified_serial': '2019112501', 'serial': '2019112501', + 'url': '/api/v1/servers/localhost/zones/test.example.com.'}] + powerdns._get = Mock(path) + powerdns._get.side_effect = [get_response] + mock_current_app.config.get = Mock(return_value="localhost") + result = powerdns.get_zones(account_number) + self.assertEqual(result, zones) \ No newline at end of file From 71f43dfcc13bb68d488a5b322bbf5f45e1447fa4 Mon Sep 17 00:00:00 2001 From: Gutttlt <43376523+Gutttlt@users.noreply.github.com> Date: Tue, 21 Jan 2020 08:40:54 +0100 Subject: [PATCH 037/150] Fixing "'Role' object has no attribute 'set_third_party'" error. --- lemur/auth/ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index f4ceab03..ed87b76c 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -105,7 +105,7 @@ class LdapPrincipal: role = role_service.get_by_name(self.ldap_default_role) if role: if not role.third_party: - role = role.set_third_party(role.id, third_party_status=True) + role = role_service.set_third_party(role.id, third_party_status=True) roles.add(role) # update their 'roles' From 915ec0ba631fdac9ec6f44c71e90acfba27d32a0 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 21 Jan 2020 17:08:59 -0800 Subject: [PATCH 038/150] added PowerDNS support for create_txt_record and associated tests --- lemur/plugins/lemur_acme/powerdns.py | 81 +++++++++++++++++- .../plugins/lemur_acme/tests/test_powerdns.py | 84 ++++++++++--------- 2 files changed, 123 insertions(+), 42 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 2f80e54f..f68828d1 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -57,6 +57,18 @@ def _get(path, params=None): resp.raise_for_status() return resp.json() +def _patch(path, payload): + """ + Function to execute a Patch request on the given URL (base_uri + path) with given data + """ + base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") + resp = requests.patch( + f"{base_uri}{path}", + headers=_generate_header(), + data=json.dumps(payload) + ) + resp.raise_for_status() + def get_zones(account_number): """Get zones from the PowerDNS""" @@ -69,6 +81,23 @@ def get_zones(account_number): zones.append(zone.name) return zones +def _get_zone_name(domain, account_number): + """Get the matching zone for the given domain""" + zones = get_zones(account_number) + zone_name = "" + for z in zones: + if domain.endswith(z): + # Find the most specific zone possible for the domain + # Ex: If fqdn is a.b.c.com, there is a zone for c.com, + # and a zone for b.c.com, we want to use b.c.com. + if z.count(".") > zone_name.count("."): + zone_name = z + if not zone_name: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.fail", "counter", 1) + raise Exception(f"No PowerDNS zone found for domain: {domain}") + return zone_name + def create_txt_record(domain, token, account_number): """ Create a TXT record for the given domain. @@ -81,7 +110,57 @@ def create_txt_record(domain, token, account_number): Matching zone - example.com Owner name - _acme-challenge.lemur """ - pass + + zone_name = _get_zone_name(domain, account_number) + node_name = domain[:-len(".".join(zone_name))] + + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") + zone_id = zone_name.join(".") + domain_id = domain.join(".") + + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + payload = { + "rrsets": [ + { + "name": f"{domain_id}", + "type": "TXT", + "ttl": "300", + "changetype": "REPLACE", + "records": [ + { + "content": f"{token}", + "disabled": "false" + } + ], + "comments": [] + } + ] + } + + try: + _patch(path, payload) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token, + "message": "TXT record successfully created" + } + current_app.logger.debug(log_data) + except Exception as e: + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "domain": domain, + "token": token, + "Exception": e, + "message": "Unable to create TXT record" + } + current_app.logger.debug(log_data) + + change_id = (domain, token) + return change_id + def wait_for_dns_change(change_id, account_number=None): """ diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index f39ef3c5..e0808d68 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -5,6 +5,7 @@ from mock import MagicMock, Mock, patch from lemur.plugins.lemur_acme import plugin, powerdns + class TestPowerdns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def setUp(self, mock_dns_provider_service): @@ -19,37 +20,6 @@ class TestPowerdns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - @patch("lemur.plugins.lemur_acme.powerdns.requests") - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_powerdns_get_token(self, mock_current_app, mock_requests): - # ret_val = json.dumps({"access_token": "access"}) - the_response = Response() - the_response._content = b'{"access_token": "access"}' - mock_requests.post = Mock(return_value=the_response) - mock_current_app.config.get = Mock(return_value="Test") - result = powerdns.get_powerdns_token() - self.assertTrue(len(result) > 0) - - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_powerdns_create_txt_record(self, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - powerdns.get_zone_name = Mock(return_value=zone) - mock_current_app.logger.debug = Mock() - powerdns._post = Mock() - log_data = { - "function": "create_txt_record", - "fqdn": domain, - "token": token, - "message": "TXT record created" - } - result = powerdns.create_txt_record(domain, token, account_number) - mock_current_app.logger.debug.assert_called_with(log_data) - self.assertEqual(result, change_id) - @patch("lemur.plugins.lemur_acme.powerdns.current_app") @patch("lemur.extensions.metrics") def test_powerdns_delete_txt_record(self, mock_metrics, mock_current_app): @@ -94,15 +64,6 @@ class TestPowerdns(unittest.TestCase): } mock_current_app.logger.debug.assert_called_with(log_data) - def test_powerdns_get_zone_name(self): - zones = ['example.com', 'test.example.com'] - zone = "test.example.com" - domain = "_acme-challenge.test.example.com" - account_number = "1234567890" - powerdns.get_zones = Mock(return_value=zones) - result = powerdns.get_zone_name(domain, account_number) - self.assertEqual(result, zone) - @patch("lemur.plugins.lemur_acme.powerdns.current_app") def test_powerdns_get_zones(self, mock_current_app): account_number = "1234567890" @@ -121,4 +82,45 @@ class TestPowerdns(unittest.TestCase): powerdns._get.side_effect = [get_response] mock_current_app.config.get = Mock(return_value="localhost") result = powerdns.get_zones(account_number) - self.assertEqual(result, zones) \ No newline at end of file + self.assertEqual(result, zones) + + def test_powerdns_get_zone_name(self): + zones = ['example.com', 'test.example.com'] + zone = "test.example.com" + domain = "_acme-challenge.test.example.com" + account_number = "1234567890" + powerdns.get_zones = Mock(return_value=zones) + result = powerdns._get_zone_name(domain, account_number) + self.assertEqual(result, zone) + + def mock_current_app_config_get(a, b): + """ Mock of current_app.config.get() """ + config = { + 'ACME_POWERDNS_APIKEYNAME': 'X-API-Key', + 'ACME_POWERDNS_APIKEY': 'KEY', + 'ACME_POWERDNS_DOMAIN': 'http://internal-dnshiddenmaster-1486232504.us-east-1.elb.amazonaws.com', + 'ACME_POWERDNS_SERVERID': 'localhost' + } + return config[a] + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + # @patch("lemur.plugins.lemur_acme.powerdns.current_app.config.get", side_effect=mock_current_app_config_get) + def test_powerdns_create_txt_record(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + powerdns._get_zone_name = Mock(return_value=zone) + mock_current_app.logger.debug = Mock() + mock_current_app.config.get = Mock(return_value="localhost") + powerdns._patch = Mock() + log_data = { + "function": "create_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record successfully created" + } + result = powerdns.create_txt_record(domain, token, account_number) + mock_current_app.logger.debug.assert_called_with(log_data) + self.assertEqual(result, change_id) \ No newline at end of file From 52c7686d58d116f22d28147b42532dcb1bc25759 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 21 Jan 2020 18:47:21 -0800 Subject: [PATCH 039/150] adding wait_for_dns_change() and tests for PowerDNS ACME plugin --- lemur/plugins/lemur_acme/powerdns.py | 141 +++++++++++++++++- .../plugins/lemur_acme/tests/test_powerdns.py | 44 +++--- 2 files changed, 161 insertions(+), 24 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index f68828d1..0a3135e6 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -33,6 +33,34 @@ class Zone: """ Indicates whether the zone is setup as a PRIMARY or SECONDARY """ return self._data["kind"] +class Record: + """ + This class implements a PowerDNS record. + + Accepts the response from the API call as the argument. + """ + + def __init__(self, _data): + # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response. + # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response. + self._data = _data + + @property + def name(self): + return self._data["name"] + + @property + def disabled(self): + return self._data["disabled"] + + @property + def content(self): + return self._data["content"] + + @property + def ttl(self): + return self._data["ttl"] + def _generate_header(): """Function to generate the header for a request using the PowerDNS API Key""" @@ -147,7 +175,7 @@ def create_txt_record(domain, token, account_number): "message": "TXT record successfully created" } current_app.logger.debug(log_data) - except Exception as e: + except requests.exceptions.RequestException as e: function = sys._getframe().f_code.co_name log_data = { "function": function, @@ -161,6 +189,78 @@ def create_txt_record(domain, token, account_number): change_id = (domain, token) return change_id +def _get_authoritative_nameserver(domain): + """Get the authoritative nameserver for the given domain""" + n = dns.name.from_text(domain) + + depth = 2 + default = dns.resolver.get_default_resolver() + nameserver = default.nameservers[0] + + last = False + while not last: + s = n.split(depth) + + last = s[0].to_unicode() == u"@" + sub = s[1] + + query = dns.message.make_query(sub, dns.rdatatype.NS) + response = dns.query.udp(query, nameserver) + + rcode = response.rcode() + if rcode != dns.rcode.NOERROR: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.error", "counter", 1) + if rcode == dns.rcode.NXDOMAIN: + raise Exception("%s does not exist." % sub) + else: + raise Exception("Error %s" % dns.rcode.to_text(rcode)) + + if len(response.authority) > 0: + rrset = response.authority[0] + else: + rrset = response.answer[0] + + rr = rrset[0] + if rr.rdtype != dns.rdatatype.SOA: + authority = rr.target + nameserver = default.query(authority).rrset[0].to_text() + + depth += 1 + + return nameserver + + +def _get_public_authoritative_nameserver(): + return "8.8.8.8" + +def _has_dns_propagated(name, token, domain): + """ + Check whether the DNS change made by Lemur have propagated to the public DNS or not. + + Invoked by wait_for_dns_change() function + """ + txt_records = [] + try: + dns_resolver = dns.resolver.Resolver() + dns_resolver.nameservers = [domain] + dns_response = dns_resolver.query(name, "TXT") + for rdata in dns_response: + for txt_record in rdata.strings: + txt_records.append(txt_record.decode("utf-8")) + except dns.exception.DNSException: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.fail", "counter", 1) + return False + + for txt_record in txt_records: + if txt_record == token: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.success", "counter", 1) + return True + + return False + def wait_for_dns_change(change_id, account_number=None): """ @@ -169,7 +269,44 @@ def wait_for_dns_change(change_id, account_number=None): First check the domains authoritative server. Once this succeeds, we ask a public DNS server (Google <8.8.8.8> in our case). """ - pass + domain, token = change_id + number_of_attempts = 20 + + # Check if Record exists via DNS + nameserver = _get_authoritative_nameserver(domain) + for attempts in range(0, number_of_attempts): + status = _has_dns_propagated(domain, token, nameserver) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "status": status, + "message": "Record status on ultraDNS authoritative server" + } + current_app.logger.debug(log_data) + if status: + time.sleep(10) + break + time.sleep(10) + if status: + nameserver = _get_public_authoritative_nameserver() + for attempts in range(0, number_of_attempts): + status = _has_dns_propagated(domain, token, nameserver) + log_data = { + "function": function, + "fqdn": domain, + "status": status, + "message": "Record status on Public DNS" + } + current_app.logger.debug(log_data) + if status: + metrics.send(f"{function}.success", "counter", 1) + break + time.sleep(10) + if not status: + metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) + sentry.captureException(extra={"fqdn": str(domain), "txt_record": str(token)}) + return def delete_txt_record(change_id, account_number, domain, token): """ diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index e0808d68..d69f890c 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -43,27 +43,6 @@ class TestPowerdns(unittest.TestCase): mock_current_app.logger.debug.assert_not_called() mock_metrics.send.assert_not_called() - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - @patch("lemur.extensions.metrics") - def test_powerdns_wait_for_dns_change(self, mock_metrics, mock_current_app): - powerdns._has_dns_propagated = Mock(return_value=True) - nameserver = "1.1.1.1" - powerdns.get_authoritative_nameserver = Mock(return_value=nameserver) - mock_metrics.send = Mock() - domain = "_acme-challenge.test.example.com" - token = "ABCDEFGHIJ" - change_id = (domain, token) - mock_current_app.logger.debug = Mock() - powerdns.wait_for_dns_change(change_id) - # mock_metrics.send.assert_not_called() - log_data = { - "function": "wait_for_dns_change", - "fqdn": domain, - "status": True, - "message": "Record status on Public DNS" - } - mock_current_app.logger.debug.assert_called_with(log_data) - @patch("lemur.plugins.lemur_acme.powerdns.current_app") def test_powerdns_get_zones(self, mock_current_app): account_number = "1234567890" @@ -123,4 +102,25 @@ class TestPowerdns(unittest.TestCase): } result = powerdns.create_txt_record(domain, token, account_number) mock_current_app.logger.debug.assert_called_with(log_data) - self.assertEqual(result, change_id) \ No newline at end of file + self.assertEqual(result, change_id) + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + @patch("lemur.extensions.metrics") + def test_powerdns_wait_for_dns_change(self, mock_metrics, mock_current_app): + powerdns._has_dns_propagated = Mock(return_value=True) + nameserver = "1.1.1.1" + powerdns._get_authoritative_nameserver = Mock(return_value=nameserver) + mock_metrics.send = Mock() + domain = "_acme-challenge.test.example.com" + token = "ABCDEFGHIJ" + change_id = (domain, token) + mock_current_app.logger.debug = Mock() + powerdns.wait_for_dns_change(change_id) + # mock_metrics.send.assert_not_called() + log_data = { + "function": "wait_for_dns_change", + "fqdn": domain, + "status": True, + "message": "Record status on Public DNS" + } + mock_current_app.logger.debug.assert_called_with(log_data) \ No newline at end of file From bddae6e4287f8c26e48ff49a185847edd3015f60 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 22 Jan 2020 16:18:52 -0800 Subject: [PATCH 040/150] adding PowerDNS delete_txt_record with associated tests --- lemur/plugins/lemur_acme/powerdns.py | 289 +++++++++--------- .../plugins/lemur_acme/tests/test_powerdns.py | 79 +++-- 2 files changed, 184 insertions(+), 184 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 0a3135e6..9591cd01 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -12,6 +12,7 @@ import dns.resolver from flask import current_app from lemur.extensions import metrics, sentry + class Zone: """ This class implements a PowerDNS zone in JSON. """ @@ -33,16 +34,11 @@ class Zone: """ Indicates whether the zone is setup as a PRIMARY or SECONDARY """ return self._data["kind"] -class Record: - """ - This class implements a PowerDNS record. - Accepts the response from the API call as the argument. - """ +class Record: + """ This class implements a PowerDNS record. """ def __init__(self, _data): - # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response. - # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response. self._data = _data @property @@ -62,44 +58,8 @@ class Record: return self._data["ttl"] -def _generate_header(): - """Function to generate the header for a request using the PowerDNS API Key""" - - api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME", "") - api_key = current_app.config.get("ACME_POWERDNS_APIKEY", "") - return {api_key_name: api_key} - - -def _get(path, params=None): - """ - Function to execute a GET request on the given URL (base_uri + path) with given params - Returns JSON response object - """ - base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") - resp = requests.get( - f"{base_uri}{path}", - headers=_generate_header(), - params=params, - verify=True, - ) - resp.raise_for_status() - return resp.json() - -def _patch(path, payload): - """ - Function to execute a Patch request on the given URL (base_uri + path) with given data - """ - base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") - resp = requests.patch( - f"{base_uri}{path}", - headers=_generate_header(), - data=json.dumps(payload) - ) - resp.raise_for_status() - - def get_zones(account_number): - """Get zones from the PowerDNS""" + """Retrieve authoritative zones from the PowerDNS API and return a list""" server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") path = f"/api/v1/servers/{server_id}/zones" zones = [] @@ -109,43 +69,13 @@ def get_zones(account_number): zones.append(zone.name) return zones -def _get_zone_name(domain, account_number): - """Get the matching zone for the given domain""" - zones = get_zones(account_number) - zone_name = "" - for z in zones: - if domain.endswith(z): - # Find the most specific zone possible for the domain - # Ex: If fqdn is a.b.c.com, there is a zone for c.com, - # and a zone for b.c.com, we want to use b.c.com. - if z.count(".") > zone_name.count("."): - zone_name = z - if not zone_name: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.fail", "counter", 1) - raise Exception(f"No PowerDNS zone found for domain: {domain}") - return zone_name def create_txt_record(domain, token, account_number): - """ - Create a TXT record for the given domain. - - The part of the domain that matches with the zone becomes the zone name. - The remainder becomes the owner name (referred to as node name here) - Example: Let's say we have a zone named "exmaple.com" in PowerDNS and we - get a request to create a cert for lemur.example.com - Domain - _acme-challenge.lemur.example.com - Matching zone - example.com - Owner name - _acme-challenge.lemur - """ - + """ Create a TXT record for the given domain and token and return a change_id tuple """ zone_name = _get_zone_name(domain, account_number) - node_name = domain[:-len(".".join(zone_name))] - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") zone_id = zone_name.join(".") domain_id = domain.join(".") - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" payload = { "rrsets": [ @@ -189,6 +119,146 @@ def create_txt_record(domain, token, account_number): change_id = (domain, token) return change_id + +def wait_for_dns_change(change_id, account_number=None): + """ + Checks if changes have propagated to DNS + Verifies both the authoritative DNS server and a public DNS server(Google <8.8.8.8> in our case) + Retries and waits until successful. + """ + domain, token = change_id + number_of_attempts = 20 + + nameserver = _get_authoritative_nameserver(domain) + status = False + for attempts in range(0, number_of_attempts): + status = _has_dns_propagated(domain, token, nameserver) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "status": status, + "message": "Record status on UltraDNS authoritative server" + } + current_app.logger.debug(log_data) + if status: + time.sleep(10) + break + time.sleep(10) + if status: + nameserver = _get_public_authoritative_nameserver() + for attempts in range(0, number_of_attempts): + status = _has_dns_propagated(domain, token, nameserver) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "status": status, + "message": "Record status on Public DNS" + } + current_app.logger.debug(log_data) + if status: + metrics.send(f"{function}.success", "counter", 1) + break + time.sleep(10) + if not status: + metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) + sentry.captureException(extra={"fqdn": str(domain), "txt_record": str(token)}) + + +def delete_txt_record(change_id, account_number, domain, token): + """ Delete the TXT record for the given domain and token """ + zone_name = _get_zone_name(domain, account_number) + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") + zone_id = zone_name.join(".") + domain_id = domain.join(".") + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + payload = { + "rrsets": [ + { + "name": f"{domain_id}", + "type": "TXT", + "ttl": "300", + "changetype": "DELETE", + "records": [ + { + "content": f"{token}", + "disabled": "false" + } + ], + "comments": [] + } + ] + } + + try: + _patch(path, payload) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token, + "message": "TXT record successfully deleted" + } + current_app.logger.debug(log_data) + except requests.exceptions.RequestException as e: + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "domain": domain, + "token": token, + "Exception": e, + "message": "Unable to delete TXT record" + } + current_app.logger.debug(log_data) + + +def _generate_header(): + """Generate a PowerDNS API header and return it as a dictionary""" + api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME", "") + api_key = current_app.config.get("ACME_POWERDNS_APIKEY", "") + return {api_key_name: api_key} + + +def _get(path, params=None): + """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ + base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") + resp = requests.get( + f"{base_uri}{path}", + headers=_generate_header(), + params=params, + verify=True, + ) + resp.raise_for_status() + return resp.json() + + +def _patch(path, payload): + """ Execute a Patch request on the given URL (base_uri + path) with given payload """ + base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") + resp = requests.patch( + f"{base_uri}{path}", + headers=_generate_header(), + data=json.dumps(payload) + ) + resp.raise_for_status() + + +def _get_zone_name(domain, account_number): + """Get most specific matching zone for the given domain and return as a String""" + zones = get_zones(account_number) + zone_name = "" + for z in zones: + if domain.endswith(z): + if z.count(".") > zone_name.count("."): + zone_name = z + if not zone_name: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.fail", "counter", 1) + raise Exception(f"No PowerDNS zone found for domain: {domain}") + return zone_name + + def _get_authoritative_nameserver(domain): """Get the authoritative nameserver for the given domain""" n = dns.name.from_text(domain) @@ -234,12 +304,9 @@ def _get_authoritative_nameserver(domain): def _get_public_authoritative_nameserver(): return "8.8.8.8" -def _has_dns_propagated(name, token, domain): - """ - Check whether the DNS change made by Lemur have propagated to the public DNS or not. - Invoked by wait_for_dns_change() function - """ +def _has_dns_propagated(name, token, domain): + """Check whether the DNS change has propagated to the public DNS""" txt_records = [] try: dns_resolver = dns.resolver.Resolver() @@ -260,65 +327,3 @@ def _has_dns_propagated(name, token, domain): return True return False - - -def wait_for_dns_change(change_id, account_number=None): - """ - Waits and checks if the DNS changes have propagated or not. - - First check the domains authoritative server. Once this succeeds, - we ask a public DNS server (Google <8.8.8.8> in our case). - """ - domain, token = change_id - number_of_attempts = 20 - - # Check if Record exists via DNS - nameserver = _get_authoritative_nameserver(domain) - for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(domain, token, nameserver) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "status": status, - "message": "Record status on ultraDNS authoritative server" - } - current_app.logger.debug(log_data) - if status: - time.sleep(10) - break - time.sleep(10) - if status: - nameserver = _get_public_authoritative_nameserver() - for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(domain, token, nameserver) - log_data = { - "function": function, - "fqdn": domain, - "status": status, - "message": "Record status on Public DNS" - } - current_app.logger.debug(log_data) - if status: - metrics.send(f"{function}.success", "counter", 1) - break - time.sleep(10) - if not status: - metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) - sentry.captureException(extra={"fqdn": str(domain), "txt_record": str(token)}) - return - -def delete_txt_record(change_id, account_number, domain, token): - """ - Delete the TXT record that was created in the create_txt_record() function. - - UltraDNS handles records differently compared to Dyn. It creates an RRSet - which is a set of records of the same type and owner. This means - that while deleting the record, we cannot delete any individual record from - the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are - being created for the same domain at the same time, the challenge TXT records - that are created will be added under the same RRSet. If the RRSet had more - than 1 record, then we create a new RRSet on UltraDNS minus the record that - has to be deleted. - """ - pass diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index d69f890c..be3a590a 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -21,30 +21,7 @@ class TestPowerdns(unittest.TestCase): } @patch("lemur.plugins.lemur_acme.powerdns.current_app") - @patch("lemur.extensions.metrics") - def test_powerdns_delete_txt_record(self, mock_metrics, mock_current_app): - domain = "_acme_challenge.test.example.com" - zone = "test.example.com" - token = "ABCDEFGHIJ" - account_number = "1234567890" - change_id = (domain, token) - mock_current_app.logger.debug = Mock() - powerdns.get_zone_name = Mock(return_value=zone) - powerdns._post = Mock() - powerdns._get = Mock() - powerdns._get.return_value = {'zoneName': 'test.example.com.com', - 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', - 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], - 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, - 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} - powerdns._delete = Mock() - mock_metrics.send = Mock() - powerdns.delete_txt_record(change_id, account_number, domain, token) - mock_current_app.logger.debug.assert_not_called() - mock_metrics.send.assert_not_called() - - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_powerdns_get_zones(self, mock_current_app): + def test_get_zones(self, mock_current_app): account_number = "1234567890" path = "a/b/c" zones = ['example.com', 'test.example.com'] @@ -63,7 +40,7 @@ class TestPowerdns(unittest.TestCase): result = powerdns.get_zones(account_number) self.assertEqual(result, zones) - def test_powerdns_get_zone_name(self): + def test_get_zone_name(self): zones = ['example.com', 'test.example.com'] zone = "test.example.com" domain = "_acme-challenge.test.example.com" @@ -72,19 +49,8 @@ class TestPowerdns(unittest.TestCase): result = powerdns._get_zone_name(domain, account_number) self.assertEqual(result, zone) - def mock_current_app_config_get(a, b): - """ Mock of current_app.config.get() """ - config = { - 'ACME_POWERDNS_APIKEYNAME': 'X-API-Key', - 'ACME_POWERDNS_APIKEY': 'KEY', - 'ACME_POWERDNS_DOMAIN': 'http://internal-dnshiddenmaster-1486232504.us-east-1.elb.amazonaws.com', - 'ACME_POWERDNS_SERVERID': 'localhost' - } - return config[a] - @patch("lemur.plugins.lemur_acme.powerdns.current_app") - # @patch("lemur.plugins.lemur_acme.powerdns.current_app.config.get", side_effect=mock_current_app_config_get) - def test_powerdns_create_txt_record(self, mock_current_app): + def test_create_txt_record(self, mock_current_app): domain = "_acme_challenge.test.example.com" zone = "test.example.com" token = "ABCDEFGHIJ" @@ -106,21 +72,50 @@ class TestPowerdns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.powerdns.current_app") @patch("lemur.extensions.metrics") - def test_powerdns_wait_for_dns_change(self, mock_metrics, mock_current_app): - powerdns._has_dns_propagated = Mock(return_value=True) + @patch("time.sleep") + def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app): nameserver = "1.1.1.1" powerdns._get_authoritative_nameserver = Mock(return_value=nameserver) + powerdns._has_dns_propagated = Mock(return_value=True) mock_metrics.send = Mock() + mock_sleep.return_value = False domain = "_acme-challenge.test.example.com" token = "ABCDEFGHIJ" change_id = (domain, token) mock_current_app.logger.debug = Mock() powerdns.wait_for_dns_change(change_id) - # mock_metrics.send.assert_not_called() - log_data = { + + auth_log_data = { + "function": "wait_for_dns_change", + "fqdn": domain, + "status": True, + "message": "Record status on UltraDNS authoritative server" + } + pub_log_data = { "function": "wait_for_dns_change", "fqdn": domain, "status": True, "message": "Record status on Public DNS" } - mock_current_app.logger.debug.assert_called_with(log_data) \ No newline at end of file + mock_current_app.logger.debug.assert_any_call(auth_log_data) + mock_current_app.logger.debug.assert_any_call(pub_log_data) + + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_delete_txt_record(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + powerdns._get_zone_name = Mock(return_value=zone) + mock_current_app.logger.debug = Mock() + mock_current_app.config.get = Mock(return_value="localhost") + powerdns._patch = Mock() + log_data = { + "function": "delete_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record successfully deleted" + } + powerdns.delete_txt_record(change_id, account_number, domain, token) + mock_current_app.logger.debug.assert_called_with(log_data) From 9984470b5846fbbe956e9b4432b1b01ca905d4c9 Mon Sep 17 00:00:00 2001 From: rajatsharma94 Date: Thu, 23 Jan 2020 12:35:57 +0100 Subject: [PATCH 041/150] fix fatal error in schema validator --- lemur/certificates/schemas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index c987e5fa..8f15542d 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -119,6 +119,9 @@ class CertificateInputSchema(CertificateCreationSchema): @validates_schema def validate_authority(self, data): + if 'authority' not in data: + raise ValidationError("Missing Authority.") + if isinstance(data["authority"], str): raise ValidationError("Authority not found.") From c465062673e83bf721c5fd69906445ad035aca05 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 23 Jan 2020 23:53:38 -0800 Subject: [PATCH 042/150] integrated PowerDNS plugin into dns_providers --- lemur/dns_providers/service.py | 1 + lemur/plugins/lemur_acme/plugin.py | 4 ++-- lemur/plugins/lemur_acme/powerdns.py | 8 ++++---- lemur/plugins/lemur_acme/tests/test_powerdns.py | 5 +---- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lemur/dns_providers/service.py b/lemur/dns_providers/service.py index 29f98a5b..7052b55b 100644 --- a/lemur/dns_providers/service.py +++ b/lemur/dns_providers/service.py @@ -99,6 +99,7 @@ def get_types(): }, {"name": "dyn"}, {"name": "ultradns"}, + {"name": "powerdns"}, ] }, ) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 93628905..8991efdf 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -377,7 +377,7 @@ class AcmeHandler(object): "dyn": dyn, "route53": route53, "ultradns": ultradns, - # "powerdns": powerdns, + "powerdns": powerdns } provider = provider_types.get(type) if not provider: @@ -437,7 +437,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "dyn": dyn, "route53": route53, "ultradns": ultradns, - # "powerdns": powerdns, + "powerdns": powerdns } provider = provider_types.get(type) if not provider: diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 9591cd01..e0a145e6 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -74,8 +74,8 @@ def create_txt_record(domain, token, account_number): """ Create a TXT record for the given domain and token and return a change_id tuple """ zone_name = _get_zone_name(domain, account_number) server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") - zone_id = zone_name.join(".") - domain_id = domain.join(".") + zone_id = zone_name + "." + domain_id = domain + "." path = f"/api/v1/servers/{server_id}/zones/{zone_id}" payload = { "rrsets": [ @@ -170,8 +170,8 @@ def delete_txt_record(change_id, account_number, domain, token): """ Delete the TXT record for the given domain and token """ zone_name = _get_zone_name(domain, account_number) server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") - zone_id = zone_name.join(".") - domain_id = domain.join(".") + zone_id = zone_name + "." + domain_id = domain + "." path = f"/api/v1/servers/{server_id}/zones/{zone_id}" payload = { "rrsets": [ diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index be3a590a..f1190732 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,8 +1,5 @@ import unittest -from requests.models import Response - -from mock import MagicMock, Mock, patch - +from mock import Mock, patch from lemur.plugins.lemur_acme import plugin, powerdns From 620f972635b9b16be68c0d3ba0c2199640fa030a Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 27 Jan 2020 11:04:49 +0100 Subject: [PATCH 043/150] Fixed an error Found out that I introduced an error when I changed code up for publishig. The certserv.py I use does not return the ID of the certificate created. For now I just leave the field empty. I will create another issue , so that the ID is filled up. --- lemur/plugins/lemur_adcs/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_adcs/plugin.py b/lemur/plugins/lemur_adcs/plugin.py index bc07ede3..a69afc90 100644 --- a/lemur/plugins/lemur_adcs/plugin.py +++ b/lemur/plugins/lemur_adcs/plugin.py @@ -46,7 +46,7 @@ class ADCSIssuerPlugin(IssuerPlugin): ) current_app.logger.info("Requesting CSR: {0}".format(csr)) current_app.logger.info("Issuer options: {0}".format(issuer_options)) - cert, req_id = ( + cert = ( ca_server.get_cert(csr, adcs_template, encoding="b64") .decode("utf-8") .replace("\r\n", "\n") @@ -54,7 +54,7 @@ class ADCSIssuerPlugin(IssuerPlugin): chain = ( ca_server.get_ca_cert(encoding="b64").decode("utf-8").replace("\r\n", "\n") ) - return cert, chain, req_id + return cert, chain, None def revoke_certificate(self, certificate, comments): raise NotImplementedError("Not implemented\n", self, certificate, comments) From 0129e97dfe779c2d46208979c232c812d795793c Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 13:56:24 -0600 Subject: [PATCH 044/150] address karma CVEs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b899176..20e1e356 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "gulp-util": "^3.0.1", "http-proxy": "~1.16.2", "jshint-stylish": "^2.2.1", - "karma": "~1.3.0", + "karma": "^4.4.1", "karma-jasmine": "^1.1.0", "main-bower-files": "^2.13.1", "merge-stream": "^1.0.1", From a0a8c155f6f2bdae75720b1165710c180c540c92 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 13:59:41 -0600 Subject: [PATCH 045/150] patch gulp-protractor --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 20e1e356..2cee77ab 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "gulp-notify": "^2.2.0", "gulp-plumber": "^1.1.0", "gulp-print": "^2.0.1", - "gulp-protractor": "3.0.0", + "gulp-protractor": "^4.1.1", "gulp-replace": "~0.5.3", "gulp-replace-task": "~0.11.0", "gulp-rev": "^7.1.2", From 7a2ca8969eec68fc932dd4c9b8660dbd9fe26d62 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 14:08:22 -0600 Subject: [PATCH 046/150] patch gulp-imagemin --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2cee77ab..df0bbe9a 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "gulp-flatten": "^0.3.1", "gulp-foreach": "0.1.0", "gulp-if": "^2.0.2", - "gulp-imagemin": "^3.1.1", + "gulp-imagemin": "^7.1.0", "gulp-inject": "~4.1.0", "gulp-jshint": "^2.0.4", "gulp-less": "^3.0.3", From 980360b94a1530e049aee88360e374a10eda73d7 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 14:09:57 -0600 Subject: [PATCH 047/150] patch gulp-less --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index df0bbe9a..a34fc745 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "gulp-imagemin": "^7.1.0", "gulp-inject": "~4.1.0", "gulp-jshint": "^2.0.4", - "gulp-less": "^3.0.3", + "gulp-less": "^4.0.1", "gulp-load-plugins": "^1.4.0", "gulp-minify-css": "^1.2.4", "gulp-minify-html": "~1.0.6", From 192ecb3ce0391267b6e43b6bc3e19ebd3aeecaf7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 28 Jan 2020 16:24:50 -0800 Subject: [PATCH 048/150] DNS provider: adding more logging --- lemur/dns_providers/cli.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index 72f9c874..aa0d97f5 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -1,8 +1,10 @@ from flask_script import Manager +import sys + from lemur.constants import SUCCESS_METRIC_STATUS from lemur.dns_providers.service import get_all_dns_providers, set_domains -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.plugins.base import plugins manager = Manager( @@ -19,12 +21,20 @@ def get_all_zones(): dns_providers = get_all_dns_providers() acme_plugin = plugins.get("acme-issuer") + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "", + } + for dns_provider in dns_providers: try: zones = acme_plugin.get_all_zones(dns_provider) set_domains(dns_provider, zones) except Exception as e: print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e)) + log_data["message"] = f"get all zones failed for {dns_provider} {e}." + sentry.captureException(extra=log_data) set_domains(dns_provider, []) status = SUCCESS_METRIC_STATUS From b91899fe9992c4b8f78f88d39e5c5297f2ada448 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 28 Jan 2020 19:13:28 -0800 Subject: [PATCH 049/150] created CLI options for testin ACME over dns. Examle: `acme dnstest -d _acme-chall.foo.com -t token1` --- __init__.py | 0 lemur/acme_providers/__init__.py | 0 lemur/acme_providers/cli.py | 92 ++++++++++++++ lemur/dns_providers/util.py | 103 +++++++++++++++ lemur/manage.py | 2 + lemur/plugins/lemur_acme/powerdns.py | 179 +++++++++------------------ 6 files changed, 254 insertions(+), 122 deletions(-) create mode 100644 __init__.py create mode 100644 lemur/acme_providers/__init__.py create mode 100644 lemur/acme_providers/cli.py create mode 100644 lemur/dns_providers/util.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lemur/acme_providers/__init__.py b/lemur/acme_providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py new file mode 100644 index 00000000..fcf426fa --- /dev/null +++ b/lemur/acme_providers/cli.py @@ -0,0 +1,92 @@ +import time +import json + +from flask_script import Manager +from flask import current_app + +from lemur.extensions import sentry +from lemur.extensions import metrics +from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins.lemur_acme.plugin import AcmeHandler + +manager = Manager( + usage="This provides ability to test ACME issuance" +) + + +@manager.option( + "-d", + "--domain", + dest="domain", + required=True, + help="Name of the Domain to store to (ex. \"_acme-chall.test.com\".", +) +@manager.option( + "-t", + "--token", + dest="token", + required=True, + help="Value of the Token to store in DNS as content.", +) +def dnstest(domain, token): + """ + Attempts to create, verify, and delete DNS TXT records with an autodetected provider. + """ + print("[+] Starting ACME Tests.") + change_id = (domain, token) + + acme_handler = AcmeHandler() + acme_handler.autodetect_dns_providers(domain) + if not acme_handler.dns_providers_for_domain[domain]: + metrics.send( + "get_authorizations_no_dns_provider_for_domain", "counter", 1 + ) + raise Exception(f"No DNS providers found for domain: {format(domain)}.") + + # Create TXT Records + for dns_provider in acme_handler.dns_providers_for_domain[domain]: + dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + + print(f"[+] Creating TXT Record in `{dns_provider.name}` provider") + change_id = dns_provider_plugin.create_txt_record(domain, token, account_number) + + print("[+] Verifying TXT Record has propagated to DNS.") + print("[+] Waiting 60 second before continuing...") + time.sleep(10) + + # Verify TXT Records + for dns_provider in acme_handler.dns_providers_for_domain[domain]: + dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + + try: + dns_provider_plugin.wait_for_dns_change(change_id, account_number) + print(f"[+] Verfied TXT Record in `{dns_provider.name}` provider") + except Exception: + metrics.send("complete_dns_challenge_error", "counter", 1) + sentry.captureException() + current_app.logger.debug( + f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " + f"{account_number}", + exc_info=True, + ) + print(f"[+] Unable to Verify TXT Record in `{dns_provider.name}` provider") + + time.sleep(10) + + # Delete TXT Records + for dns_provider in acme_handler.dns_providers_for_domain[domain]: + dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + + # TODO(csine@: Add Exception Handling + dns_provider_plugin.delete_txt_record(change_id, account_number, domain, token) + print(f"[+] Deleted TXT Record in `{dns_provider.name}` provider") + + status = SUCCESS_METRIC_STATUS + metrics.send("dnstest", "counter", 1, metric_tags={"status": status}) + print("[+] Done with ACME Tests.") diff --git a/lemur/dns_providers/util.py b/lemur/dns_providers/util.py new file mode 100644 index 00000000..6534f6eb --- /dev/null +++ b/lemur/dns_providers/util.py @@ -0,0 +1,103 @@ +import sys +import dns +import dns.exception +import dns.name +import dns.query +import dns.resolver +import re + +from flask import current_app +from lemur.extensions import metrics, sentry + + +class DNSError(Exception): + """Base class for DNS Exceptions.""" + pass + + +class BadDomainError(DNSError): + """Error for when a Bad Domain Name is given.""" + + def __init__(self, message): + self.message = message + + +class DNSResolveError(DNSError): + """Error for DNS Resolution Errors.""" + + def __init__(self, message): + self.message = message + + +def is_valid_domain(domain): + """Checks if a domain is syntactically valid and returns a bool""" + if len(domain) > 253: + return False + if domain[-1] == ".": + domain = domain[:-1] + fqdn_re = re.compile("(?=^.{1,254}$)(^(?:(?!\d+\.|-)[a-zA-Z0-9_\-]{1,63}(? 0: + rrset = response.authority[0] + else: + rrset = response.answer[0] + + rr = rrset[0] + if rr.rdtype != dns.rdatatype.SOA: + authority = rr.target + nameserver = default.query(authority).rrset[0].to_text() + + depth += 1 + + return nameserver + + +def get_dns_records(domain, rdtype, nameserver): + """Retrieves the DNS records matching the name and type and returns a list of records""" + # if not nameserver: + # nameserver = get_authoritative_nameserver(domain)[0] + + records = [] + try: + dns_resolver = dns.resolver.Resolver() + dns_resolver.nameservers = [nameserver] + dns_response = dns_resolver.query(domain, rdtype) + for rdata in dns_response: + for record in rdata.strings: + records.append(record.decode("utf-8")) + except dns.exception.DNSException: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.fail", "counter", 1) + return records diff --git a/lemur/manage.py b/lemur/manage.py index 7dd3b3b4..2fbbe893 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -17,6 +17,7 @@ from flask_migrate import Migrate, MigrateCommand, stamp from flask_script.commands import ShowUrls, Clean, Server from lemur.dns_providers.cli import manager as dns_provider_manager +from lemur.acme_providers.cli import manager as acme_manager from lemur.sources.cli import manager as source_manager from lemur.policies.cli import manager as policy_manager from lemur.reporting.cli import manager as report_manager @@ -584,6 +585,7 @@ def main(): manager.add_command("policy", policy_manager) manager.add_command("pending_certs", pending_certificate_manager) manager.add_command("dns_providers", dns_provider_manager) + manager.add_command("acme", acme_manager) manager.run() diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index e0a145e6..7e45d581 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -3,11 +3,7 @@ import requests import json import sys -import dns -import dns.exception -import dns.name -import dns.query -import dns.resolver +import lemur.dns_providers.util as dnsutil from flask import current_app from lemur.extensions import metrics, sentry @@ -63,8 +59,25 @@ def get_zones(account_number): server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") path = f"/api/v1/servers/{server_id}/zones" zones = [] - for elem in _get(path): - zone = Zone(elem) + try: + records = _get(path) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "message": "Retrieved Zones Successfully" + } + current_app.logger.debug(log_data) + except Exception as e: + records = _get(path) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "message": "Failed to Retrieve Zone Data" + } + current_app.logger.debug(log_data) + + for record in records: + zone = Zone(record) if zone.kind == 'Master': zones.append(zone.name) return zones @@ -80,14 +93,14 @@ def create_txt_record(domain, token, account_number): payload = { "rrsets": [ { - "name": f"{domain_id}", + "name": domain_id, "type": "TXT", - "ttl": "300", + "ttl": 300, "changetype": "REPLACE", "records": [ { - "content": f"{token}", - "disabled": "false" + "content": f"\"{token}\"", + "disabled": False } ], "comments": [] @@ -105,7 +118,7 @@ def create_txt_record(domain, token, account_number): "message": "TXT record successfully created" } current_app.logger.debug(log_data) - except requests.exceptions.RequestException as e: + except Exception as e: function = sys._getframe().f_code.co_name log_data = { "function": function, @@ -122,46 +135,36 @@ def create_txt_record(domain, token, account_number): def wait_for_dns_change(change_id, account_number=None): """ - Checks if changes have propagated to DNS - Verifies both the authoritative DNS server and a public DNS server(Google <8.8.8.8> in our case) + Checks the authoritative DNS Server to see if changes have propagated to DNS Retries and waits until successful. """ domain, token = change_id number_of_attempts = 20 - - nameserver = _get_authoritative_nameserver(domain) - status = False + zone_name = _get_zone_name(domain, account_number) + nameserver = dnsutil.get_authoritative_nameserver(zone_name) + record_found = False for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(domain, token, nameserver) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "status": status, - "message": "Record status on UltraDNS authoritative server" - } - current_app.logger.debug(log_data) - if status: - time.sleep(10) + txt_records = dnsutil.get_dns_records(domain, "TXT", nameserver) + for txt_record in txt_records: + if txt_record == token: + record_found = True + break + if record_found: break time.sleep(10) - if status: - nameserver = _get_public_authoritative_nameserver() - for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(domain, token, nameserver) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "status": status, - "message": "Record status on Public DNS" - } - current_app.logger.debug(log_data) - if status: - metrics.send(f"{function}.success", "counter", 1) - break - time.sleep(10) - if not status: + + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "status": record_found, + "message": "Record status on PowerDNS authoritative server" + } + current_app.logger.debug(log_data) + + if record_found: + metrics.send(f"{function}.success", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) + else: metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) sentry.captureException(extra={"fqdn": str(domain), "txt_record": str(token)}) @@ -176,14 +179,14 @@ def delete_txt_record(change_id, account_number, domain, token): payload = { "rrsets": [ { - "name": f"{domain_id}", + "name": domain_id, "type": "TXT", - "ttl": "300", + "ttl": 300, "changetype": "DELETE", "records": [ { - "content": f"{token}", - "disabled": "false" + "content": f"\"{token}\"", + "disabled": False } ], "comments": [] @@ -201,7 +204,7 @@ def delete_txt_record(change_id, account_number, domain, token): "message": "TXT record successfully deleted" } current_app.logger.debug(log_data) - except requests.exceptions.RequestException as e: + except Exception as e: function = sys._getframe().f_code.co_name log_data = { "function": function, @@ -217,7 +220,8 @@ def _generate_header(): """Generate a PowerDNS API header and return it as a dictionary""" api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME", "") api_key = current_app.config.get("ACME_POWERDNS_APIKEY", "") - return {api_key_name: api_key} + headers = {api_key_name: api_key} + return headers def _get(path, params=None): @@ -238,8 +242,8 @@ def _patch(path, payload): base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") resp = requests.patch( f"{base_uri}{path}", - headers=_generate_header(), - data=json.dumps(payload) + data=json.dumps(payload), + headers=_generate_header() ) resp.raise_for_status() @@ -258,72 +262,3 @@ def _get_zone_name(domain, account_number): raise Exception(f"No PowerDNS zone found for domain: {domain}") return zone_name - -def _get_authoritative_nameserver(domain): - """Get the authoritative nameserver for the given domain""" - n = dns.name.from_text(domain) - - depth = 2 - default = dns.resolver.get_default_resolver() - nameserver = default.nameservers[0] - - last = False - while not last: - s = n.split(depth) - - last = s[0].to_unicode() == u"@" - sub = s[1] - - query = dns.message.make_query(sub, dns.rdatatype.NS) - response = dns.query.udp(query, nameserver) - - rcode = response.rcode() - if rcode != dns.rcode.NOERROR: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.error", "counter", 1) - if rcode == dns.rcode.NXDOMAIN: - raise Exception("%s does not exist." % sub) - else: - raise Exception("Error %s" % dns.rcode.to_text(rcode)) - - if len(response.authority) > 0: - rrset = response.authority[0] - else: - rrset = response.answer[0] - - rr = rrset[0] - if rr.rdtype != dns.rdatatype.SOA: - authority = rr.target - nameserver = default.query(authority).rrset[0].to_text() - - depth += 1 - - return nameserver - - -def _get_public_authoritative_nameserver(): - return "8.8.8.8" - - -def _has_dns_propagated(name, token, domain): - """Check whether the DNS change has propagated to the public DNS""" - txt_records = [] - try: - dns_resolver = dns.resolver.Resolver() - dns_resolver.nameservers = [domain] - dns_response = dns_resolver.query(name, "TXT") - for rdata in dns_response: - for txt_record in rdata.strings: - txt_records.append(txt_record.decode("utf-8")) - except dns.exception.DNSException: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.fail", "counter", 1) - return False - - for txt_record in txt_records: - if txt_record == token: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.success", "counter", 1) - return True - - return False From f3039a1210a8ec0e9325f7ce1503fd8b9cebf2c0 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 11:05:46 -0800 Subject: [PATCH 050/150] removed accidently added __init__py file --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 From ef115ef2b12ee8bf4c7a7f90bb0b1bf176c51632 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 11:20:39 -0800 Subject: [PATCH 051/150] moving PowerDNS number_of_attempts to global config variable ACME_POWERDNS_RETRIES --- lemur/plugins/lemur_acme/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 7e45d581..688a84f2 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -139,7 +139,7 @@ def wait_for_dns_change(change_id, account_number=None): Retries and waits until successful. """ domain, token = change_id - number_of_attempts = 20 + number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", "") zone_name = _get_zone_name(domain, account_number) nameserver = dnsutil.get_authoritative_nameserver(zone_name) record_found = False From b885244aa77d571911121493c8700b35b3d868c2 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 11:26:53 -0800 Subject: [PATCH 052/150] fixing issue where set_domains() is still called when get_all_zones() throws an exception --- lemur/dns_providers/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index aa0d97f5..b14e0339 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -35,7 +35,6 @@ def get_all_zones(): print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e)) log_data["message"] = f"get all zones failed for {dns_provider} {e}." sentry.captureException(extra=log_data) - set_domains(dns_provider, []) status = SUCCESS_METRIC_STATUS From 969a7107fef2ed256eccce99a623eda5b22d36e1 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 13:12:09 -0800 Subject: [PATCH 053/150] fixed PowerDNS Tests --- lemur/plugins/lemur_acme/powerdns.py | 31 +++++++++-------- .../plugins/lemur_acme/tests/test_powerdns.py | 33 +++++++++---------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 688a84f2..1efe0752 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -224,6 +224,21 @@ def _generate_header(): return headers +def _get_zone_name(domain, account_number): + """Get most specific matching zone for the given domain and return as a String""" + zones = get_zones(account_number) + zone_name = "" + for z in zones: + if domain.endswith(z): + if z.count(".") > zone_name.count("."): + zone_name = z + if not zone_name: + function = sys._getframe().f_code.co_name + metrics.send(f"{function}.fail", "counter", 1) + raise Exception(f"No PowerDNS zone found for domain: {domain}") + return zone_name + + def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") @@ -246,19 +261,3 @@ def _patch(path, payload): headers=_generate_header() ) resp.raise_for_status() - - -def _get_zone_name(domain, account_number): - """Get most specific matching zone for the given domain and return as a String""" - zones = get_zones(account_number) - zone_name = "" - for z in zones: - if domain.endswith(z): - if z.count(".") > zone_name.count("."): - zone_name = z - if not zone_name: - function = sys._getframe().f_code.co_name - metrics.send(f"{function}.fail", "counter", 1) - raise Exception(f"No PowerDNS zone found for domain: {domain}") - return zone_name - diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index f1190732..4c483741 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -67,35 +67,34 @@ class TestPowerdns(unittest.TestCase): mock_current_app.logger.debug.assert_called_with(log_data) self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.dnsutil") @patch("lemur.plugins.lemur_acme.powerdns.current_app") @patch("lemur.extensions.metrics") @patch("time.sleep") - def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app): - nameserver = "1.1.1.1" - powerdns._get_authoritative_nameserver = Mock(return_value=nameserver) - powerdns._has_dns_propagated = Mock(return_value=True) - mock_metrics.send = Mock() - mock_sleep.return_value = False + def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil): domain = "_acme-challenge.test.example.com" - token = "ABCDEFGHIJ" + token = "ABCDEFG" + zone_name = "test.example.com" + nameserver = "1.1.1.1" change_id = (domain, token) + mock_records = (token,) + + mock_current_app.config.get = Mock(return_value=1) + powerdns._get_zone_name = Mock(return_value=zone_name) + mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) + mock_dnsutil.get_dns_records = Mock(return_value=mock_records) + mock_sleep.return_value = False + mock_metrics.send = Mock() mock_current_app.logger.debug = Mock() powerdns.wait_for_dns_change(change_id) - auth_log_data = { + log_data = { "function": "wait_for_dns_change", "fqdn": domain, "status": True, - "message": "Record status on UltraDNS authoritative server" + "message": "Record status on PowerDNS authoritative server" } - pub_log_data = { - "function": "wait_for_dns_change", - "fqdn": domain, - "status": True, - "message": "Record status on Public DNS" - } - mock_current_app.logger.debug.assert_any_call(auth_log_data) - mock_current_app.logger.debug.assert_any_call(pub_log_data) + mock_current_app.logger.debug.assert_called_with(log_data) @patch("lemur.plugins.lemur_acme.powerdns.current_app") def test_delete_txt_record(self, mock_current_app): From 48d8e1d235e04fb4370006e7f7da02a15d1a30cd Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 15:30:08 -0800 Subject: [PATCH 054/150] adding support for pip version >=19.3 by supporting change to PipSession() location in setup.py --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90c0b2f8..fa5a23bc 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,11 @@ from setuptools import setup, find_packages from subprocess import check_output import pip -if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0): +if tuple(map(int, pip.__version__.split('.'))) >= (19, 3, 0): + from pip._internal.network.session import PipSession + from pip._internal.req import parse_requirements + +elif tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0): from pip._internal.download import PipSession from pip._internal.req import parse_requirements else: From 75d4699c7a48f1e755a68b009ed7a3607b10a966 Mon Sep 17 00:00:00 2001 From: Ilya Makarov Date: Fri, 31 Jan 2020 22:52:59 +0300 Subject: [PATCH 055/150] Fix nginx ssl. Add env vars. Opt docker --- docker/.dockerignore | 3 +++ docker/Dockerfile | 14 +++++++------- docker/docker-compose.yml | 29 +++++++++++++++++++++++++++++ docker/entrypoint | 17 ++++++++++------- docker/lemur-env | 25 +++++++++++++++++++++++++ docker/nginx/default-ssl.conf | 2 +- docker/pgsql-env | 4 ++++ docker/src/lemur.conf.py | 12 ++++++++++++ 8 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 docker/.dockerignore create mode 100644 docker/docker-compose.yml create mode 100644 docker/lemur-env create mode 100644 docker/pgsql-env diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 00000000..2199292b --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,3 @@ +*-env +docker-compose.yml +Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile index f7d1caf7..5c80606f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,12 +8,6 @@ ENV gid 1337 ENV user lemur ENV group lemur -COPY entrypoint / -COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py -COPY supervisor.conf / -COPY nginx/default.conf /etc/nginx/conf.d/ -COPY nginx/default-ssl.conf /etc/nginx/conf.d/ - RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \ @@ -40,7 +34,6 @@ RUN addgroup -S ${group} -g ${gid} && \ curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ - chmod +x /entrypoint && \ mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ @@ -52,6 +45,13 @@ RUN npm install --unsafe-perm && \ node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \ apk del build-dependencies +COPY entrypoint / +COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py +COPY supervisor.conf / +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/default-ssl.conf /etc/nginx/conf.d/ + +RUN chmod +x /entrypoint WORKDIR / HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..77293f43 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,29 @@ +version: '3' + +services: + postgres: + image: "postgres:10" + restart: always + volumes: + - pg_data:/var/lib/postgresql/data + env_file: + - pgsql-env + + lemur: + # image: "netlix-lemur:latest" + build: . + depends_on: + - postgres + - redis + env_file: + - lemur-env + - pgsql-env + ports: + - 80:80 + - 443:443 + + redis: + image: "redis:alpine" + +volumes: + pg_data: {} diff --git a/docker/entrypoint b/docker/entrypoint index 6077167a..2a3a84e3 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +set -eo pipefail if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then echo "Database vars not set" @@ -7,22 +9,23 @@ fi export POSTGRES_PORT="${POSTGRES_PORT:-5432}" -echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile +export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}" + +export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" -source /etc/profile PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' echo " # Create Postgres trgm extension" -PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' echo " # Done" if [ -z "${SKIP_SSL}" ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi - mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf - mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a + [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf + [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then @@ -33,7 +36,7 @@ fi # fi echo " # Running init" -su lemur -c "python3 /opt/lemur/lemur/manage.py init" +su lemur -s /bin/bash -c "cd /opt/lemur/lemur; python3 /opt/lemur/lemur/manage.py init -p ${LEMUR_ADMIN_PASSWORD}" echo " # Done" # echo "Creating user" diff --git a/docker/lemur-env b/docker/lemur-env new file mode 100644 index 00000000..419a9858 --- /dev/null +++ b/docker/lemur-env @@ -0,0 +1,25 @@ +# SKIP_SSL=1 +# LEMUR_TOKEN_SECRET= +# LEMUR_DEFAULT_COUNTRY= +# LEMUR_DEFAULT_STATE= +# LEMUR_DEFAULT_LOCATION= +# LEMUR_DEFAULT_ORGANIZATION= +# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT= +# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer +# LEMUR_DEFAULT_AUTHORITY=cryptography +# MAIL_SERVER=mail.example.com +# MAIL_PORT=25 +# LEMUR_EMAIL=lemur@example.com +# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com'] +# LEMUR_TOKEN_SECRET= +# LEMUR_ENCRYPTION_KEYS=[''] +# DEBUG=True +# LDAP_DEBUG=True +# LDAP_AUTH=True +# LDAP_BIND_URI=ldap://example.com +# LDAP_BASE_DN=DC=example,DC=com +# LDAP_EMAIL_DOMAIN=example.com +# LDAP_USE_TLS=False +# LDAP_REQUIRED_GROUP=certificate-management-admins +# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'} +# LDAP_IS_ACTIVE_DIRECTORY=False diff --git a/docker/nginx/default-ssl.conf b/docker/nginx/default-ssl.conf index 86c770df..43d40f38 100644 --- a/docker/nginx/default-ssl.conf +++ b/docker/nginx/default-ssl.conf @@ -9,7 +9,7 @@ server { } server { - listen 443; + listen 443 ssl; server_name _; access_log /dev/stdout; error_log /dev/stderr; diff --git a/docker/pgsql-env b/docker/pgsql-env new file mode 100644 index 00000000..70d73fcb --- /dev/null +++ b/docker/pgsql-env @@ -0,0 +1,4 @@ +POSTGRES_USER=lemur +POSTGRES_PASSWORD=12345 +POSTGRES_DB=lemur +POSTGRES_HOST=postgres diff --git a/docker/src/lemur.conf.py b/docker/src/lemur.conf.py index a5f7e8b6..0f294b28 100644 --- a/docker/src/lemur.conf.py +++ b/docker/src/lemur.conf.py @@ -1,4 +1,6 @@ import os +from ast import literal_eval + _basedir = os.path.abspath(os.path.dirname(__file__)) CORS = os.environ.get("CORS") == "True" @@ -29,3 +31,13 @@ LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log')) SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') + +LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True" +LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True" +LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True" +LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI','')) +LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN','')) +LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN','')) +LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS','')) +LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP','')) +LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}") From be7736d350e57d0f0630fe957a30bc0c90a8daa0 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 31 Jan 2020 13:16:37 -0800 Subject: [PATCH 056/150] adding dns tests and assorted exception handling --- lemur/acme_providers/cli.py | 8 ++++---- lemur/dns_providers/util.py | 6 +----- lemur/plugins/lemur_acme/powerdns.py | 14 +++++++------- lemur/tests/test_dns_providers.py | 13 +++++++++++++ 4 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 lemur/tests/test_dns_providers.py diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index fcf426fa..a7510d36 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -10,7 +10,7 @@ from lemur.constants import SUCCESS_METRIC_STATUS from lemur.plugins.lemur_acme.plugin import AcmeHandler manager = Manager( - usage="This provides ability to test ACME issuance" + usage="Handles all ACME related tasks" ) @@ -30,7 +30,7 @@ manager = Manager( ) def dnstest(domain, token): """ - Attempts to create, verify, and delete DNS TXT records with an autodetected provider. + Create, verify, and delete DNS TXT records using an autodetected provider. """ print("[+] Starting ACME Tests.") change_id = (domain, token) @@ -53,7 +53,7 @@ def dnstest(domain, token): change_id = dns_provider_plugin.create_txt_record(domain, token, account_number) print("[+] Verifying TXT Record has propagated to DNS.") - print("[+] Waiting 60 second before continuing...") + print("[+] This step could take a while...") time.sleep(10) # Verify TXT Records @@ -64,7 +64,7 @@ def dnstest(domain, token): try: dns_provider_plugin.wait_for_dns_change(change_id, account_number) - print(f"[+] Verfied TXT Record in `{dns_provider.name}` provider") + print(f"[+] Verified TXT Record in `{dns_provider.name}` provider") except Exception: metrics.send("complete_dns_challenge_error", "counter", 1) sentry.captureException() diff --git a/lemur/dns_providers/util.py b/lemur/dns_providers/util.py index 6534f6eb..9154cb92 100644 --- a/lemur/dns_providers/util.py +++ b/lemur/dns_providers/util.py @@ -6,8 +6,7 @@ import dns.query import dns.resolver import re -from flask import current_app -from lemur.extensions import metrics, sentry +from lemur.extensions import metrics class DNSError(Exception): @@ -86,9 +85,6 @@ def get_authoritative_nameserver(domain): def get_dns_records(domain, rdtype, nameserver): """Retrieves the DNS records matching the name and type and returns a list of records""" - # if not nameserver: - # nameserver = get_authoritative_nameserver(domain)[0] - records = [] try: dns_resolver = dns.resolver.Resolver() diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 1efe0752..e30d7ca6 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -67,20 +67,20 @@ def get_zones(account_number): "message": "Retrieved Zones Successfully" } current_app.logger.debug(log_data) + for record in records: + zone = Zone(record) + if zone.kind == 'Master': + zones.append(zone.name) + return zones + except Exception as e: - records = _get(path) function = sys._getframe().f_code.co_name log_data = { "function": function, "message": "Failed to Retrieve Zone Data" } current_app.logger.debug(log_data) - - for record in records: - zone = Zone(record) - if zone.kind == 'Master': - zones.append(zone.name) - return zones + raise def create_txt_record(domain, token, account_number): diff --git a/lemur/tests/test_dns_providers.py b/lemur/tests/test_dns_providers.py new file mode 100644 index 00000000..42a86cca --- /dev/null +++ b/lemur/tests/test_dns_providers.py @@ -0,0 +1,13 @@ +import unittest +from mock import Mock, patch +from lemur.dns_providers import util as dnsutil + + +class TestDNSProvider(unittest.TestCase): + def test_is_valid_domain(self): + self.assertTrue(dnsutil.is_valid_domain("example.com")) + self.assertTrue(dnsutil.is_valid_domain("foo.bar.org")) + self.assertTrue(dnsutil.is_valid_domain("_acme-chall.example.com")) + self.assertFalse(dnsutil.is_valid_domain("e/xample.com")) + self.assertFalse(dnsutil.is_valid_domain("exam\ple.com")) + self.assertFalse(dnsutil.is_valid_domain("*.example.com")) From fb6d369130add96c55c6358476790db208d975c0 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 31 Jan 2020 16:18:22 -0800 Subject: [PATCH 057/150] removed unnecessary imports in test_dns_providers.py --- lemur/tests/test_dns_providers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/tests/test_dns_providers.py b/lemur/tests/test_dns_providers.py index 42a86cca..b8714a2d 100644 --- a/lemur/tests/test_dns_providers.py +++ b/lemur/tests/test_dns_providers.py @@ -1,5 +1,4 @@ import unittest -from mock import Mock, patch from lemur.dns_providers import util as dnsutil From 7dac0e1dd8978c055a6201d80e67a5cb30525362 Mon Sep 17 00:00:00 2001 From: csine-nflx <59457265+csine-nflx@users.noreply.github.com> Date: Fri, 31 Jan 2020 16:54:25 -0800 Subject: [PATCH 058/150] Update administration.rst --- docs/administration.rst | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index e292ae03..252a7dec 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -973,6 +973,41 @@ Will be the sender of all notifications, so ensure that it is verified with AWS. SES if the default notification gateway and will be used unless SMTP settings are configured in the application configuration settings. +PowerDNS ACME Plugin +~~~~~~~~~~~~~~~~~~~~~~ + +The following configuration properties are required to use the PowerDNS ACME Plugin for domain validation. + + +.. data:: ACME_POWERDNS_DOMAIN + :noindex: + + This is the FQDN for the PowerDNS API (without path) + + +.. data:: ACME_POWERDNS_SERVERID + :noindex: + + This is the ServerID attribute of the PowerDNS API Server (i.e. "localhost" + + +.. data:: ACME_POWERDNS_APIKEYNAME + :noindex: + + This is the Key name to use for authentication (i.e. "X-API-Key") + + +.. data:: ACME_POWERDNS_APIKEY + :noindex: + + This is the API Key to use for authentication (i.e. "Password") + + +.. data:: ACME_POWERDNS_RETRIES + :noindex: + + This is the number of times DNS Verification should be attempted (i.e. 20) + .. _CommandLineInterface: Command Line Interface @@ -1172,11 +1207,12 @@ Acme Kevin Glisson , Curtis Castrapel , Hossein Shafagh , - Mikhail Khodorovskiy + Mikhail Khodorovskiy , + Chad Sine :Type: Issuer :Description: - Adds support for the ACME protocol (including LetsEncrypt) with domain validation being handled Route53. + Adds support for the ACME protocol (including LetsEncrypt) with domain validation using several providers. Atlas From ac0282529ef9ec60050be4681c1ce278cb0eab04 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 3 Feb 2020 11:05:20 -0800 Subject: [PATCH 059/150] adding basic logging on success --- lemur/plugins/lemur_acme/plugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e38870d8..54493f97 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -183,6 +183,10 @@ class AcmeHandler(object): else: raise + current_app.logger.debug( + f"Successfully resolved Acme order: {order.uri}", exc_info=True + ) + pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate( From 53f81fb09f6ce6d8bb2e1522c78b8e9c694abe55 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 3 Feb 2020 18:58:31 -0800 Subject: [PATCH 060/150] updating based on suggestions in 2911 --- lemur/acme_providers/cli.py | 6 - lemur/dns_providers/util.py | 2 + lemur/plugins/lemur_acme/powerdns.py | 112 +++++++++--------- .../plugins/lemur_acme/tests/test_powerdns.py | 5 +- 4 files changed, 62 insertions(+), 63 deletions(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index a7510d36..310efad1 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -5,7 +5,6 @@ from flask_script import Manager from flask import current_app from lemur.extensions import sentry -from lemur.extensions import metrics from lemur.constants import SUCCESS_METRIC_STATUS from lemur.plugins.lemur_acme.plugin import AcmeHandler @@ -38,9 +37,6 @@ def dnstest(domain, token): acme_handler = AcmeHandler() acme_handler.autodetect_dns_providers(domain) if not acme_handler.dns_providers_for_domain[domain]: - metrics.send( - "get_authorizations_no_dns_provider_for_domain", "counter", 1 - ) raise Exception(f"No DNS providers found for domain: {format(domain)}.") # Create TXT Records @@ -66,7 +62,6 @@ def dnstest(domain, token): dns_provider_plugin.wait_for_dns_change(change_id, account_number) print(f"[+] Verified TXT Record in `{dns_provider.name}` provider") except Exception: - metrics.send("complete_dns_challenge_error", "counter", 1) sentry.captureException() current_app.logger.debug( f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " @@ -88,5 +83,4 @@ def dnstest(domain, token): print(f"[+] Deleted TXT Record in `{dns_provider.name}` provider") status = SUCCESS_METRIC_STATUS - metrics.send("dnstest", "counter", 1, metric_tags={"status": status}) print("[+] Done with ACME Tests.") diff --git a/lemur/dns_providers/util.py b/lemur/dns_providers/util.py index 9154cb92..cc8d9bb3 100644 --- a/lemur/dns_providers/util.py +++ b/lemur/dns_providers/util.py @@ -6,6 +6,7 @@ import dns.query import dns.resolver import re +from lemur.extensions import sentry from lemur.extensions import metrics @@ -94,6 +95,7 @@ def get_dns_records(domain, rdtype, nameserver): for record in rdata.strings: records.append(record.decode("utf-8")) except dns.exception.DNSException: + sentry.captureException() function = sys._getframe().f_code.co_name metrics.send(f"{function}.fail", "counter", 1) return records diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index e30d7ca6..4aa55cf8 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -3,11 +3,22 @@ import requests import json import sys +import lemur.common.utils as utils import lemur.dns_providers.util as dnsutil from flask import current_app from lemur.extensions import metrics, sentry +REQUIRED_VARIABLES = [ + "ACME_POWERDNS_APIKEYNAME", + "ACME_POWERDNS_APIKEY", + "ACME_POWERDNS_DOMAIN", +] + + +def _check_conf(): + utils.validate_conf(current_app, REQUIRED_VARIABLES) + class Zone: """ This class implements a PowerDNS zone in JSON. """ @@ -56,37 +67,37 @@ class Record: def get_zones(account_number): """Retrieve authoritative zones from the PowerDNS API and return a list""" - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") + _check_conf() + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/zones" zones = [] + function = sys._getframe().f_code.co_name + log_data = { + "function": function + } try: records = _get(path) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "message": "Retrieved Zones Successfully" - } + log_data["message"] = "Retrieved Zones Successfully" current_app.logger.debug(log_data) - for record in records: - zone = Zone(record) - if zone.kind == 'Master': - zones.append(zone.name) - return zones except Exception as e: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "message": "Failed to Retrieve Zone Data" - } + sentry.captureException() + log_data["message"] = "Failed to Retrieve Zone Data" current_app.logger.debug(log_data) raise + for record in records: + zone = Zone(record) + if zone.kind == 'Master': + zones.append(zone.name) + return zones + def create_txt_record(domain, token, account_number): """ Create a TXT record for the given domain and token and return a change_id tuple """ + _check_conf() zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") zone_id = zone_name + "." domain_id = domain + "." path = f"/api/v1/servers/{server_id}/zones/{zone_id}" @@ -107,26 +118,20 @@ def create_txt_record(domain, token, account_number): } ] } - + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token, + } try: _patch(path, payload) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token, - "message": "TXT record successfully created" - } + log_data["message"] = "TXT record successfully created" current_app.logger.debug(log_data) except Exception as e: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "domain": domain, - "token": token, - "Exception": e, - "message": "Unable to create TXT record" - } + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to create TXT record" current_app.logger.debug(log_data) change_id = (domain, token) @@ -138,8 +143,9 @@ def wait_for_dns_change(change_id, account_number=None): Checks the authoritative DNS Server to see if changes have propagated to DNS Retries and waits until successful. """ + _check_conf() domain, token = change_id - number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", "") + number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", "3") zone_name = _get_zone_name(domain, account_number) nameserver = dnsutil.get_authoritative_nameserver(zone_name) record_found = False @@ -166,13 +172,13 @@ def wait_for_dns_change(change_id, account_number=None): metrics.send(f"{function}.success", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) else: metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token}) - sentry.captureException(extra={"fqdn": str(domain), "txt_record": str(token)}) def delete_txt_record(change_id, account_number, domain, token): """ Delete the TXT record for the given domain and token """ + _check_conf() zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "") + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") zone_id = zone_name + "." domain_id = domain + "." path = f"/api/v1/servers/{server_id}/zones/{zone_id}" @@ -193,33 +199,27 @@ def delete_txt_record(change_id, account_number, domain, token): } ] } - + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token + } try: _patch(path, payload) - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "fqdn": domain, - "token": token, - "message": "TXT record successfully deleted" - } + log_data["message"] = "TXT record successfully deleted" current_app.logger.debug(log_data) except Exception as e: - function = sys._getframe().f_code.co_name - log_data = { - "function": function, - "domain": domain, - "token": token, - "Exception": e, - "message": "Unable to delete TXT record" - } + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record" current_app.logger.debug(log_data) def _generate_header(): """Generate a PowerDNS API header and return it as a dictionary""" - api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME", "") - api_key = current_app.config.get("ACME_POWERDNS_APIKEY", "") + api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME") + api_key = current_app.config.get("ACME_POWERDNS_APIKEY") headers = {api_key_name: api_key} return headers @@ -241,7 +241,7 @@ def _get_zone_name(domain, account_number): def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ - base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") + base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") resp = requests.get( f"{base_uri}{path}", headers=_generate_header(), @@ -254,7 +254,7 @@ def _get(path, params=None): def _patch(path, payload): """ Execute a Patch request on the given URL (base_uri + path) with given payload """ - base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN", "") + base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") resp = requests.patch( f"{base_uri}{path}", data=json.dumps(payload), diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 4c483741..c8b0a11e 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -31,6 +31,7 @@ class TestPowerdns(unittest.TestCase): {'account': '', 'dnssec': 'False', 'id': 'test.example.com.', 'kind': 'Master', 'last_check': 0, 'masters': [], 'name': 'test.example.com.', 'notified_serial': '2019112501', 'serial': '2019112501', 'url': '/api/v1/servers/localhost/zones/test.example.com.'}] + powerdns._check_conf = Mock() powerdns._get = Mock(path) powerdns._get.side_effect = [get_response] mock_current_app.config.get = Mock(return_value="localhost") @@ -53,6 +54,7 @@ class TestPowerdns(unittest.TestCase): token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) + powerdns._check_conf = Mock() powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() mock_current_app.config.get = Mock(return_value="localhost") @@ -77,8 +79,8 @@ class TestPowerdns(unittest.TestCase): zone_name = "test.example.com" nameserver = "1.1.1.1" change_id = (domain, token) + powerdns._check_conf = Mock() mock_records = (token,) - mock_current_app.config.get = Mock(return_value=1) powerdns._get_zone_name = Mock(return_value=zone_name) mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) @@ -103,6 +105,7 @@ class TestPowerdns(unittest.TestCase): token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) + powerdns._check_conf = Mock() powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() mock_current_app.config.get = Mock(return_value="localhost") From 48bccd6f68f539dbfb1a8be8f303486de6cdf28a Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 3 Feb 2020 19:08:28 -0800 Subject: [PATCH 061/150] moving _check_config() lower in file, near other private methods --- lemur/plugins/lemur_acme/powerdns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 4aa55cf8..39793eaf 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -16,10 +16,6 @@ REQUIRED_VARIABLES = [ ] -def _check_conf(): - utils.validate_conf(current_app, REQUIRED_VARIABLES) - - class Zone: """ This class implements a PowerDNS zone in JSON. """ @@ -216,6 +212,10 @@ def delete_txt_record(change_id, account_number, domain, token): current_app.logger.debug(log_data) +def _check_conf(): + utils.validate_conf(current_app, REQUIRED_VARIABLES) + + def _generate_header(): """Generate a PowerDNS API header and return it as a dictionary""" api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME") From 8ea54d7db2ead9af179c224bf8bf2587eb05260a Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 4 Feb 2020 14:50:56 -0800 Subject: [PATCH 062/150] removing exception if domain zone not found. Logging the issue instead --- lemur/plugins/lemur_acme/powerdns.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 39793eaf..617dbf7a 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -234,8 +234,12 @@ def _get_zone_name(domain, account_number): zone_name = z if not zone_name: function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "message": "No PowerDNS zone name found.", + } metrics.send(f"{function}.fail", "counter", 1) - raise Exception(f"No PowerDNS zone found for domain: {domain}") return zone_name From 5324290234fdb865a89fcdabe6c4811f0889060b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 4 Feb 2020 16:23:53 -0800 Subject: [PATCH 063/150] updating documentation based on feedback --- docs/administration.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 252a7dec..8f055147 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -988,7 +988,7 @@ The following configuration properties are required to use the PowerDNS ACME Plu .. data:: ACME_POWERDNS_SERVERID :noindex: - This is the ServerID attribute of the PowerDNS API Server (i.e. "localhost" + This is the ServerID attribute of the PowerDNS API Server (i.e. "localhost") .. data:: ACME_POWERDNS_APIKEYNAME @@ -1106,6 +1106,15 @@ All commands default to `~/.lemur/lemur.conf.py` if a configuration is not speci lemur notify +.. data:: acme + + Handles all ACME related tasks, like ACME plugin testing. + + :: + + lemur acme + + Sub-commands ------------ From bcdb3173bd0b08e0808955d534f745df4176652d Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 4 Feb 2020 18:23:17 -0800 Subject: [PATCH 064/150] ensuring that "3" is set as an integer instead of a string --- lemur/plugins/lemur_acme/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 617dbf7a..f3ad9965 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -141,7 +141,7 @@ def wait_for_dns_change(change_id, account_number=None): """ _check_conf() domain, token = change_id - number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", "3") + number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", 3) zone_name = _get_zone_name(domain, account_number) nameserver = dnsutil.get_authoritative_nameserver(zone_name) record_found = False From ca8e73286f6f0ade5ffb130e276397e9d6b96273 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 12 Feb 2020 15:10:24 -0800 Subject: [PATCH 065/150] fixed get_domains() to remove duplicate entries, updated usage and tests --- lemur/plugins/lemur_acme/plugin.py | 14 ++++---------- lemur/plugins/lemur_acme/tests/test_acme.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 8991efdf..95689a13 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -254,8 +254,9 @@ class AcmeHandler(object): domains = [options["common_name"]] if options.get("extensions"): - for name in options["extensions"]["sub_alt_names"]["names"]: - domains.append(name) + for dns_name in options["extensions"]["sub_alt_names"]["names"]: + if dns_name.value not in domains: + domains.append(dns_name.value) current_app.logger.debug("Got these domains: {0}".format(domains)) return domains @@ -640,15 +641,8 @@ class ACMEIssuerPlugin(IssuerPlugin): domains = self.acme.get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation - authz_domains = [] - for d in domains: - if type(d) == str: - authz_domains.append(d) - else: - authz_domains.append(d.value) - dns_authorization = authorization_service.create( - account_number, authz_domains, provider_type + account_number, domains, provider_type ) # Return id of the DNS Authorization return None, None, dns_authorization.id diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 04997ace..990a556e 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,4 +1,6 @@ import unittest + +from cryptography.x509 import DNSName from requests.models import Response from mock import MagicMock, Mock, patch @@ -74,12 +76,14 @@ class TestAcme(unittest.TestCase): @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") + @patch("time.sleep") def test_complete_dns_challenge_success( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) mock_authz = Mock() + mock_sleep.return_value = False mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True) mock_authz.authz = [] @@ -179,7 +183,7 @@ class TestAcme(unittest.TestCase): options = { "common_name": "test.netflix.net", "extensions": { - "sub_alt_names": {"names": ["test2.netflix.net", "test3.netflix.net"]} + "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]} }, } result = self.acme.get_domains(options) @@ -187,6 +191,19 @@ class TestAcme(unittest.TestCase): result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] ) + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_get_domains_san(self, mock_current_app): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net"] + ) + @patch( "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", return_value="test", From 6c7bb5f9b73fe9bca34bfadb2f8a1eae01a5bcd4 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Thu, 13 Feb 2020 07:35:35 +0100 Subject: [PATCH 066/150] Fixed TLS secret format ( #2913 ) The Plugin handled the TLS secret format wrong: it sent chain certificate instead of requested public certificate #2913 --- lemur/plugins/lemur_kubernetes/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index 62ffffda..f7ff00f7 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -96,7 +96,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): if secret_format == "TLS": secret["type"] = "kubernetes.io/tls" secret["data"] = { - "tls.crt": base64encode(cert_chain), + "tls.crt": base64encode(body), "tls.key": base64encode(private_key), } if secret_format == "Certificate": From 571c8bf42d83f32ee808840edd2287a665a2f6fc Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Thu, 13 Feb 2020 07:38:04 +0100 Subject: [PATCH 067/150] Error when validity_end date is empty #2905 this lines of code (114ff) in threw an error, when the validity_end date was empty: if options.get("validity_end") > arrow.utcnow().shift(years=2): raise Exception( "Verisign issued certificates cannot exceed two years in validity" ) Actually, they are not needed, because immidiately following is a check for an empty validity_end and for the length of the entered period. When I commented it out for testing, the error was gone and everything worked as expected. --- lemur/plugins/lemur_verisign/plugin.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 7bf517b7..6a49364f 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -111,11 +111,7 @@ def process_options(options): data["subject_alt_names"] = ",".join(get_additional_names(options)) - if options.get("validity_end") > arrow.utcnow().shift(years=2): - raise Exception( - "Verisign issued certificates cannot exceed two years in validity" - ) - + if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date if options.get("validity_end") < arrow.utcnow().shift(years=2, days=-1): From 2b849a65205a00ea129247263c4ac300f8ebd25d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 13 Feb 2020 15:58:07 -0800 Subject: [PATCH 068/150] Update plugin.py making lint happy --- lemur/plugins/lemur_verisign/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 6a49364f..3e7c383f 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -110,7 +110,6 @@ def process_options(options): } data["subject_alt_names"] = ",".join(get_additional_names(options)) - if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date From af21225918929e936ed5781246c3cca05fe44d4d Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 13 Feb 2020 16:38:33 -0800 Subject: [PATCH 069/150] adding logging on sucess and metric submission of URL for certificate issuance --- lemur/plugins/lemur_acme/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 54493f97..179285b3 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -172,7 +172,7 @@ class AcmeHandler(object): except (AcmeError, TimeoutError): sentry.captureException(extra={"order_url": str(order.uri)}) - metrics.send("request_certificate_error", "counter", 1) + metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri}) current_app.logger.error( f"Unable to resolve Acme order: {order.uri}", exc_info=True ) @@ -183,7 +183,8 @@ class AcmeHandler(object): else: raise - current_app.logger.debug( + metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri}) + current_app.logger.info( f"Successfully resolved Acme order: {order.uri}", exc_info=True ) From 8e3cc93d6ae64bd6b75b1d54a7903d87052bb96f Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Fri, 14 Feb 2020 07:50:18 +0100 Subject: [PATCH 070/150] Whitespaces in empty line 113 removed --- lemur/plugins/lemur_verisign/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 3e7c383f..a0e2d1cb 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -110,7 +110,7 @@ def process_options(options): } data["subject_alt_names"] = ",".join(get_additional_names(options)) - + if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date if options.get("validity_end") < arrow.utcnow().shift(years=2, days=-1): From fabcad1e46c44d86c7dfe69204fdf45cbf1c189c Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 15:52:24 +0100 Subject: [PATCH 071/150] New variable VERISIGN_PRODUCT_(authority.name) If there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type else default to "Server", to be compatoible with former versions. This enables the use of different Verisign authorities for differnt cert-products eg. EV or Standard Certs --- lemur/plugins/lemur_verisign/plugin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index a0e2d1cb..c74a71f1 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -98,10 +98,18 @@ def process_options(options): :param options: :return: dict or valid verisign options """ + + # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type + # else default to "Server", to be compatoible with former versions + authority = options.get("authority").name.upper() + product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) + if product_type is None: + product_type ="Server" + data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", - "certProductType": "Server", + "certProductType": product_type, "firstName": current_app.config.get("VERISIGN_FIRST_NAME"), "lastName": current_app.config.get("VERISIGN_LAST_NAME"), "signatureAlgorithm": "sha256WithRSAEncryption", From bfa953270d3840b9d96807b69c30466138f89b39 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 16:04:44 +0100 Subject: [PATCH 072/150] Fixed whitespace error --- lemur/plugins/lemur_verisign/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index c74a71f1..cd716e84 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -104,7 +104,7 @@ def process_options(options): authority = options.get("authority").name.upper() product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) if product_type is None: - product_type ="Server" + product_type = "Server" data = { "challenge": get_psuedo_random_string(), From 3693bc2d8be12cafce118c196fadf72e8606d672 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 16:09:25 +0100 Subject: [PATCH 073/150] removed whitespaces inserted by online editor --- lemur/plugins/lemur_verisign/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index cd716e84..ca606baf 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -98,14 +98,14 @@ def process_options(options): :param options: :return: dict or valid verisign options """ - + # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) if product_type is None: product_type = "Server" - + data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", From a70a49e4e9cffa757a608022e51e14646d766513 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 16:11:58 +0100 Subject: [PATCH 074/150] Update plugin.py --- lemur/plugins/lemur_verisign/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index ca606baf..6d9182df 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -98,14 +98,12 @@ def process_options(options): :param options: :return: dict or valid verisign options """ - # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) if product_type is None: product_type = "Server" - data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", From 1815c8997064130e903e172a7afc4a13df23f714 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sun, 16 Feb 2020 09:28:52 +0100 Subject: [PATCH 075/150] Made the change more elegant As suggested by @hosseinsh. This is of course more elegant. --- lemur/plugins/lemur_verisign/plugin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 6d9182df..0864657a 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -101,9 +101,7 @@ def process_options(options): # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() - product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) - if product_type is None: - product_type = "Server" + product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority), "Server") data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", From 3fd0d3e1416fcb79aedb122c83c8c23213967e11 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 17 Feb 2020 12:40:36 +0100 Subject: [PATCH 076/150] Added VERISIGN_INTERMEDIATE_ parameter When using the VERISIGN_PRODUCT_ Parameter one also has to add this parameter: VERISIGN_INTERMEDIATE_ = """ """ While doing this, I also added code, so the external_id field is filled with data from CA-Answer --- lemur/plugins/lemur_verisign/plugin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 0864657a..2d108071 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -209,7 +209,7 @@ class VerisignIssuerPlugin(IssuerPlugin): response = self.session.post(url, data=data) try: - cert = handle_response(response.content)["Response"]["Certificate"] + response_dict = handle_response(response.content) except KeyError: metrics.send( "verisign_create_certificate_error", @@ -221,8 +221,12 @@ class VerisignIssuerPlugin(IssuerPlugin): extra={"common_name": issuer_options.get("common_name", "")} ) raise Exception(f"Error with Verisign: {response.content}") - # TODO add external id - return cert, current_app.config.get("VERISIGN_INTERMEDIATE"), None + # DONE: TODO add external id + authority = issuer_options.get("authority").name.upper() + cert = response_dict['Response']['Certificate'] + external_id = response_dict['Response']['Transaction_ID'] + chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) + return cert, chain, external_id @staticmethod def create_authority(options): From ed3472d029c88b678e1e9e5104151bd888b207b4 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 17 Feb 2020 15:21:29 +0100 Subject: [PATCH 077/150] Update plugin.py --- lemur/plugins/lemur_verisign/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 2d108071..479f3215 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -209,7 +209,7 @@ class VerisignIssuerPlugin(IssuerPlugin): response = self.session.post(url, data=data) try: - response_dict = handle_response(response.content) + response_dict = handle_response(response.content) except KeyError: metrics.send( "verisign_create_certificate_error", @@ -225,7 +225,7 @@ class VerisignIssuerPlugin(IssuerPlugin): authority = issuer_options.get("authority").name.upper() cert = response_dict['Response']['Certificate'] external_id = response_dict['Response']['Transaction_ID'] - chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) + chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) return cert, chain, external_id @staticmethod From e75df1ddc92a79dc197c698d38078f8c3d80157a Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 17 Feb 2020 19:04:20 +0100 Subject: [PATCH 078/150] Update plugin.py --- lemur/plugins/lemur_verisign/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 479f3215..f913861c 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -221,10 +221,11 @@ class VerisignIssuerPlugin(IssuerPlugin): extra={"common_name": issuer_options.get("common_name", "")} ) raise Exception(f"Error with Verisign: {response.content}") - # DONE: TODO add external id authority = issuer_options.get("authority").name.upper() cert = response_dict['Response']['Certificate'] - external_id = response_dict['Response']['Transaction_ID'] + external_id = None + if 'Transaction_ID' in response_dict['Response'].keys(): + external_id = response_dict['Response']['Transaction_ID'] chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) return cert, chain, external_id From 64a7faffd982213acf566f8b9b79229ba559c2a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 18:30:10 +0000 Subject: [PATCH 079/150] Bump bleach from 3.1.0 to 3.1.1 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.0...v3.1.1) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d1423888..224789f6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,37 +5,39 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.0 # via readme-renderer +bleach==3.1.1 # via readme-renderer certifi==2019.11.28 # via requests +cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests +cryptography==2.8 # via secretstorage docutils==0.15.2 # via readme-renderer flake8==3.5.0 identify==1.4.9 # via pre-commit idna==2.8 # via requests -importlib-metadata==1.3.0 # via keyring, pre-commit, twine invoke==1.3.0 +jeepney==0.4.2 # via secretstorage keyring==21.0.0 # via twine mccabe==0.6.1 # via flake8 -more-itertools==8.0.2 # via zipp nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine pre-commit==1.21.0 pycodestyle==2.3.1 # via flake8 +pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.5.2 # via readme-renderer pyyaml==5.2 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine -six==1.13.0 # via bleach, cfgv, pre-commit, readme-renderer +secretstorage==3.1.2 # via keyring +six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.41.1 # via twine twine==3.1.1 urllib3==1.25.7 # via requests virtualenv==16.7.9 # via pre-commit webencodings==0.5.1 # via bleach -zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 318292704d3c2167c9291bbb26e77ca2df8c543e Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 14:29:17 -0800 Subject: [PATCH 080/150] fixing default/max DigiCert validity values --- lemur/plugins/lemur_digicert/plugin.py | 88 +++--- .../lemur_digicert/tests/test_digicert.py | 266 ++++++++++-------- 2 files changed, 202 insertions(+), 152 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 88ea5b6b..b9508357 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -14,21 +14,17 @@ .. moduleauthor:: Kevin Glisson """ import json + import arrow -import requests - import pem -from retrying import retry - -from flask import current_app - +import requests from cryptography import x509 - -from lemur.extensions import metrics +from flask import current_app from lemur.common.utils import validate_conf -from lemur.plugins.bases import IssuerPlugin, SourcePlugin - +from lemur.extensions import metrics from lemur.plugins import lemur_digicert as digicert +from lemur.plugins.bases import IssuerPlugin, SourcePlugin +from retrying import retry def log_status_code(r, *args, **kwargs): @@ -64,24 +60,38 @@ def signature_hash(signing_algorithm): raise Exception("Unsupported signing algorithm.") -def determine_validity_years(end_date): +def determine_validity_years(years): """Given an end date determine how many years into the future that date is. + :param years: + :return: validity in years + """ + default_years = current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) + max_years = current_app.config.get("DIGICERT_MAX_VALIDITY", default_years) + + if years > max_years: + return max_years + if years not in [1, 2, 3]: + return default_years + else: + return years + + +def determine_end_date(end_date): + """ + Determine appropriate end date :param end_date: - :return: str validity in years + :return: validity_end """ - now = arrow.utcnow() + default_years = current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) + max_validity_end = arrow.utcnow().shift(years=current_app.config.get("DIGICERT_MAX_VALIDITY", default_years)) - if end_date < now.shift(years=+1): - return 1 - elif end_date < now.shift(years=+2): - return 2 - elif end_date < now.shift(years=+3): - return 3 + if not end_date: + end_date = arrow.utcnow().shift(years=default_years) - raise Exception( - "DigiCert issued certificates cannot exceed three" " years in validity" - ) + if end_date > max_validity_end: + end_date = max_validity_end + return end_date def get_additional_names(options): @@ -107,12 +117,6 @@ def map_fields(options, csr): :param csr: :return: dict or valid DigiCert options """ - if not options.get("validity_years"): - if not options.get("validity_end"): - options["validity_years"] = current_app.config.get( - "DIGICERT_DEFAULT_VALIDITY", 1 - ) - data = dict( certificate={ "common_name": options["common_name"], @@ -125,9 +129,11 @@ def map_fields(options, csr): data["certificate"]["dns_names"] = get_additional_names(options) if options.get("validity_years"): - data["validity_years"] = options["validity_years"] + data["validity_years"] = determine_validity_years(options.get("validity_years")) + elif options.get("validity_end"): + data["custom_expiration_date"] = determine_end_date(options.get("validity_end")).format("YYYY-MM-DD") else: - data["custom_expiration_date"] = options["validity_end"].format("YYYY-MM-DD") + data["validity_years"] = determine_validity_years(0) if current_app.config.get("DIGICERT_PRIVATE", False): if "product" in data: @@ -144,18 +150,15 @@ def map_cis_fields(options, csr): :param options: :param csr: - :return: + :return: data """ - if not options.get("validity_years"): - if not options.get("validity_end"): - options["validity_end"] = arrow.utcnow().shift( - years=current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) - ) - options["validity_years"] = determine_validity_years(options["validity_end"]) + + if options.get("validity_years"): + validity_end = determine_end_date(arrow.utcnow().shift(years=options["validity_years"])) + elif options.get("validity_end"): + validity_end = determine_end_date(options.get("validity_end")) else: - options["validity_end"] = arrow.utcnow().shift( - years=options["validity_years"] - ) + validity_end = determine_end_date(False) data = { "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES", {}).get(options['authority'].name), @@ -164,7 +167,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": options["validity_end"].format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z" }, "organization": { "name": options["organization"], @@ -173,7 +176,8 @@ def map_cis_fields(options, csr): } # possibility to default to a SIGNING_ALGORITHM for a given profile if current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name): - data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name) + data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get( + options['authority'].name) return data diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 77b0a1fa..b9576f51 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -1,117 +1,125 @@ -import pytest -import arrow import json -from unittest.mock import patch - -from freezegun import freeze_time - -from lemur.tests.vectors import CSR_STR +import arrow +import pytest from cryptography import x509 +from freezegun import freeze_time +from lemur.plugins.lemur_digicert import plugin +from lemur.tests.vectors import CSR_STR +from mock import Mock, patch -def test_map_fields_with_validity_end_and_start(app): - from lemur.plugins.lemur_digicert.plugin import map_fields - - names = [u"one.example.com", u"two.example.com", u"three.example.com"] - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "validity_end": arrow.get(2017, 5, 7), - "validity_start": arrow.get(2016, 10, 30), - } - - data = map_fields(options, CSR_STR) - - assert data == { - "certificate": { - "csr": CSR_STR, - "common_name": "example.com", - "dns_names": names, - "signature_hash": "sha256", - }, - "organization": {"id": 111111}, - "custom_expiration_date": arrow.get(2017, 5, 7).format("YYYY-MM-DD"), +def config_mock(*args): + values = { + "DIGICERT_ORG_ID": 111111, + "DIGICERT_PRIVATE": False, + "DIGICERT_DEFAULT_SIGNING_ALGORITHM": "sha256", + "DIGICERT_DEFAULT_VALIDITY": 1, + "DIGICERT_MAX_VALIDITY": 2, + "DIGICERT_CIS_PROFILE_NAMES": {"digicert": 'digicert'}, + "DIGICERT_CIS_SIGNING_ALGORITHMS": {"verisign-sha1-intermediary": 'sha1'}, } + return values[args[0]] -def test_map_fields_with_validity_years(app): - from lemur.plugins.lemur_digicert.plugin import map_fields - - names = [u"one.example.com", u"two.example.com", u"three.example.com"] - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "validity_years": 2, - "validity_end": arrow.get(2017, 10, 30), - } - - data = map_fields(options, CSR_STR) - - assert data == { - "certificate": { - "csr": CSR_STR, - "common_name": "example.com", - "dns_names": names, - "signature_hash": "sha256", - }, - "organization": {"id": 111111}, - "validity_years": 2, - } +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_determine_validity_years(mock_current_app): + mock_current_app.config.get = Mock(return_value=2) + assert plugin.determine_validity_years(1) == 1 + assert plugin.determine_validity_years(0) == 2 + assert plugin.determine_validity_years(3) == 2 -def test_map_cis_fields(app, authority): - from lemur.plugins.lemur_digicert.plugin import map_cis_fields - - names = [u"one.example.com", u"two.example.com", u"three.example.com"] - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "organization": "Example, Inc.", - "organizational_unit": "Example Org", - "validity_end": arrow.get(2017, 5, 7), - "validity_start": arrow.get(2016, 10, 30), - "authority": authority, - } - - data = map_cis_fields(options, CSR_STR) - - assert data == { - "common_name": "example.com", - "csr": CSR_STR, - "additional_dns_names": names, - "signature_hash": "sha256", - "organization": {"name": "Example, Inc.", "units": ["Example Org"]}, - "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" - }, - "profile_name": None, - } - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "organization": "Example, Inc.", - "organizational_unit": "Example Org", - "validity_years": 2, - "authority": authority, - } - +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_determine_end_date(mock_current_app): + mock_current_app.config.get = Mock(return_value=2) with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime): - data = map_cis_fields(options, CSR_STR) + assert arrow.get(2018, 11, 3) == plugin.determine_end_date(0) + assert arrow.get(2018, 5, 7) == plugin.determine_end_date(arrow.get(2018, 5, 7)) + assert arrow.get(2018, 11, 3) == plugin.determine_end_date(arrow.get(2020, 5, 7)) - assert data == { + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_fields_with_validity_years(mock_current_app): + mock_current_app.config.get = Mock(side_effect=config_mock) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "validity_years": 2 + } + expected = { + "certificate": { + "csr": CSR_STR, + "common_name": "example.com", + "dns_names": names, + "signature_hash": "sha256", + }, + "organization": {"id": 111111}, + "validity_years": 2, + } + assert expected == plugin.map_fields(options, CSR_STR) + + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_fields_with_validity_end_and_start(mock_current_app): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7)) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "validity_end": arrow.get(2017, 5, 7), + "validity_start": arrow.get(2016, 10, 30), + } + + expected = { + "certificate": { + "csr": CSR_STR, + "common_name": "example.com", + "dns_names": names, + "signature_hash": "sha256", + }, + "organization": {"id": 111111}, + "custom_expiration_date": arrow.get(2017, 5, 7).format("YYYY-MM-DD"), + } + + assert expected == plugin.map_fields(options, CSR_STR) + + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_cis_fields_with_validity_years(mock_current_app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2018, 11, 3)) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_years": 2, + "authority": authority, + } + + expected = { "common_name": "example.com", "csr": CSR_STR, "additional_dns_names": names, @@ -123,21 +131,59 @@ def test_map_cis_fields(app, authority): "profile_name": None, } + assert expected == plugin.map_cis_fields(options, CSR_STR) -def test_signature_hash(app): - from lemur.plugins.lemur_digicert.plugin import signature_hash - assert signature_hash(None) == "sha256" - assert signature_hash("sha256WithRSA") == "sha256" - assert signature_hash("sha384WithRSA") == "sha384" - assert signature_hash("sha512WithRSA") == "sha512" +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7)) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_end": arrow.get(2017, 5, 7), + "validity_start": arrow.get(2016, 10, 30), + "authority": authority + } + + expected = { + "common_name": "example.com", + "csr": CSR_STR, + "additional_dns_names": names, + "signature_hash": "sha256", + "organization": {"name": "Example, Inc.", "units": ["Example Org"]}, + "validity": { + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" + }, + "profile_name": None, + } + + assert expected == plugin.map_cis_fields(options, CSR_STR) + + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_signature_hash(mock_current_app, app): + mock_current_app.config.get = Mock(side_effect=config_mock) + assert plugin.signature_hash(None) == "sha256" + assert plugin.signature_hash("sha256WithRSA") == "sha256" + assert plugin.signature_hash("sha384WithRSA") == "sha384" + assert plugin.signature_hash("sha512WithRSA") == "sha512" with pytest.raises(Exception): - signature_hash("sdfdsf") + plugin.signature_hash("sdfdsf") def test_issuer_plugin_create_certificate( - certificate_="""\ + certificate_="""\ -----BEGIN CERTIFICATE----- abc -----END CERTIFICATE----- From 6c46481ffd9dd3dc0479a6de351b51fceb653091 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 14:40:50 -0800 Subject: [PATCH 081/150] simplifying return statement for validity years --- lemur/plugins/lemur_digicert/plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index b9508357..e5c4b2ce 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -72,8 +72,7 @@ def determine_validity_years(years): return max_years if years not in [1, 2, 3]: return default_years - else: - return years + return years def determine_end_date(end_date): From fdc1e20c234303adc27ea7768b72978d6bf5d80b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 17:27:15 -0800 Subject: [PATCH 082/150] updating config_mock defaults --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index b9576f51..1e9ebca4 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -17,7 +17,7 @@ def config_mock(*args): "DIGICERT_DEFAULT_VALIDITY": 1, "DIGICERT_MAX_VALIDITY": 2, "DIGICERT_CIS_PROFILE_NAMES": {"digicert": 'digicert'}, - "DIGICERT_CIS_SIGNING_ALGORITHMS": {"verisign-sha1-intermediary": 'sha1'}, + "DIGICERT_CIS_SIGNING_ALGORITHMS": {"digicert": 'digicert'}, } return values[args[0]] From a873d69859d2a68d3dd36a2981a721c951ec19ef Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 18:24:48 -0800 Subject: [PATCH 083/150] adding documentation for DIGICERT_MAX_VALIDITY --- docs/administration.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 8f055147..341eda6a 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -735,6 +735,12 @@ The following configuration properties are required to use the Digicert issuer p This is the default validity (in years), if no end date is specified. (Default: 1) +.. data:: DIGICERT_MAX_VALIDITY + :noindex: + + This is the maximum validity (in years). (Default: DIGICERT_DEFAULT_VALIDITY) + + .. data:: DIGICERT_PRIVATE :noindex: From 9ef538305d8a91b06cd128945fe8f19e063ddc8a Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 4 Mar 2020 11:45:47 -0800 Subject: [PATCH 084/150] updating default language for digicert max validity option --- docs/administration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 341eda6a..6cbf42bd 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -738,7 +738,7 @@ The following configuration properties are required to use the Digicert issuer p .. data:: DIGICERT_MAX_VALIDITY :noindex: - This is the maximum validity (in years). (Default: DIGICERT_DEFAULT_VALIDITY) + This is the maximum validity (in years). (Default: value of DIGICERT_DEFAULT_VALIDITY) .. data:: DIGICERT_PRIVATE From c0004e506e538c849e6c69c6e7b42640256dc64f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 4 Mar 2020 14:50:44 -0800 Subject: [PATCH 085/150] removing 2 year option from Lemur certificate request form --- .../app/angular/certificates/certificate/tracking.tpl.html | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index 7ac2107f..027add0f 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -140,7 +140,6 @@
From 5dfb6acb17eaf34ce4970da3154934113188577b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 14:59:21 -0800 Subject: [PATCH 086/150] adding support for ACME_POWERDNS_VERIFY option to support CA Bundles and disabling Server validation --- docs/administration.rst | 9 +++++++++ lemur/plugins/lemur_acme/powerdns.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 8f055147..ea9537da 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1008,6 +1008,15 @@ The following configuration properties are required to use the PowerDNS ACME Plu This is the number of times DNS Verification should be attempted (i.e. 20) + +.. data:: ACME_POWERDNS_VERIFY + :noindex: + + This configures how PowerDNS verifies TLS certificates. The PowerDNS Plugin relies on the requests library, supported options are as follows: + * True: Verifies the certificate chains to a known publicly-trusted CA. (Default) + * False: Disable certificate validation (Not Recommended) + * File/Dir path to CA Bundle: Verify that the certificate chains to a Certificate Authority in the provided CA bundle. + .. _CommandLineInterface: Command Line Interface diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index f3ad9965..1b7cf1d4 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -246,11 +246,12 @@ def _get_zone_name(domain, account_number): def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") + verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.get( f"{base_uri}{path}", headers=_generate_header(), params=params, - verify=True, + verify=verify_value, ) resp.raise_for_status() return resp.json() @@ -259,9 +260,11 @@ def _get(path, params=None): def _patch(path, payload): """ Execute a Patch request on the given URL (base_uri + path) with given payload """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") + verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.patch( f"{base_uri}{path}", data=json.dumps(payload), - headers=_generate_header() + headers=_generate_header(), + verify=verify_value, ) resp.raise_for_status() From b85fe2f2b5c95ba0d2f3a31c22d5daeade8ca378 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 15:03:43 -0800 Subject: [PATCH 087/150] updated documentation language --- docs/administration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index ea9537da..df3a5cff 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1012,10 +1012,10 @@ The following configuration properties are required to use the PowerDNS ACME Plu .. data:: ACME_POWERDNS_VERIFY :noindex: - This configures how PowerDNS verifies TLS certificates. The PowerDNS Plugin relies on the requests library, supported options are as follows: - * True: Verifies the certificate chains to a known publicly-trusted CA. (Default) - * False: Disable certificate validation (Not Recommended) - * File/Dir path to CA Bundle: Verify that the certificate chains to a Certificate Authority in the provided CA bundle. + This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options: + * True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) + * False: Disables certificate validation (Not Recommended) + * File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. .. _CommandLineInterface: From 771e72187a48b33c6ccea78aaa5e8b58907f8f76 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 15:24:56 -0800 Subject: [PATCH 088/150] updates based on feedback --- docs/administration.rst | 2 +- lemur/plugins/lemur_acme/powerdns.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index df3a5cff..d2dcf4c2 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1012,7 +1012,7 @@ The following configuration properties are required to use the PowerDNS ACME Plu .. data:: ACME_POWERDNS_VERIFY :noindex: - This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options: + This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter: * True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) * False: Disables certificate validation (Not Recommended) * File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 1b7cf1d4..a26faaac 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -251,7 +251,7 @@ def _get(path, params=None): f"{base_uri}{path}", headers=_generate_header(), params=params, - verify=verify_value, + verify=verify_value ) resp.raise_for_status() return resp.json() @@ -265,6 +265,6 @@ def _patch(path, payload): f"{base_uri}{path}", data=json.dumps(payload), headers=_generate_header(), - verify=verify_value, + verify=verify_value ) resp.raise_for_status() From 6227e4aa89e0064de790c1152c4f045c11dfe670 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 16:51:21 -0800 Subject: [PATCH 089/150] fixing formatting of ACME_POWERDNS_VERIFY options --- docs/administration.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index d2dcf4c2..0db374ff 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1013,9 +1013,12 @@ The following configuration properties are required to use the PowerDNS ACME Plu :noindex: This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter: - * True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) - * False: Disables certificate validation (Not Recommended) - * File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. + + True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) + + False: Disables certificate validation (Not Recommended) + + File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. .. _CommandLineInterface: From 921d52b3608db7d2c9ce3dab0c64a12bb6726e14 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 13 Mar 2020 00:03:31 -0700 Subject: [PATCH 090/150] fixing get_dns_challenge() logic so duplicate domains (such as wildcard and not wildcard) do not match the wrong authorziations --- lemur/plugins/lemur_acme/plugin.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 91781966..e566352c 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -54,18 +54,30 @@ class AcmeHandler(object): current_app.logger.error(f"Unable to fetch DNS Providers: {e}") self.all_dns_providers = [] - def find_dns_challenge(self, host, authorizations): + def get_dns_challenges(self, host, authorizations): + """Get final domain to validate and dns challenges for it""" + + domain_to_validate, is_wildcard = self.strip_wildcard(host) dns_challenges = [] for authz in authorizations: - if not authz.body.identifier.value.lower() == host.lower(): + if not authz.body.identifier.value.lower() == domain_to_validate.lower(): + continue + if is_wildcard and not authz.body.wildcard: + continue + if not is_wildcard and authz.body.wildcard: continue for combo in authz.body.challenges: if isinstance(combo.chall, challenges.DNS01): dns_challenges.append(combo) - return dns_challenges - def maybe_remove_wildcard(self, host): - return host.replace("*.", "") + return domain_to_validate, dns_challenges + + def strip_wildcard(self, host): + """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" + prefix = "*." + if host.startswith(prefix): + return host[len(prefix):], True + return host, False def maybe_add_extension(self, host, dns_provider_options): if dns_provider_options and dns_provider_options.get( @@ -86,9 +98,7 @@ class AcmeHandler(object): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) change_ids = [] - - host_to_validate = self.maybe_remove_wildcard(host) - dns_challenges = self.find_dns_challenge(host_to_validate, order.authorizations) + host_to_validate, dns_challenges = self.get_dns_challenges(host, order.authorizations) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) @@ -325,7 +335,7 @@ class AcmeHandler(object): ) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") - host_to_validate = self.maybe_remove_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.host) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) @@ -357,7 +367,7 @@ class AcmeHandler(object): dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_challenges = authz_record.dns_challenge - host_to_validate = self.maybe_remove_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.host) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) From 593c35776c868cfe937c79a119bc52a03dec446d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:17:05 -0700 Subject: [PATCH 091/150] adding new methods for getting pending clean --- lemur/certificates/service.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 0e91b563..ff558284 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -117,6 +117,41 @@ def get_all_pending_cleaning(source): ) +def get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire): + """ + Retrieves all certificates that are available for cleaning: not attached to endpoint, + and within X days from expiration. + + :param days_to_expire: + :param source: + :return: + """ + expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD") + return ( + Certificate.query.filter(Certificate.sources.any(id=source.id)) + .filter(not_(Certificate.endpoints.any())) + .filter(Certificate.not_after < expiration_window) + .all() + ) + + +def get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance): + """ + Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance. + + :param days_since_issuance: + :param source: + :return: + """ + not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + return ( + Certificate.query.filter(Certificate.sources.any(id=source.id)) + .filter(not_(Certificate.endpoints.any())) + .filter(Certificate.date_created < not_in_use_window) + .all() + ) + + def get_all_pending_reissue(): """ Retrieves all certificates that need to be rotated. From c96695c966229039e78e4a5f799bd525a0d122af Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:18:07 -0700 Subject: [PATCH 092/150] refactor --- lemur/sources/cli.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index c41a1cf7..c28600c2 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -54,6 +54,17 @@ def validate_sources(source_strings): return sources +def execute_clean(plugin, certificate, source): + try: + plugin.clean(certificate, source.options) + certificate.sources.remove(source) + certificate_service.database.update(certificate) + return SUCCESS_METRIC_STATUS + except Exception as e: + current_app.logger.exception(e) + sentry.captureException() + + @manager.option( "-s", "--sources", @@ -147,14 +158,7 @@ def clean(source_strings, commit): for certificate in certificate_service.get_all_pending_cleaning(source): status = FAILURE_METRIC_STATUS if commit: - try: - s.clean(certificate, source.options) - certificate.sources.remove(source) - certificate_service.database.update(certificate) - status = SUCCESS_METRIC_STATUS - except Exception as e: - current_app.logger.exception(e) - sentry.captureException() + status = execute_clean(s, certificate, source) metrics.send( "clean", From b28b4f9a28feba72c1d705d2085b30e57c75574d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:19:26 -0700 Subject: [PATCH 093/150] adding to new cli commands for cleaning certificates from source: a) either about to expire in X days and not attached to an endpoint a) or issued since X days but still not attached to an endpoint --- lemur/sources/cli.py | 155 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 19 deletions(-) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index c28600c2..a5b670a0 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -143,11 +143,9 @@ def clean(source_strings, commit): s = plugins.get(source.plugin_name) if not hasattr(s, "clean"): - print( - "Cannot clean source: {0}, source plugin does not implement 'clean()'".format( - source.label - ) - ) + info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + print(info_text) continue start_time = time.time() @@ -155,28 +153,147 @@ def clean(source_strings, commit): print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - for certificate in certificate_service.get_all_pending_cleaning(source): + certificates = certificate_service.get_all_pending_cleaning(source) + for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: status = execute_clean(s, certificate, source) metrics.send( - "clean", + "certificate_clean", "counter", 1, - metric_tags={"source": source.label, "status": status}, + metric_tags={"status": status, "source": source.label, "certificate": certificate.name}, ) - - current_app.logger.warning( - "Removed {0} from source {1} during cleaning".format( - certificate.name, source.label - ) - ) - + current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning") cleaned += 1 - print( - "[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format( - label=source.label, time=(time.time() - start_time), cleaned=cleaned + info_text = f"[+] Finished cleaning source: {source.label}. " \ + f"Removed {cleaned} certificates from source. " \ + f"Run Time: {(time.time() - start_time)}\n" + print(info_text) + current_app.logger.warning(info_text) + + +@manager.option( + "-s", + "--sources", + dest="source_strings", + action="append", + help="Sources to operate on.", +) +@manager.option( + "-d", + "--days", + dest="days_to_expire", + type=int, + action="store", + required=True, + help="The expiry range within days.", +) +@manager.option( + "-c", + "--commit", + dest="commit", + action="store_true", + default=False, + help="Persist changes.", +) +def clean_unused_and_expiring_within_days(source_strings, days_to_expire, commit): + sources = validate_sources(source_strings) + for source in sources: + s = plugins.get(source.plugin_name) + + if not hasattr(s, "clean"): + info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + print(info_text) + continue + + start_time = time.time() + + print("[+] Staring to clean source: {label}!\n".format(label=source.label)) + + cleaned = 0 + certificates = certificate_service.get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire) + for certificate in certificates: + status = FAILURE_METRIC_STATUS + if commit: + status = execute_clean(s, certificate, source) + + metrics.send( + "certificate_clean", + "counter", + 1, + metric_tags={"status": status, "source": source.label, "certificate": certificate.name}, ) - ) + current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning") + cleaned += 1 + + info_text = f"[+] Finished cleaning source: {source.label}. " \ + f"Removed {cleaned} certificates from source. " \ + f"Run Time: {(time.time() - start_time)}\n" + print(info_text) + current_app.logger.warning(info_text) + + +@manager.option( + "-s", + "--sources", + dest="source_strings", + action="append", + help="Sources to operate on.", +) +@manager.option( + "-d", + "--days", + dest="days_since_issuance", + type=int, + action="store", + required=True, + help="Days since issuance.", +) +@manager.option( + "-c", + "--commit", + dest="commit", + action="store_true", + default=False, + help="Persist changes.", +) +def clean_unused_and_issued_since_days(source_strings, days_since_issuance, commit): + sources = validate_sources(source_strings) + for source in sources: + s = plugins.get(source.plugin_name) + + if not hasattr(s, "clean"): + info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + print(info_text) + continue + + start_time = time.time() + + print("[+] Staring to clean source: {label}!\n".format(label=source.label)) + + cleaned = 0 + certificates = certificate_service.get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance) + for certificate in certificates: + status = FAILURE_METRIC_STATUS + if commit: + status = execute_clean(s, certificate, source) + + metrics.send( + "certificate_clean", + "counter", + 1, + metric_tags={"status": status, "source": source.label, "certificate": certificate.name}, + ) + current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning") + cleaned += 1 + + info_text = f"[+] Finished cleaning source: {source.label}. " \ + f"Removed {cleaned} certificates from source. " \ + f"Run Time: {(time.time() - start_time)}\n" + print(info_text) + current_app.logger.warning(info_text) From 34d23503def14520e951d2b976d9db6ad6e4b5e8 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:41:03 -0700 Subject: [PATCH 094/150] fixing the data bug --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index ff558284..0889f64c 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -147,7 +147,7 @@ def get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance): return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) .filter(not_(Certificate.endpoints.any())) - .filter(Certificate.date_created < not_in_use_window) + .filter(Certificate.date_created > not_in_use_window) .all() ) From 1a19e250bb6561e52e3ab20bbb8dd9b82b36009b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 16 Mar 2020 11:24:17 -0700 Subject: [PATCH 095/150] updating and cleaning up tests --- lemur/plugins/lemur_acme/plugin.py | 7 +- lemur/plugins/lemur_acme/tests/test_acme.py | 105 +++++++++++--------- 2 files changed, 63 insertions(+), 49 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e566352c..3fc1df61 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -55,7 +55,7 @@ class AcmeHandler(object): self.all_dns_providers = [] def get_dns_challenges(self, host, authorizations): - """Get final domain to validate and dns challenges for it""" + """Get dns challenges for provided domain""" domain_to_validate, is_wildcard = self.strip_wildcard(host) dns_challenges = [] @@ -70,7 +70,7 @@ class AcmeHandler(object): if isinstance(combo.chall, challenges.DNS01): dns_challenges.append(combo) - return domain_to_validate, dns_challenges + return dns_challenges def strip_wildcard(self, host): """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" @@ -98,7 +98,8 @@ class AcmeHandler(object): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) change_ids = [] - host_to_validate, dns_challenges = self.get_dns_challenges(host, order.authorizations) + dns_challenges = self.get_dns_challenges(host, order.authorizations) + host_to_validate, _ = self.strip_wildcard(host) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 990a556e..eefd6c5b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -23,11 +23,12 @@ class TestAcme(unittest.TestCase): } @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - def test_find_dns_challenge(self, mock_len): + def test_get_dns_challenges(self, mock_len): assert mock_len from acme import challenges + host = "example.com" c = challenges.DNS01() mock_authz = Mock() @@ -35,9 +36,18 @@ class TestAcme(unittest.TestCase): mock_entry = Mock() mock_entry.chall = c mock_authz.body.resolved_combinations.append(mock_entry) - result = yield self.acme.find_dns_challenge(mock_authz) + result = yield self.acme.get_dns_challenges(host, mock_authz) self.assertEqual(result, mock_entry) + def test_strip_wildcard(self): + expected = ("example.com", False) + result = self.acme.strip_wildcard("example.com") + self.assertEqual(expected, result) + + expected = ("example.com", True) + result = self.acme.strip_wildcard("*.example.com") + self.assertEqual(expected, result) + def test_authz_record(self): a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") self.assertEqual(type(a), plugin.AuthorizationRecord) @@ -45,9 +55,9 @@ class TestAcme(unittest.TestCase): @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_find_dns_challenge, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): assert mock_len mock_order = Mock() @@ -65,9 +75,12 @@ class TestAcme(unittest.TestCase): mock_dns_provider.create_txt_record = Mock(return_value=1) values = [mock_entry] - iterable = mock_find_dns_challenge.return_value + iterable = mock_get_dns_challenges.return_value iterator = iter(values) iterable.__iter__.return_value = iterator + + # mock_get_dns_challenges = Mock(return_value="") + result = self.acme.start_dns_challenge( mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} ) @@ -102,7 +115,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -127,15 +140,15 @@ class TestAcme(unittest.TestCase): @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_find_dns_challenge, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -256,11 +269,11 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.cloudflare.current_app") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def test_get_dns_provider( - self, - mock_dns_provider_service, - mock_current_app_cloudflare, - mock_current_app_dyn, - mock_current_app, + self, + mock_dns_provider_service, + mock_current_app_cloudflare, + mock_current_app_dyn, + mock_current_app, ): provider = plugin.ACMEIssuerPlugin() route53 = provider.get_dns_provider("route53") @@ -278,14 +291,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -309,14 +322,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -349,14 +362,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( - self, - mock_authorization_service, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_current_app, - mock_dns_provider_service, - mock_acme, + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_acme, ): provider = plugin.ACMEIssuerPlugin() mock_authority = Mock() @@ -423,10 +436,10 @@ class TestAcme(unittest.TestCase): ultradns._post = Mock() ultradns._get = Mock() ultradns._get.return_value = {'zoneName': 'test.example.com.com', - 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', - 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], - 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, - 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} + 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', + 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], + 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, + 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} ultradns._delete = Mock() mock_metrics.send = Mock() ultradns.delete_txt_record(change_id, account_number, domain, token) From 07dc31bed7c92650f7ab5e5e3247bf36bebb2c52 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 16 Mar 2020 11:41:05 -0700 Subject: [PATCH 096/150] cleaning up whitespace changes --- lemur/plugins/lemur_acme/tests/test_acme.py | 49 ++++++++++----------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index eefd6c5b..b2c32eec 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -78,9 +78,6 @@ class TestAcme(unittest.TestCase): iterable = mock_get_dns_challenges.return_value iterator = iter(values) iterable.__iter__.return_value = iterator - - # mock_get_dns_challenges = Mock(return_value="") - result = self.acme.start_dns_challenge( mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} ) @@ -115,7 +112,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -143,12 +140,12 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -291,14 +288,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -322,14 +319,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") From ecca003ab4a2d8710caa55b620a92e1a39f62650 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 17 Mar 2020 16:55:36 -0700 Subject: [PATCH 097/150] improving the documentation and method naming --- lemur/certificates/service.py | 27 ++++++++++++++------------- lemur/sources/cli.py | 6 +++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 0889f64c..1fa4d64e 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -102,12 +102,13 @@ def get_all_certs(): return Certificate.query.all() -def get_all_pending_cleaning(source): +def get_all_pending_cleaning_expired(source): """ - Retrieves all certificates that are available for cleaning. + Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not + attached to any endpoints. - :param source: - :return: + :param source: the source to search for certificates + :return: the pending certificates """ return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) @@ -117,14 +118,14 @@ def get_all_pending_cleaning(source): ) -def get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire): +def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): """ - Retrieves all certificates that are available for cleaning: not attached to endpoint, + Retrieves all certificates that are available for cleaning, not attached to endpoint, and within X days from expiration. - :param days_to_expire: - :param source: - :return: + :param days_to_expire: defines how many days till the certificate is expired + :param source: the source to search for certificates + :return: the pending certificates """ expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD") return ( @@ -135,13 +136,13 @@ def get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire): ) -def get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance): +def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): """ Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance. - :param days_since_issuance: - :param source: - :return: + :param days_since_issuance: defines how many days since the certificate is issued + :param source: the source to search for certificates + :return: the pending certificates """ not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") return ( diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index a5b670a0..0d537500 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -153,7 +153,7 @@ def clean(source_strings, commit): print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - certificates = certificate_service.get_all_pending_cleaning(source) + certificates = certificate_service.get_all_pending_cleaning_expired(source) for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: @@ -215,7 +215,7 @@ def clean_unused_and_expiring_within_days(source_strings, days_to_expire, commit print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - certificates = certificate_service.get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire) + certificates = certificate_service.get_all_pending_cleaning_expiring_in_days(source, days_to_expire) for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: @@ -277,7 +277,7 @@ def clean_unused_and_issued_since_days(source_strings, days_since_issuance, comm print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - certificates = certificate_service.get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance) + certificates = certificate_service.get_all_pending_cleaning_issued_since_days(source, days_since_issuance) for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: From 1d4da0e3d808b1b747daab2196364498c8f61f2a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 17 Mar 2020 16:59:09 -0700 Subject: [PATCH 098/150] another polish --- lemur/certificates/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1fa4d64e..a6bbba30 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -108,7 +108,7 @@ def get_all_pending_cleaning_expired(source): attached to any endpoints. :param source: the source to search for certificates - :return: the pending certificates + :return: list of pending certificates """ return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) @@ -125,7 +125,7 @@ def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): :param days_to_expire: defines how many days till the certificate is expired :param source: the source to search for certificates - :return: the pending certificates + :return: list of pending certificates """ expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD") return ( @@ -142,7 +142,7 @@ def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): :param days_since_issuance: defines how many days since the certificate is issued :param source: the source to search for certificates - :return: the pending certificates + :return: list of pending certificates """ not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") return ( From 697215f8bc47433c27dcd1b371aac5e19057088f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 21 Mar 2020 20:05:35 -0700 Subject: [PATCH 099/150] better handling of destination plugin errors, and also checking cert expiration before upload --- lemur/certificates/models.py | 3 +++ lemur/plugins/lemur_aws/plugin.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 0a76cd6b..2ca88b00 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -445,6 +445,9 @@ def update_destinations(target, value, initiator): """ destination_plugin = plugins.get(value.plugin_name) status = FAILURE_METRIC_STATUS + + if target.expired: + return try: if target.private_key or not destination_plugin.requires_key: destination_plugin.upload( diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 6669f641..7bb7a3a2 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -325,14 +325,17 @@ class AWSDestinationPlugin(DestinationPlugin): ] def upload(self, name, body, private_key, cert_chain, options, **kwargs): - iam.upload_cert( - name, - body, - private_key, - self.get_option("path", options), - cert_chain=cert_chain, - account_number=self.get_option("accountNumber", options), - ) + try: + iam.upload_cert( + name, + body, + private_key, + self.get_option("path", options), + cert_chain=cert_chain, + account_number=self.get_option("accountNumber", options), + ) + except ClientError: + sentry.captureException() def deploy(self, elb_name, account, region, certificate): pass From 8ba9ae1148bfdadc3121b8c4ffb69689648f5725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Mar 2020 16:18:43 +0000 Subject: [PATCH 100/150] Bump bleach from 3.1.1 to 3.1.2 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.1 to 3.1.2. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.1...v3.1.2) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 224789f6..d369cef4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,28 +5,28 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.1 # via readme-renderer +bleach==3.1.2 # via readme-renderer certifi==2019.11.28 # via requests cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests cryptography==2.8 # via secretstorage docutils==0.15.2 # via readme-renderer -flake8==3.5.0 +flake8==3.5.0 # via -r requirements-dev.in identify==1.4.9 # via pre-commit idna==2.8 # via requests -invoke==1.3.0 +invoke==1.3.0 # via -r requirements-dev.in jeepney==0.4.2 # via secretstorage keyring==21.0.0 # via twine mccabe==0.6.1 # via flake8 -nodeenv==1.3.3 +nodeenv==1.3.3 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==1.21.0 +pre-commit==1.21.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.5.2 # via readme-renderer -pyyaml==5.2 +pyyaml==5.2 # via -r requirements-dev.in, aspy.yaml, pre-commit readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine @@ -34,7 +34,7 @@ secretstorage==3.1.2 # via keyring six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.41.1 # via twine -twine==3.1.1 +twine==3.1.1 # via -r requirements-dev.in urllib3==1.25.7 # via requests virtualenv==16.7.9 # via pre-commit webencodings==0.5.1 # via bleach From 5206997468a3deb66be0393342d1d20561067605 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 26 Mar 2020 19:01:07 -0700 Subject: [PATCH 101/150] expired is now called for new certs, where the not_after field might be in datetime format, and not comparable to utc --- lemur/certificates/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 2ca88b00..9d7a459c 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -8,6 +8,8 @@ from datetime import timedelta import arrow +import pytz +import datetime from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa from flask import current_app @@ -321,8 +323,13 @@ class Certificate(db.Model): @hybrid_property def expired(self): - if self.not_after <= arrow.utcnow(): - return True + if isinstance(self.not_after, datetime.datetime): + # can't compare offset-naive and offset-aware datetimes + if self.not_after.replace(tzinfo=pytz.UTC) <= arrow.utcnow(): + return True + else: + if self.not_after <= arrow.utcnow(): + return True @expired.expression def expired(cls): From 2a2499a929bd30a80d19f4cca2be35dfdfa67098 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 26 Mar 2020 20:43:52 -0700 Subject: [PATCH 102/150] simplifying code --- lemur/certificates/models.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 9d7a459c..58630ee6 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -8,8 +8,6 @@ from datetime import timedelta import arrow -import pytz -import datetime from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa from flask import current_app @@ -323,13 +321,9 @@ class Certificate(db.Model): @hybrid_property def expired(self): - if isinstance(self.not_after, datetime.datetime): - # can't compare offset-naive and offset-aware datetimes - if self.not_after.replace(tzinfo=pytz.UTC) <= arrow.utcnow(): - return True - else: - if self.not_after <= arrow.utcnow(): - return True + # can't compare offset-naive and offset-aware datetimes + if arrow.Arrow.fromdatetime(self.not_after) <= arrow.utcnow(): + return True @expired.expression def expired(cls): From 0149f8b0d3b9c86739000c8372d92d7ac2d2ace3 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 26 Mar 2020 22:15:10 -0700 Subject: [PATCH 103/150] add support for wildcard and naked domains to PowerDNS module --- lemur/plugins/lemur_acme/powerdns.py | 210 +++++++++++++----- .../plugins/lemur_acme/tests/test_powerdns.py | 63 +++++- 2 files changed, 207 insertions(+), 66 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index a26faaac..e3f7e575 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -49,16 +49,20 @@ class Record: return self._data["name"] @property - def disabled(self): - return self._data["disabled"] + def type(self): + return self._data["type"] + + @property + def ttl(self): + return self._data["ttl"] @property def content(self): return self._data["content"] @property - def ttl(self): - return self._data["ttl"] + def disabled(self): + return self._data["disabled"] def get_zones(account_number): @@ -92,42 +96,32 @@ def get_zones(account_number): def create_txt_record(domain, token, account_number): """ Create a TXT record for the given domain and token and return a change_id tuple """ _check_conf() - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "REPLACE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } + function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": domain, "token": token, } + + # Create new record + domain_id = domain + "." + records = [Record({'name': domain_id, 'content': f"\"{token}\"", 'disabled': False})] + + # Get current records + cur_records = _get_txt_records(domain) + for record in cur_records: + if record.content != token: + records.append(record) + try: - _patch(path, payload) - log_data["message"] = "TXT record successfully created" + _patch_txt_records(domain, account_number, records) + log_data["message"] = "TXT record(s) successfully created" current_app.logger.debug(log_data) except Exception as e: sentry.captureException() log_data["Exception"] = e - log_data["message"] = "Unable to create TXT record" + log_data["message"] = "Unable to create TXT record(s)" current_app.logger.debug(log_data) change_id = (domain, token) @@ -173,43 +167,78 @@ def wait_for_dns_change(change_id, account_number=None): def delete_txt_record(change_id, account_number, domain, token): """ Delete the TXT record for the given domain and token """ _check_conf() - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "DELETE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } + function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": domain, - "token": token + "token": token, } - try: - _patch(path, payload) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as e: - sentry.captureException() - log_data["Exception"] = e - log_data["message"] = "Unable to delete TXT record" + + # Determine if we can delete whole RRset or just one record + cur_records = _get_txt_records(domain) + found = False + new_records = [] + for record in cur_records: + if record.content == f"\"{token}\"": + found = True + else: + new_records.append(record) + + if not found: # Record not found in DNS + log_data["message"] = "Unable to delete TXT record: TXT record not found" current_app.logger.debug(log_data) + return + + elif new_records: # Removing Record from RRSet via Patch + try: + _patch_txt_records(domain, account_number, new_records) + log_data["message"] = "TXT record successfully deleted" + current_app.logger.debug(log_data) + except Exception as e: + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record: patching exception" + current_app.logger.debug(log_data) + + else: # Delete current records + zone_name = _get_zone_name(domain, account_number) + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + zone_id = zone_name + "." + domain_id = domain + "." + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + payload = { + "rrsets": [ + { + "name": domain_id, + "type": "TXT", + "ttl": 300, + "changetype": "DELETE", + "records": [ + { + "content": f"\"{token}\"", + "disabled": False + } + ], + "comments": [] + } + ] + } + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token + } + try: + _patch(path, payload) + log_data["message"] = "TXT record successfully deleted" + current_app.logger.debug(log_data) + except Exception as e: + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record" + current_app.logger.debug(log_data) def _check_conf(): @@ -243,6 +272,33 @@ def _get_zone_name(domain, account_number): return zone_name +def _get_txt_records(domain): + """Retrieve TXT records for a given domain and return list of Record Objects""" + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + + path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" + function = sys._getframe().f_code.co_name + log_data = { + "function": function + } + try: + records = _get(path) + log_data["message"] = "Retrieved TXT Records Successfully" + current_app.logger.debug(log_data) + + except Exception as e: + sentry.captureException() + log_data["message"] = "Failed to Retrieve TXT Records" + current_app.logger.debug(log_data) + raise + + txt_records = [] + for record in records: + cur_record = Record(record) + txt_records.append(cur_record) + return txt_records + + def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") @@ -257,6 +313,40 @@ def _get(path, params=None): return resp.json() +def _patch_txt_records(domain, account_number, records): + """Send Patch request to PowerDNS Server""" + + domain_id = domain + "." + + # Create records + txt_records = [] + for record in records: + txt_records.append( + {'content': record.content, 'disabled': record.disabled} + ) + + # Create RRSet + payload = { + "rrsets": [ + { + "name": domain_id, + "type": "TXT", + "ttl": 300, + "changetype": "REPLACE", + "records": txt_records, + "comments": [] + } + ] + } + + # Create Txt Records + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + zone_name = _get_zone_name(domain, account_number) + zone_id = zone_name + "." + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + _patch(path, payload) + + def _patch(path, payload): """ Execute a Patch request on the given URL (base_uri + path) with given payload """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index c8b0a11e..d24827bb 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -48,13 +48,14 @@ class TestPowerdns(unittest.TestCase): self.assertEqual(result, zone) @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_create_txt_record(self, mock_current_app): + def test_create_txt_record_write_only(self, mock_current_app): domain = "_acme_challenge.test.example.com" zone = "test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) powerdns._check_conf = Mock() + powerdns._get_txt_records = Mock(return_value=[]) powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() mock_current_app.config.get = Mock(return_value="localhost") @@ -63,24 +64,74 @@ class TestPowerdns(unittest.TestCase): "function": "create_txt_record", "fqdn": domain, "token": token, - "message": "TXT record successfully created" + "message": "TXT record(s) successfully created" } result = powerdns.create_txt_record(domain, token, account_number) mock_current_app.logger.debug.assert_called_with(log_data) self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_create_txt_record_append(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + powerdns._check_conf = Mock() + cur_token = "123456" + cur_records = [powerdns.Record({'name': domain, 'content': cur_token, 'disabled': False})] + powerdns._get_txt_records = Mock(return_value=cur_records) + powerdns._get_zone_name = Mock(return_value=zone) + mock_current_app.logger.debug = Mock() + mock_current_app.config.get = Mock(return_value="localhost") + powerdns._patch = Mock() + log_data = { + "function": "create_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record(s) successfully created" + } + expected_path = f"/api/v1/servers/localhost/zones/test.example.com." + expected_payload = { + "rrsets": [ + { + "name": domain + ".", + "type": "TXT", + "ttl": 300, + "changetype": "REPLACE", + "records": [ + { + "content": f"\"{token}\"", + "disabled": False + }, + { + "content": f"\"{cur_token}\"", + "disabled": False + } + ], + "comments": [] + } + ] + } + + result = powerdns.create_txt_record(domain, token, account_number) + mock_current_app.logger.debug.assert_called_with(log_data) + powerdns._patch.assert_called_with(expected_path, expected_payload) + self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.dnsutil") @patch("lemur.plugins.lemur_acme.powerdns.current_app") @patch("lemur.extensions.metrics") @patch("time.sleep") def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil): domain = "_acme-challenge.test.example.com" - token = "ABCDEFG" + token1 = "ABCDEFG" + token2 = "HIJKLMN" zone_name = "test.example.com" nameserver = "1.1.1.1" - change_id = (domain, token) + change_id = (domain, token1) powerdns._check_conf = Mock() - mock_records = (token,) + mock_records = (token2, token1) mock_current_app.config.get = Mock(return_value=1) powerdns._get_zone_name = Mock(return_value=zone_name) mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) @@ -114,7 +165,7 @@ class TestPowerdns(unittest.TestCase): "function": "delete_txt_record", "fqdn": domain, "token": token, - "message": "TXT record successfully deleted" + "message": "Unable to delete TXT record: TXT record not found" } powerdns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_called_with(log_data) From 0e314d00281fbdcd31769cda2335d27df22d0d0b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 27 Mar 2020 10:18:38 -0700 Subject: [PATCH 104/150] adding documentation and final cleanup --- lemur/plugins/lemur_acme/powerdns.py | 97 ++++++++++++++++--- .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- 2 files changed, 82 insertions(+), 17 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index e3f7e575..c988fac9 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -1,11 +1,10 @@ -import time -import requests import json import sys +import time import lemur.common.utils as utils import lemur.dns_providers.util as dnsutil - +import requests from flask import current_app from lemur.extensions import metrics, sentry @@ -17,7 +16,9 @@ REQUIRED_VARIABLES = [ class Zone: - """ This class implements a PowerDNS zone in JSON. """ + """ + This class implements a PowerDNS zone in JSON. + """ def __init__(self, _data): self._data = _data @@ -39,7 +40,9 @@ class Zone: class Record: - """ This class implements a PowerDNS record. """ + """ + This class implements a PowerDNS record. + """ def __init__(self, _data): self._data = _data @@ -66,7 +69,12 @@ class Record: def get_zones(account_number): - """Retrieve authoritative zones from the PowerDNS API and return a list""" + """ + Retrieve authoritative zones from the PowerDNS API and return a list + :param account_number: + :raise: Exception + :return: list of Zone Objects + """ _check_conf() server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/zones" @@ -94,7 +102,14 @@ def get_zones(account_number): def create_txt_record(domain, token, account_number): - """ Create a TXT record for the given domain and token and return a change_id tuple """ + """ + Create a TXT record for the given domain and token and return a change_id tuple + + :param domain: FQDN + :param token: challenge value + :param account_number: + :return: tuple of domain/token + """ _check_conf() function = sys._getframe().f_code.co_name @@ -130,8 +145,11 @@ def create_txt_record(domain, token, account_number): def wait_for_dns_change(change_id, account_number=None): """ - Checks the authoritative DNS Server to see if changes have propagated to DNS - Retries and waits until successful. + Checks the authoritative DNS Server to see if changes have propagated. + + :param change_id: tuple of domain/token + :param account_number: + :return: """ _check_conf() domain, token = change_id @@ -165,7 +183,15 @@ def wait_for_dns_change(change_id, account_number=None): def delete_txt_record(change_id, account_number, domain, token): - """ Delete the TXT record for the given domain and token """ + """ + Delete the TXT record for the given domain and token + + :param change_id: tuple of domain/token + :param account_number: + :param domain: FQDN + :param token: challenge to delete + :return: + """ _check_conf() function = sys._getframe().f_code.co_name @@ -242,11 +268,20 @@ def delete_txt_record(change_id, account_number, domain, token): def _check_conf(): + """ + Verifies required configuration variables are set + + :return: + """ utils.validate_conf(current_app, REQUIRED_VARIABLES) def _generate_header(): - """Generate a PowerDNS API header and return it as a dictionary""" + """ + Generate a PowerDNS API header and return it as a dictionary + + :return: Dict of header parameters + """ api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME") api_key = current_app.config.get("ACME_POWERDNS_APIKEY") headers = {api_key_name: api_key} @@ -254,7 +289,13 @@ def _generate_header(): def _get_zone_name(domain, account_number): - """Get most specific matching zone for the given domain and return as a String""" + """ + Get most specific matching zone for the given domain and return as a String + + :param domain: FQDN + :param account_number: + :return: FQDN of domain + """ zones = get_zones(account_number) zone_name = "" for z in zones: @@ -273,7 +314,13 @@ def _get_zone_name(domain, account_number): def _get_txt_records(domain): - """Retrieve TXT records for a given domain and return list of Record Objects""" + """ + Retrieve TXT records for a given domain and return list of Record Objects + + :param domain: FQDN + :raise: Exception + :return: list of Record objects + """ server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" @@ -300,7 +347,13 @@ def _get_txt_records(domain): def _get(path, params=None): - """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ + """ + Execute a GET request on the given URL (base_uri + path) and return response as JSON object + + :param path: Relative URL path + :param params: additional parameters + :return: json response + """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.get( @@ -314,8 +367,14 @@ def _get(path, params=None): def _patch_txt_records(domain, account_number, records): - """Send Patch request to PowerDNS Server""" + """ + Send Patch request to PowerDNS Server + :param domain: FQDN + :param account_number: + :param records: List of Record objects + :return: + """ domain_id = domain + "." # Create records @@ -348,7 +407,13 @@ def _patch_txt_records(domain, account_number, records): def _patch(path, payload): - """ Execute a Patch request on the given URL (base_uri + path) with given payload """ + """ + Execute a Patch request on the given URL (base_uri + path) with given payload + + :param path: + :param payload: + :return: + """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.patch( diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index d24827bb..707ce9e4 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -79,7 +79,7 @@ class TestPowerdns(unittest.TestCase): change_id = (domain, token) powerdns._check_conf = Mock() cur_token = "123456" - cur_records = [powerdns.Record({'name': domain, 'content': cur_token, 'disabled': False})] + cur_records = [powerdns.Record({'name': domain, 'content': f"\"{cur_token}\"", 'disabled': False})] powerdns._get_txt_records = Mock(return_value=cur_records) powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() From d6cc8a8a9a86f6c1353d9ab15c9c0391856a139f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 30 Mar 2020 09:01:28 -0700 Subject: [PATCH 105/150] fixing whitespace --- lemur/plugins/lemur_acme/powerdns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index c988fac9..1542ad7b 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -71,6 +71,7 @@ class Record: def get_zones(account_number): """ Retrieve authoritative zones from the PowerDNS API and return a list + :param account_number: :raise: Exception :return: list of Zone Objects From 6f3ba23fa0bb3f046fd76ac04cc2629dcf48eac7 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 30 Mar 2020 13:34:24 -0700 Subject: [PATCH 106/150] updating sinlge line of comments --- lemur/plugins/lemur_acme/powerdns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 1542ad7b..c1e5e19b 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -70,8 +70,8 @@ class Record: def get_zones(account_number): """ - Retrieve authoritative zones from the PowerDNS API and return a list - + Retrieve authoritative zones from the PowerDNS API and return a list of zones + :param account_number: :raise: Exception :return: list of Zone Objects From 67d24caef586920ee0d91f790596dead5f119bb5 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:31:12 -0700 Subject: [PATCH 107/150] Remove equivalent destinations when cleaning certificates Remove equivalent destinations when cleaning certificates. This will prevent Lemur from attempting to re-upload a certificate after it has been cleaned. --- lemur/sources/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 0d537500..c415b567 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -58,6 +58,13 @@ def execute_clean(plugin, certificate, source): try: plugin.clean(certificate, source.options) certificate.sources.remove(source) + + # If we want to remove the source from the certificate, we also need to clear any equivalent destinations to + # prevent Lemur from re-uploading the certificate. + for destination in certificate.destinations: + if destination.label == source.label: + certificate.destinations.remove(destination) + certificate_service.database.update(certificate) return SUCCESS_METRIC_STATUS except Exception as e: From e25f97fce7da04b316ba2f6fc7359630f236f99f Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:50:24 -0700 Subject: [PATCH 108/150] Bump time limit for clean_source Celery job For larger accounts, I've hit SoftTimeLimit exceptions before completion of this celery job. Bumping up the time limit on this job. --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 4af33d86..ebf85ed7 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -332,7 +332,7 @@ def clean_all_sources(): metrics.send(f"{function}.success", 'counter', 1) -@celery.task(soft_time_limit=600) +@celery.task(soft_time_limit=3600) def clean_source(source): """ This celery task will clean the specified source. This is a destructive operation that will delete unused From d825616ea6cef6fd02ee80eedd7a75f5fc4cea61 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:53:17 -0700 Subject: [PATCH 109/150] No need to retry 25 times on DeleteConflict errors --- lemur/plugins/lemur_aws/iam.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 13590ddd..8d80e020 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -24,6 +24,12 @@ def retry_throttled(exception): if exception.response["Error"]["Code"] == "NoSuchEntity": return False + # No need to retry deletion requests if there is a DeleteConflict error. + # This error indicates that the certificate is still attached to an entity + # and cannot be deleted. + if exception.response["Error"]["Code"] == "DeleteConflict": + return False + metrics.send("iam_retry", "counter", 1, metric_tags={"exception": str(exception)}) return True From 2e1b58c70a233ad656fa88f69004b786d00dd4b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2020 18:27:38 +0000 Subject: [PATCH 110/150] Bump bleach from 3.1.2 to 3.1.4 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.2...v3.1.4) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d369cef4..b5521d38 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.2 # via readme-renderer +bleach==3.1.4 # via readme-renderer certifi==2019.11.28 # via requests cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit From 5add64714883d5c07b688a50f5fc5bbc9722839f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 3 Apr 2020 16:51:24 -0700 Subject: [PATCH 111/150] # emitting the count of certificates on the source --- lemur/sources/service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f4783313..408d411a 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -193,6 +193,11 @@ def sync_certificates(source, user): s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) + # emitting the count of certificates on the source + metrics.send("sync_certificates_count", + "gauge", len(certificates), + metric_tags={"source": source.label}) + for certificate in certificates: exists, updated_by_hash = find_cert(certificate) From 5c2a2f8ff24939b297788abff8322001e9b82513 Mon Sep 17 00:00:00 2001 From: David Stipp Date: Sat, 4 Apr 2020 11:24:04 -0400 Subject: [PATCH 112/150] OAUTH2 fixes * Use OAUTH2 variable instead of PING while using OAUTH * Some IDPs require a POST instead of a GET to user data --- lemur/auth/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lemur/auth/views.py b/lemur/auth/views.py index e7f87356..eaed419d 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -127,6 +127,10 @@ def retrieve_user(user_api_url, access_token): # retrieve information about the current user. r = requests.get(user_api_url, params=user_params, headers=headers) + # Some IDPs, like "Keycloak", require a POST instead of a GET + if r.status_code == 400: + r = requests.post(user_api_url, data=user_params, headers=headers) + profile = r.json() user = user_service.get_by_email(profile["email"]) @@ -434,7 +438,7 @@ class OAuth2(Resource): verify_cert=verify_cert, ) - jwks_url = current_app.config.get("PING_JWKS_URL") + jwks_url = current_app.config.get("OAUTH2_JWKS_URL") error_code = validate_id_token(id_token, args["clientId"], jwks_url) if error_code: return error_code From f82ec24dfaf6a1e8fb56fac3a394f2990085b711 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Sun, 5 Apr 2020 21:46:33 -0700 Subject: [PATCH 113/150] updating _get_txt_records return values and docstrings --- lemur/plugins/lemur_acme/powerdns.py | 24 +++++++++++++------ .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index c1e5e19b..a5d02353 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -202,7 +202,11 @@ def delete_txt_record(change_id, account_number, domain, token): "token": token, } - # Determine if we can delete whole RRset or just one record + """ + Get existing TXT records matching the domain from DNS + The token to be deleted should already exist + There may be other records with different tokens as well + """ cur_records = _get_txt_records(domain) found = False new_records = [] @@ -212,12 +216,16 @@ def delete_txt_record(change_id, account_number, domain, token): else: new_records.append(record) - if not found: # Record not found in DNS - log_data["message"] = "Unable to delete TXT record: TXT record not found" + # Since the matching token is not in DNS, there is nothing to delete + if not found: + log_data["message"] = "Unable to delete TXT record: Token not found in existing TXT records" current_app.logger.debug(log_data) return - elif new_records: # Removing Record from RRSet via Patch + # The record to delete has been found AND there are other tokens set on the same domain + # Since we only want to delete one token value from the RRSet, we need to use the Patch command to + # overwrite the current RRSet with the existing records. + elif new_records: try: _patch_txt_records(domain, account_number, new_records) log_data["message"] = "TXT record successfully deleted" @@ -228,7 +236,9 @@ def delete_txt_record(change_id, account_number, domain, token): log_data["message"] = "Unable to delete TXT record: patching exception" current_app.logger.debug(log_data) - else: # Delete current records + # The record to delete has been found AND there are no other token values set on the same domain + # Use the Delete command to delete the whole RRSet. + else: zone_name = _get_zone_name(domain, account_number) server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") zone_id = zone_name + "." @@ -319,7 +329,6 @@ def _get_txt_records(domain): Retrieve TXT records for a given domain and return list of Record Objects :param domain: FQDN - :raise: Exception :return: list of Record objects """ server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") @@ -336,9 +345,10 @@ def _get_txt_records(domain): except Exception as e: sentry.captureException() + log_data["Exception"] = e log_data["message"] = "Failed to Retrieve TXT Records" current_app.logger.debug(log_data) - raise + return [] txt_records = [] for record in records: diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 707ce9e4..167381f2 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -165,7 +165,7 @@ class TestPowerdns(unittest.TestCase): "function": "delete_txt_record", "fqdn": domain, "token": token, - "message": "Unable to delete TXT record: TXT record not found" + "message": "Unable to delete TXT record: Token not found in existing TXT records" } powerdns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_called_with(log_data) From eb138fc96011167138159590178f9584d6972014 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 08:38:40 -0700 Subject: [PATCH 114/150] Add default celery metrics and logging using celery signals --- lemur/common/celery.py | 223 +++++++++++++++++++++++++++++++++-------- 1 file changed, 180 insertions(+), 43 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index ebf85ed7..b0193515 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -10,27 +10,27 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B import copy import sys import time -from datetime import datetime, timezone, timedelta - from celery import Celery +from celery.app.task import Context from celery.exceptions import SoftTimeLimitExceeded +from celery.signals import task_failure, task_received, task_revoked, task_success +from datetime import datetime, timezone, timedelta from flask import current_app from lemur.authorities.service import get as get_authority +from lemur.certificates import cli as cli_certificate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service +from lemur.dns_providers import cli as cli_dns_providers +from lemur.endpoints import cli as cli_endpoints from lemur.extensions import metrics, sentry from lemur.factory import create_app +from lemur.notifications import cli as cli_notification from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources from lemur.sources.service import add_aws_destination_to_sources -from lemur.certificates import cli as cli_certificate -from lemur.dns_providers import cli as cli_dns_providers -from lemur.notifications import cli as cli_notification -from lemur.endpoints import cli as cli_endpoints - if current_app: flask_app = current_app @@ -67,7 +67,7 @@ def is_task_active(fun, task_id, args): from celery.task.control import inspect if not args: - args = '()' # empty args + args = "()" # empty args i = inspect() active_tasks = i.active() @@ -80,6 +80,37 @@ def is_task_active(fun, task_id, args): return False +def get_celery_request_tags(**kwargs): + request = kwargs.get("request") + sender_hostname = "unknown" + sender = kwargs.get("sender") + if sender: + try: + sender_hostname = sender.hostname + except AttributeError: + sender_hostname = vars(sender.request).get("origin", "unknown") + if request and not isinstance( + request, Context + ): # unlike others, task_revoked sends a Context for `request` + task_name = request.name + task_id = request.id + receiver_hostname = request.hostname + else: + task_name = sender.name + task_id = sender.request.id + receiver_hostname = sender.request.hostname + + tags = { + "task_name": task_name, + "task_id": task_id, + "sender_hostname": sender_hostname, + "receiver_hostname": receiver_hostname, + } + if kwargs.get("exception"): + tags["error"] = repr(kwargs["exception"]) + return tags + + @celery.task() def report_celery_last_success_metrics(): """ @@ -108,15 +139,115 @@ def report_celery_last_success_metrics(): return current_time = int(time.time()) - schedule = current_app.config.get('CELERYBEAT_SCHEDULE') + schedule = current_app.config.get("CELERYBEAT_SCHEDULE") for _, t in schedule.items(): task = t.get("task") last_success = int(red.get(f"{task}.last_success") or 0) - metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success) + metrics.send( + f"{task}.time_since_last_success", "gauge", current_time - last_success + ) red.set( f"{function}.last_success", int(time.time()) ) # Alert if this metric is not seen - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + + +@task_received.connect +def report_number_pending_tasks(**kwargs): + """ + Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used + for autoscaling workers. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + metrics.send( + "celery.new_pending_task", + "TIMER", + 1, + metric_tags=get_celery_request_tags(**kwargs), + ) + + +@task_success.connect +def report_successful_task(**kwargs): + """ + Report a generic success metric as tasks to our metrics broker every time a task finished correctly. + This metric can be used for autoscaling workers. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + tags = get_celery_request_tags(**kwargs) + red.set(f"{tags['task_name']}.last_success", int(time.time())) + metrics.send("celery.successful_task", "TIMER", 1, metric_tags=tags) + + +@task_failure.connect +def report_failed_task(**kwargs): + """ + Report a generic failure metric as tasks to our metrics broker every time a task fails. + This metric can be used for alerting. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "Message": "Celery Task Failure", + } + + # Add traceback if exception info is in the kwargs + einfo = kwargs.get("einfo") + if einfo: + log_data["traceback"] = einfo.traceback + + error_tags = get_celery_request_tags(**kwargs) + + log_data.update(error_tags) + current_app.logger.error(log_data) + metrics.send("celery.failed_task", "TIMER", 1, metric_tags=error_tags) + + +@task_revoked.connect +def report_revoked_task(**kwargs): + """ + Report a generic failure metric as tasks to our metrics broker every time a task is revoked. + This metric can be used for alerting. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "Message": "Celery Task Revoked", + } + + error_tags = get_celery_request_tags(**kwargs) + + log_data.update(error_tags) + current_app.logger.error(log_data) + metrics.send("celery.revoked_task", "TIMER", 1, metric_tags=error_tags) @celery.task(soft_time_limit=600) @@ -217,15 +348,15 @@ def fetch_acme_cert(id): log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) - metrics.send(f"{function}.resolved", 'gauge', new) - metrics.send(f"{function}.failed", 'gauge', failed) - metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer) + metrics.send(f"{function}.resolved", "gauge", new) + metrics.send(f"{function}.failed", "gauge", failed) + metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, failed=failed, wrong_issuer=wrong_issuer ) ) - red.set(f'{function}.last_success', int(time.time())) + return log_data @celery.task() @@ -262,8 +393,8 @@ def fetch_all_pending_acme_certs(): current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task() @@ -296,8 +427,8 @@ def remove_old_acme_certs(): current_app.logger.debug(log_data) pending_certificate_service.delete(cert) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task() @@ -328,8 +459,8 @@ def clean_all_sources(): current_app.logger.debug(log_data) clean_source.delay(source.label) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -366,6 +497,7 @@ def clean_source(source): current_app.logger.error(log_data) sentry.captureException() metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return log_data @celery.task() @@ -395,8 +527,8 @@ def sync_all_sources(): current_app.logger.debug(log_data) sync_source.delay(source.label) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=7200) @@ -428,19 +560,23 @@ def sync_source(source): current_app.logger.debug(log_data) try: sync([source]) - metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) + metrics.send( + f"{function}.success", "counter", 1, metric_tags={"source": source} + ) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source}) + metrics.send( + "sync_source_timeout", "counter", 1, metric_tags={"source": source} + ) metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) - metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) - red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", "counter", 1, metric_tags={"source": source}) + return log_data @celery.task() @@ -477,8 +613,8 @@ def sync_source_destination(): log_data["message"] = "completed Syncing AWS destinations and sources" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -515,8 +651,8 @@ def certificate_reissue(): log_data["message"] = "reissuance completed" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -534,7 +670,6 @@ def certificate_rotate(): "function": function, "message": "rotating certificates", "task_id": task_id, - } if task_id and is_task_active(function, task_id, None): @@ -554,8 +689,8 @@ def certificate_rotate(): log_data["message"] = "rotation completed" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -590,8 +725,8 @@ def endpoints_expire(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=600) @@ -626,8 +761,8 @@ def get_all_zones(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -662,8 +797,8 @@ def check_revoked(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -690,7 +825,9 @@ def notify_expirations(): current_app.logger.debug(log_data) try: - cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) + cli_notification.expirations( + current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []) + ) except SoftTimeLimitExceeded: log_data["message"] = "Notify expiring Time limit exceeded." current_app.logger.error(log_data) @@ -698,5 +835,5 @@ def notify_expirations(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data From 11b15e7e234e0d5b7b494e35697d2ced5fbb5972 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 08:41:48 -0700 Subject: [PATCH 115/150] Clean up docstrings --- lemur/common/celery.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b0193515..7c183dc9 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -120,7 +120,6 @@ def report_celery_last_success_metrics(): report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful. Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted from this function. - """ function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None @@ -158,12 +157,6 @@ def report_number_pending_tasks(**kwargs): Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used for autoscaling workers. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): metrics.send( @@ -180,12 +173,6 @@ def report_successful_task(**kwargs): Report a generic success metric as tasks to our metrics broker every time a task finished correctly. This metric can be used for autoscaling workers. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): tags = get_celery_request_tags(**kwargs) @@ -199,12 +186,6 @@ def report_failed_task(**kwargs): Report a generic failure metric as tasks to our metrics broker every time a task fails. This metric can be used for alerting. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): log_data = { @@ -230,12 +211,6 @@ def report_revoked_task(**kwargs): Report a generic failure metric as tasks to our metrics broker every time a task is revoked. This metric can be used for alerting. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): log_data = { From 1360d846fd16d0b375de8ce2b810a94aee312c49 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 11:50:42 -0700 Subject: [PATCH 116/150] Improve error logging for a couple of use cases --- lemur/common/defaults.py | 17 +++++++++++++---- lemur/plugins/lemur_aws/plugin.py | 22 ++++++++++++---------- lemur/sources/service.py | 16 ++++++++++------ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index d563dbd0..b9c88e49 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -2,6 +2,7 @@ import re import unicodedata from cryptography import x509 +from cryptography.hazmat.primitives.serialization import Encoding from flask import current_app from lemur.common.utils import is_selfsigned @@ -71,12 +72,20 @@ def common_name(cert): :return: Common name or None """ try: - return cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[ - 0 - ].value.strip() + subject_oid = cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME) + if len(subject_oid) > 0: + return subject_oid[0].value.strip() + return None except Exception as e: sentry.captureException() - current_app.logger.error("Unable to get common name! {0}".format(e)) + current_app.logger.error( + { + "message": "Unable to get common name", + "error": e, + "public_key": cert.public_bytes(Encoding.PEM).decode("utf-8") + }, + exc_info=True + ) def organization(cert): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 7bb7a3a2..8692348a 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -216,22 +216,24 @@ class AWSSourcePlugin(SourcePlugin): for region in regions: elbs = elb.get_all_elbs(account_number=account_number, region=region) - current_app.logger.info( - "Describing classic load balancers in {0}-{1}".format( - account_number, region - ) - ) + current_app.logger.info({ + "message": "Describing classic load balancers", + "account_number": account_number, + "region": region, + "number_of_load_balancers": len(elbs) + }) for e in elbs: endpoints.extend(get_elb_endpoints(account_number, region, e)) # fetch advanced ELBs elbs_v2 = elb.get_all_elbs_v2(account_number=account_number, region=region) - current_app.logger.info( - "Describing advanced load balancers in {0}-{1}".format( - account_number, region - ) - ) + current_app.logger.info({ + "message": "Describing advanced load balancers", + "account_number": account_number, + "region": region, + "number_of_load_balancers": len(elbs_v2) + }) for e in elbs_v2: endpoints.extend(get_elb_endpoints_v2(account_number, region, e)) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f4783313..e0f0aacf 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -123,15 +123,19 @@ def sync_endpoints(source): "acct": s.get_option("accountNumber", source.options)}) if not endpoint["certificate"]: - current_app.logger.error( - "Certificate Not Found. Name: {0} Endpoint: {1}".format( - certificate_name, endpoint["name"] - ) - ) + current_app.logger.error({ + "message": "Certificate Not Found", + "certificate_name": certificate_name, + "endpoint_name": endpoint["name"], + "dns_name": endpoint.get("dnsname"), + "account": s.get_option("accountNumber", source.options), + }) + metrics.send("endpoint.certificate.not.found", "counter", 1, metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], - "acct": s.get_option("accountNumber", source.options)}) + "acct": s.get_option("accountNumber", source.options), + "dnsname": endpoint.get("dnsname")}) continue policy = endpoint.pop("policy") From cee81bd693dcc2df831f9652b727542f49de1d22 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 9 Apr 2020 18:17:05 -0700 Subject: [PATCH 117/150] updated requirements, fixed unittests, pytest, and distinguidedName ordering --- lemur/plugins/lemur_acme/tests/test_acme.py | 59 +++--- .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- .../lemur_digicert/tests/test_digicert.py | 2 +- lemur/tests/test_certificates.py | 13 +- requirements-dev.txt | 46 ++-- requirements-docs.txt | 197 +++++++++--------- requirements-tests.txt | 110 +++++----- requirements.txt | 137 ++++++------ 8 files changed, 284 insertions(+), 282 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index b2c32eec..bec7be2b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,11 +1,10 @@ import unittest +from unittest.mock import patch, Mock from cryptography.x509 import DNSName -from requests.models import Response - -from mock import MagicMock, Mock, patch - from lemur.plugins.lemur_acme import plugin, ultradns +from mock import MagicMock +from requests.models import Response class TestAcme(unittest.TestCase): @@ -57,7 +56,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_get_dns_challenges, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): assert mock_len mock_order = Mock() @@ -88,7 +87,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") @patch("time.sleep") def test_complete_dns_challenge_success( - self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -112,7 +111,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -140,12 +139,12 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -182,7 +181,7 @@ class TestAcme(unittest.TestCase): assert result_client assert result_registration - @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch('lemur.plugins.lemur_acme.plugin.current_app') def test_get_domains_single(self, mock_current_app): options = {"common_name": "test.netflix.net"} result = self.acme.get_domains(options) @@ -288,14 +287,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -319,14 +318,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 167381f2..714cc938 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,5 +1,5 @@ import unittest -from mock import Mock, patch +from unittest.mock import patch, Mock from lemur.plugins.lemur_acme import plugin, powerdns diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 1e9ebca4..8bfd1dcf 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -1,4 +1,5 @@ import json +from unittest.mock import patch, Mock import arrow import pytest @@ -6,7 +7,6 @@ from cryptography import x509 from freezegun import freeze_time from lemur.plugins.lemur_digicert import plugin from lemur.tests.vectors import CSR_STR -from mock import Mock, patch def config_mock(*args): diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index adafa605..41584cb3 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -9,7 +9,8 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from marshmallow import ValidationError from freezegun import freeze_time -from mock import patch +# from mock import patch +from unittest.mock import patch from lemur.certificates.service import create_csr from lemur.certificates.views import * # noqa @@ -906,12 +907,12 @@ def test_certificate_get_body(client): assert response_body["serial"] == "211983098819107449768450703123665283596" assert response_body["serialHex"] == "9F7A75B39DAE4C3F9524C68B06DA6A0C" assert response_body["distinguishedName"] == ( - "CN=LemurTrust Unittests Class 1 CA 2018," - "O=LemurTrust Enterprises Ltd," - "OU=Unittesting Operations Center," - "C=EE," + "L=Earth," "ST=N/A," - "L=Earth" + "C=EE," + "OU=Unittesting Operations Center," + "O=LemurTrust Enterprises Ltd," + "CN=LemurTrust Unittests Class 1 CA 2018" ) diff --git a/requirements-dev.txt b/requirements-dev.txt index b5521d38..cb2edc22 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,40 +4,44 @@ # # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # -aspy.yaml==1.3.0 # via pre-commit +appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2019.11.28 # via requests +certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography -cfgv==2.0.1 # via pre-commit +cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests -cryptography==2.8 # via secretstorage -docutils==0.15.2 # via readme-renderer +cryptography==2.9 # via secretstorage +distlib==0.3.0 # via virtualenv +docutils==0.16 # via readme-renderer +filelock==3.0.12 # via virtualenv flake8==3.5.0 # via -r requirements-dev.in -identify==1.4.9 # via pre-commit -idna==2.8 # via requests -invoke==1.3.0 # via -r requirements-dev.in -jeepney==0.4.2 # via secretstorage -keyring==21.0.0 # via twine +identify==1.4.14 # via pre-commit +idna==2.9 # via requests +importlib-metadata==1.6.0 # via keyring, pre-commit, twine, virtualenv +invoke==1.4.1 # via -r requirements-dev.in +jeepney==0.4.3 # via keyring, secretstorage +keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 -nodeenv==1.3.3 # via -r requirements-dev.in, pre-commit +nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==1.21.0 # via -r requirements-dev.in +pre-commit==2.2.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi +pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 -pygments==2.5.2 # via readme-renderer -pyyaml==5.2 # via -r requirements-dev.in, aspy.yaml, pre-commit -readme-renderer==24.0 # via twine +pygments==2.6.1 # via readme-renderer +pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit +readme-renderer==25.0 # via twine requests-toolbelt==0.9.1 # via twine -requests==2.22.0 # via requests-toolbelt, twine +requests==2.23.0 # via requests-toolbelt, twine secretstorage==3.1.2 # via keyring -six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer +six==1.14.0 # via bleach, cryptography, readme-renderer, virtualenv toml==0.10.0 # via pre-commit -tqdm==4.41.1 # via twine +tqdm==4.45.0 # via twine twine==3.1.1 # via -r requirements-dev.in -urllib3==1.25.7 # via requests -virtualenv==16.7.9 # via pre-commit +urllib3==1.25.8 # via requests +virtualenv==20.0.17 # via pre-commit webencodings==0.5.1 # via bleach +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 893965ca..d3eaa4d1 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,111 +4,108 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.0.0 +acme==1.3.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx -alembic-autogenerate-enums==0.0.2 -alembic==1.3.2 -amqp==2.5.2 -aniso8601==8.0.0 -arrow==0.15.5 -asyncpool==1.0 +alembic-autogenerate-enums==0.0.2 # via -r requirements.txt +alembic==1.4.2 # via -r requirements.txt, flask-migrate +amqp==2.5.2 # via -r requirements.txt, kombu +aniso8601==8.0.0 # via -r requirements.txt, flask-restful +arrow==0.15.5 # via -r requirements.txt +asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx -bcrypt==3.1.7 -billiard==3.6.1.0 -blinker==1.4 -boto3==1.10.46 -botocore==1.13.46 -celery[redis]==4.4.0 -certifi==2019.11.28 -certsrv==2.1.1 -cffi==1.13.2 -chardet==3.0.4 -click==7.0 -cloudflare==2.3.1 -cryptography==2.8 -dnspython3==1.15.0 -dnspython==1.15.0 -docutils==0.15.2 -dyn==1.8.1 -flask-bcrypt==0.7.1 -flask-cors==3.0.8 -flask-mail==0.9.1 -flask-migrate==2.5.2 -flask-principal==0.4.0 -flask-replicated==1.3 -flask-restful==0.3.7 -flask-script==2.0.6 -flask-sqlalchemy==2.4.1 -flask==1.1.1 -future==0.18.2 -gunicorn==20.0.4 -hvac==0.9.6 -idna==2.8 +bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko +billiard==3.6.3.0 # via -r requirements.txt, celery +blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven +boto3==1.12.39 # via -r requirements.txt +botocore==1.15.39 # via -r requirements.txt, boto3, s3transfer +celery[redis]==4.4.2 # via -r requirements.txt +certifi==2020.4.5.1 # via -r requirements.txt, requests +certsrv==2.1.1 # via -r requirements.txt +cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl +chardet==3.0.4 # via -r requirements.txt, requests +click==7.1.1 # via -r requirements.txt, flask +cloudflare==2.6.5 # via -r requirements.txt +cryptography==2.9 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +dnspython3==1.15.0 # via -r requirements.txt +dnspython==1.15.0 # via -r requirements.txt, dnspython3 +docutils==0.15.2 # via -r requirements.txt, botocore, sphinx +dyn==1.8.1 # via -r requirements.txt +flask-bcrypt==0.7.1 # via -r requirements.txt +flask-cors==3.0.8 # via -r requirements.txt +flask-mail==0.9.1 # via -r requirements.txt +flask-migrate==2.5.3 # via -r requirements.txt +flask-principal==0.4.0 # via -r requirements.txt +flask-replicated==1.3 # via -r requirements.txt +flask-restful==0.3.8 # via -r requirements.txt +flask-script==2.0.6 # via -r requirements.txt +flask-sqlalchemy==2.4.1 # via -r requirements.txt, flask-migrate +flask==1.1.2 # via -r requirements.txt, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven +future==0.18.2 # via -r requirements.txt, cloudflare +gunicorn==20.0.4 # via -r requirements.txt +hvac==0.10.1 # via -r requirements.txt +idna==2.9 # via -r requirements.txt, requests imagesize==1.2.0 # via sphinx -importlib-metadata==1.3.0 -inflection==0.3.1 -itsdangerous==1.1.0 -javaobj-py3==0.4.0.1 -jinja2==2.10.3 -jmespath==0.9.4 -josepy==1.2.0 -jsonlines==1.2.0 -kombu==4.6.7 -lockfile==0.12.2 -logmatic-python==0.1.7 -mako==1.1.0 -markupsafe==1.1.1 -marshmallow-sqlalchemy==0.21.0 -marshmallow==2.20.4 -mock==3.0.5 -more-itertools==8.0.2 -ndg-httpsclient==0.5.1 -packaging==19.2 # via sphinx -paramiko==2.7.1 -pem==19.3.0 -psycopg2==2.8.4 -pyasn1-modules==0.2.7 -pyasn1==0.4.8 -pycparser==2.19 -pycryptodomex==3.9.4 -pygments==2.5.2 # via sphinx -pyjks==19.0.0 -pyjwt==1.7.1 -pynacl==1.3.0 -pyopenssl==19.1.0 -pyparsing==2.4.6 # via packaging -pyrfc3339==1.1 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-json-logger==0.1.11 -pytz==2019.3 -pyyaml==5.2 -raven[flask]==6.10.0 -redis==3.3.11 -requests-toolbelt==0.9.1 -requests[security]==2.22.0 -retrying==1.3.3 -s3transfer==0.2.1 -six==1.13.0 +importlib-metadata==1.6.0 # via -r requirements.txt, kombu +inflection==0.4.0 # via -r requirements.txt +itsdangerous==1.1.0 # via -r requirements.txt, flask +javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks +jinja2==2.11.1 # via -r requirements.txt, flask, sphinx +jmespath==0.9.5 # via -r requirements.txt, boto3, botocore +josepy==1.3.0 # via -r requirements.txt, acme +jsonlines==1.2.0 # via -r requirements.txt, cloudflare +kombu==4.6.8 # via -r requirements.txt, celery +lockfile==0.12.2 # via -r requirements.txt +logmatic-python==0.1.7 # via -r requirements.txt +mako==1.1.2 # via -r requirements.txt, alembic +markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako +marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt +marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy +mock==4.0.2 # via -r requirements.txt, acme +ndg-httpsclient==0.5.1 # via -r requirements.txt +packaging==20.3 # via sphinx +paramiko==2.7.1 # via -r requirements.txt +pem==20.1.0 # via -r requirements.txt +psycopg2==2.8.5 # via -r requirements.txt +pycparser==2.20 # via -r requirements.txt, cffi +pycryptodomex==3.9.7 # via -r requirements.txt, pyjks +pygments==2.6.1 # via sphinx +pyjks==19.0.0 # via -r requirements.txt +pyjwt==1.7.1 # via -r requirements.txt +pynacl==1.3.0 # via -r requirements.txt, paramiko +pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests +pyparsing==2.4.7 # via packaging +pyrfc3339==1.1 # via -r requirements.txt, acme +python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore +python-editor==1.0.4 # via -r requirements.txt, alembic +python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 +pyyaml==5.3.1 # via -r requirements.txt, cloudflare +raven[flask]==6.10.0 # via -r requirements.txt +redis==3.4.1 # via -r requirements.txt, celery +requests-toolbelt==0.9.1 # via -r requirements.txt, acme +requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx +retrying==1.3.3 # via -r requirements.txt +s3transfer==0.3.3 # via -r requirements.txt, boto3 +six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx -sphinx-rtd-theme==0.4.3 -sphinx==2.3.1 -sphinxcontrib-applehelp==1.0.1 # via sphinx -sphinxcontrib-devhelp==1.0.1 # via sphinx -sphinxcontrib-htmlhelp==1.0.2 # via sphinx -sphinxcontrib-httpdomain==1.7.0 +sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in +sphinx==3.0.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx +sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.2 # via sphinx -sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.12 -tabulate==0.8.6 -twofish==0.3.0 -urllib3==1.25.7 -vine==1.3.0 -werkzeug==0.16.0 -xmltodict==0.12.0 -zipp==0.6.0 +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy-utils==0.36.3 # via -r requirements.txt +sqlalchemy==1.3.16 # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.7 # via -r requirements.txt +twofish==0.3.0 # via -r requirements.txt, pyjks +urllib3==1.25.8 # via -r requirements.txt, botocore, requests +vine==1.3.0 # via -r requirements.txt, amqp, celery +werkzeug==1.0.1 # via -r requirements.txt, flask +xmltodict==0.12.0 # via -r requirements.txt +zipp==3.1.0 # via -r requirements.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 293bd350..874fbb5c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,85 +6,87 @@ # appdirs==1.4.3 # via black attrs==19.3.0 # via black, jsonschema, pytest -aws-sam-translator==1.19.1 # via cfn-lint +aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.4.3 # via moto -bandit==1.6.2 -black==19.10b0 -boto3==1.10.46 # via aws-sam-translator, moto +bandit==1.6.2 # via -r requirements-tests.in +black==19.10b0 # via -r requirements-tests.in +boto3==1.12.39 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.13.46 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.11.28 # via requests -cffi==1.13.2 # via cryptography -cfn-lint==0.26.2 # via moto +botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2020.4.5.1 # via requests +cffi==1.14.0 # via cryptography +cfn-lint==0.29.4 # via moto chardet==3.0.4 # via requests -click==7.0 # via black, flask -coverage==5.0.1 -cryptography==2.8 # via moto, sshpubkeys -docker==4.1.0 # via moto +click==7.1.1 # via black, flask +coverage==5.0.4 # via -r requirements-tests.in +cryptography==2.9 # via moto, sshpubkeys +decorator==4.4.2 # via networkx +docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys -factory-boy==2.12.0 -faker==3.0.0 -fakeredis==1.1.0 -flask==1.1.1 # via pytest-flask -freezegun==0.3.12 +factory-boy==2.12.0 # via -r requirements-tests.in +faker==4.0.2 # via -r requirements-tests.in, factory-boy +fakeredis==1.4.0 # via -r requirements-tests.in +flask==1.1.2 # via pytest-flask +freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk -gitdb2==2.0.6 # via gitpython -gitpython==3.0.5 # via bandit +gitdb==4.0.2 # via gitpython +gitpython==3.1.0 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.3.0 # via jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.10.3 # via flask, moto -jmespath==0.9.4 # via boto3, botocore +jinja2==2.11.1 # via flask, moto +jmespath==0.9.5 # via boto3, botocore jsondiff==1.1.2 # via moto -jsonpatch==1.24 # via cfn-lint -jsonpickle==1.2 # via aws-xray-sdk +jsonpatch==1.25 # via cfn-lint +jsonpickle==1.3 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 -mock==3.0.5 # via moto -more-itertools==8.0.2 # via pytest, zipp -moto==1.3.14 -nose==1.3.7 -packaging==19.2 # via pytest -pathspec==0.7.0 # via black -pbr==5.4.4 # via stevedore +mock==4.0.2 # via moto +more-itertools==8.2.0 # via pytest +moto==1.3.14 # via -r requirements-tests.in +networkx==2.4 # via cfn-lint +nose==1.3.7 # via -r requirements-tests.in +packaging==20.3 # via pytest +pathspec==0.8.0 # via black +pbr==5.4.5 # via stevedore pluggy==0.13.1 # via pytest py==1.8.1 # via pytest pyasn1==0.4.8 # via python-jose, rsa -pycparser==2.19 # via cffi -pyflakes==2.1.1 -pyparsing==2.4.6 # via packaging -pyrsistent==0.15.6 # via jsonschema -pytest-flask==0.15.0 -pytest-mock==1.13.0 -pytest==5.3.2 +pycparser==2.20 # via cffi +pyflakes==2.1.1 # via -r requirements-tests.in +pyparsing==2.4.7 # via packaging +pyrsistent==0.16.0 # via jsonschema +pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-mock==3.0.0 # via -r requirements-tests.in +pytest==5.4.1 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto -pyyaml==5.2 -redis==3.3.11 # via fakeredis -regex==2019.12.20 # via black -requests-mock==1.7.0 -requests==2.22.0 # via docker, moto, requests-mock, responses -responses==0.10.9 # via moto +pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto +redis==3.4.1 # via fakeredis +regex==2020.4.4 # via black +requests-mock==1.7.0 # via -r requirements-tests.in +requests==2.23.0 # via docker, moto, requests-mock, responses +responses==0.10.12 # via moto rsa==4.0 # via python-jose -s3transfer==0.2.1 # via boto3 -six==1.13.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, faker, fakeredis, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client -smmap2==2.0.5 # via gitdb2 +s3transfer==0.3.3 # via boto3 +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +smmap==3.0.1 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto -stevedore==1.31.0 # via bandit +stevedore==1.32.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -typed-ast==1.4.0 # via black -urllib3==1.25.7 # via botocore, requests -wcwidth==0.1.8 # via pytest +typed-ast==1.4.1 # via black +urllib3==1.25.8 # via botocore, requests +wcwidth==0.1.9 # via pytest websocket-client==0.57.0 # via docker -werkzeug==0.16.0 # via flask, moto, pytest-flask -wrapt==1.11.2 # via aws-xray-sdk +werkzeug==1.0.1 # via flask, moto, pytest-flask +wrapt==1.12.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==0.6.0 # via importlib-metadata +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.txt b/requirements.txt index 639c9377..964576d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,96 +4,95 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.0.0 -alembic-autogenerate-enums==0.0.2 -alembic==1.3.2 # via flask-migrate +acme==1.3.0 # via -r requirements.in +alembic-autogenerate-enums==0.0.2 # via -r requirements.in +alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.5 -asyncpool==1.0 +arrow==0.15.5 # via -r requirements.in +asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko -billiard==3.6.1.0 # via celery +billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.10.46 -botocore==1.13.46 -celery[redis]==4.4.0 -certifi==2019.11.28 -certsrv==2.1.1 -cffi==1.13.2 # via bcrypt, cryptography, pynacl +boto3==1.12.39 # via -r requirements.in +botocore==1.15.39 # via -r requirements.in, boto3, s3transfer +celery[redis]==4.4.2 # via -r requirements.in +certifi==2020.4.5.1 # via -r requirements.in, requests +certsrv==2.1.1 # via -r requirements.in +cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.0 # via flask -cloudflare==2.3.1 -cryptography==2.8 -dnspython3==1.15.0 +click==7.1.1 # via flask +cloudflare==2.6.5 # via -r requirements.in +cryptography==2.9 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore -dyn==1.8.1 -flask-bcrypt==0.7.1 -flask-cors==3.0.8 -flask-mail==0.9.1 -flask-migrate==2.5.2 -flask-principal==0.4.0 -flask-replicated==1.3 -flask-restful==0.3.7 -flask-script==2.0.6 -flask-sqlalchemy==2.4.1 -flask==1.1.1 -future==0.18.2 -gunicorn==20.0.4 -hvac==0.9.6 -idna==2.8 # via requests -importlib-metadata==1.3.0 # via kombu -inflection==0.3.1 +dyn==1.8.1 # via -r requirements.in +flask-bcrypt==0.7.1 # via -r requirements.in +flask-cors==3.0.8 # via -r requirements.in +flask-mail==0.9.1 # via -r requirements.in +flask-migrate==2.5.3 # via -r requirements.in +flask-principal==0.4.0 # via -r requirements.in +flask-replicated==1.3 # via -r requirements.in +flask-restful==0.3.8 # via -r requirements.in +flask-script==2.0.6 # via -r requirements.in +flask-sqlalchemy==2.4.1 # via -r requirements.in, flask-migrate +flask==1.1.2 # via -r requirements.in, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven +future==0.18.2 # via -r requirements.in, cloudflare +gunicorn==20.0.4 # via -r requirements.in +hvac==0.10.1 # via -r requirements.in +idna==2.9 # via requests +importlib-metadata==1.6.0 # via kombu +inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.10.3 -jmespath==0.9.4 # via boto3, botocore -josepy==1.2.0 # via acme +jinja2==2.11.1 # via -r requirements.in, flask +jmespath==0.9.5 # via boto3, botocore +josepy==1.3.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.6.7 # via celery -lockfile==0.12.2 -logmatic-python==0.1.7 -mako==1.1.0 # via alembic +kombu==4.6.8 # via celery +lockfile==0.12.2 # via -r requirements.in +logmatic-python==0.1.7 # via -r requirements.in +mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.21.0 -marshmallow==2.20.4 -mock==3.0.5 # via acme -more-itertools==8.0.2 # via zipp -ndg-httpsclient==0.5.1 -paramiko==2.7.1 -pem==19.3.0 -psycopg2==2.8.4 -pyasn1-modules==0.2.7 # via pyjks, python-ldap +marshmallow-sqlalchemy==0.22.3 # via -r requirements.in +marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy +mock==4.0.2 # via acme +ndg-httpsclient==0.5.1 # via -r requirements.in +paramiko==2.7.1 # via -r requirements.in +pem==20.1.0 # via -r requirements.in +psycopg2==2.8.5 # via -r requirements.in +pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap -pycparser==2.19 # via cffi -pycryptodomex==3.9.4 # via pyjks -pyjks==19.0.0 -pyjwt==1.7.1 +pycparser==2.20 # via cffi +pycryptodomex==3.9.7 # via pyjks +pyjks==19.0.0 # via -r requirements.in +pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko -pyopenssl==19.1.0 +pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests pyrfc3339==1.1 # via acme python-dateutil==2.8.1 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python -python-ldap==3.2.0 +python-ldap==3.2.0 # via -r requirements.in pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.2 -raven[flask]==6.10.0 -redis==3.3.11 +pyyaml==5.3.1 # via -r requirements.in, cloudflare +raven[flask]==6.10.0 # via -r requirements.in +redis==3.4.1 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme -requests[security]==2.22.0 -retrying==1.3.3 -s3transfer==0.2.1 # via boto3 -six==1.13.0 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.6 +requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt +retrying==1.3.3 # via -r requirements.in +s3transfer==0.3.3 # via boto3 +six==1.14.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils +sqlalchemy-utils==0.36.3 # via -r requirements.in +sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.7 # via -r requirements.in twofish==0.3.0 # via pyjks -urllib3==1.25.7 # via botocore, requests +urllib3==1.25.8 # via botocore, requests vine==1.3.0 # via amqp, celery -werkzeug==0.16.0 # via flask -xmltodict==0.12.0 -zipp==0.6.0 # via importlib-metadata +werkzeug==1.0.1 # via flask +xmltodict==0.12.0 # via -r requirements.in +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From f8657998e64cf3c4cb32406dbaa15296803962bb Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 15 Apr 2020 14:17:34 -0700 Subject: [PATCH 118/150] updated python requirements and still works --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 22 +++++++++++----------- requirements.txt | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d3eaa4d1..54bde14e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -49,7 +49,7 @@ importlib-metadata==1.6.0 # via -r requirements.txt, kombu inflection==0.4.0 # via -r requirements.txt itsdangerous==1.1.0 # via -r requirements.txt, flask javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks -jinja2==2.11.1 # via -r requirements.txt, flask, sphinx +jinja2==2.11.2 # via -r requirements.txt, flask, sphinx jmespath==0.9.5 # via -r requirements.txt, boto3, botocore josepy==1.3.0 # via -r requirements.txt, acme jsonlines==1.2.0 # via -r requirements.txt, cloudflare @@ -89,7 +89,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in -sphinx==3.0.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.0.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 874fbb5c..5f9cafcc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,7 +7,7 @@ appdirs==1.4.3 # via black attrs==19.3.0 # via black, jsonschema, pytest aws-sam-translator==1.22.0 # via cfn-lint -aws-xray-sdk==2.4.3 # via moto +aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in boto3==1.12.39 # via aws-sam-translator, moto @@ -15,31 +15,31 @@ boto==2.49.0 # via moto botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography -cfn-lint==0.29.4 # via moto +cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.1 # via black, flask -coverage==5.0.4 # via -r requirements-tests.in +coverage==5.1 # via -r requirements-tests.in cryptography==2.9 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in -faker==4.0.2 # via -r requirements-tests.in, factory-boy +faker==4.0.3 # via -r requirements-tests.in, factory-boy fakeredis==1.4.0 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk -gitdb==4.0.2 # via gitpython -gitpython==3.1.0 # via bandit +gitdb==4.0.4 # via gitpython +gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.11.1 # via flask, moto +jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore jsondiff==1.1.2 # via moto jsonpatch==1.25 # via cfn-lint -jsonpickle==1.3 # via aws-xray-sdk +jsonpickle==1.4 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 @@ -55,7 +55,7 @@ pluggy==0.13.1 # via pytest py==1.8.1 # via pytest pyasn1==0.4.8 # via python-jose, rsa pycparser==2.20 # via cffi -pyflakes==2.1.1 # via -r requirements-tests.in +pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in @@ -73,7 +73,7 @@ responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client -smmap==3.0.1 # via gitdb +smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto stevedore==1.32.0 # via bandit diff --git a/requirements.txt b/requirements.txt index 964576d2..bdad75b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ importlib-metadata==1.6.0 # via kombu inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.11.1 # via -r requirements.in, flask +jinja2==2.11.2 # via -r requirements.in, flask jmespath==0.9.5 # via boto3, botocore josepy==1.3.0 # via acme jsonlines==1.2.0 # via cloudflare From e33b767d12f9850d8fbfbf698e32ecdc3a50fbb7 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 15 Apr 2020 14:31:41 -0700 Subject: [PATCH 119/150] updating package.jso based on `npm update` --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b899176..84e5289d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ }, "devDependencies": { "gulp": "^3.9.1", - "jshint": "^2.8.0", + "jshint": "^2.11.0", "karma-chrome-launcher": "^2.0.0" } } From 4fcb050fa81b4080ff92e3665679d870fc112f57 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 16 Apr 2020 01:08:39 -0700 Subject: [PATCH 120/150] fixing bootstrap, bootswatch, and updating bower --- bower.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bower.json b/bower.json index f7d5500d..44cce994 100644 --- a/bower.json +++ b/bower.json @@ -11,12 +11,12 @@ "angular": "1.4.9", "json3": "~3.3", "es5-shim": "~4.5.0", - "bootstrap": "~3.3.6", "angular-bootstrap": "~1.1.1", "angular-animate": "~1.4.9", "restangular": "~1.5.1", "ng-table": "~0.8.3", "moment": "~2.11.1", + "bootstrap": "~3.4.1", "angular-loading-bar": "~0.8.0", "angular-moment": "~0.10.3", "moment-range": "~2.1.0", @@ -24,7 +24,7 @@ "angularjs-toaster": "~1.0.0", "angular-chart.js": "~0.8.8", "ngletteravatar": "~4.0.0", - "bootswatch": "~3.3.6", + "bootswatch": "~3.4.1", "fontawesome": "~4.5.0", "satellizer": "~0.13.4", "angular-ui-router": "~0.2.15", diff --git a/package.json b/package.json index 771a4386..1a54eccc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "url": "git://github.com/netflix/lemur.git" }, "dependencies": { - "bower": "^1.8.2", + "bower": "^1.8.8", "browser-sync": "^2.26.7", "del": "^2.2.2", "gulp-autoprefixer": "^3.1.1", From 8d0007b9c06f3e40305eab1083dcf7bb3fdda14f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 24 Apr 2020 15:48:06 -0700 Subject: [PATCH 121/150] fixing the private DNS zone issue. Private hosted zones will never be visible to third-parties like LetsEncrypt, and Lemur should not consider them as authoritative zones. This fix, make sure they are not added to the dns_provider table. --- lemur/plugins/lemur_acme/route53.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index 55da5161..aaccb57e 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -35,9 +35,10 @@ def get_zones(client=None): zones = [] for page in paginator.paginate(): for zone in page["HostedZones"]: - zones.append( - zone["Name"][:-1] - ) # We need [:-1] to strip out the trailing dot. + if not zone["Config"]["PrivateZone"]: + zones.append( + zone["Name"][:-1] + ) # We need [:-1] to strip out the trailing dot. return zones From 273c3e2793647389e00e2509b0e0fd047aa540fe Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 11:52:43 -0700 Subject: [PATCH 122/150] Celery task to enable autorotate for all certificates attached to endpoints without it enabled --- lemur/certificates/service.py | 25 ++++++++++++++++++++++--- lemur/common/celery.py | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index a6bbba30..b031d86b 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -118,6 +118,21 @@ def get_all_pending_cleaning_expired(source): ) +def get_all_certs_attached_to_endpoint_without_rotate(): + """ + Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled. + + :return: list of certificates attached to an endpoint without autorotate + """ + return ( + Certificate.query.filter(Certificate.endpoints.any()) + .filter(Certificate.rotation == False) + .filter(Certificate.not_after >= arrow.now()) + .filter(not_(Certificate.replaced.any())) + .all() # noqa + ) + + def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): """ Retrieves all certificates that are available for cleaning, not attached to endpoint, @@ -144,7 +159,9 @@ def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): :param source: the source to search for certificates :return: list of pending certificates """ - not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + not_in_use_window = ( + arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + ) return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) .filter(not_(Certificate.endpoints.any())) @@ -367,9 +384,11 @@ def render(args): show_expired = args.pop("showExpired") if show_expired != 1: - one_month_old = arrow.now()\ - .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\ + one_month_old = ( + arrow.now() + .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1)) .format("YYYY-MM-DD") + ) query = query.filter(Certificate.not_after > one_month_old) time_range = args.pop("time_range") diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 7c183dc9..a5f608b2 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -17,8 +17,10 @@ from celery.signals import task_failure, task_received, task_revoked, task_succe from datetime import datetime, timezone, timedelta from flask import current_app +from lemur import database from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate +from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_rotate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers @@ -812,3 +814,25 @@ def notify_expirations(): metrics.send(f"{function}.success", "counter", 1) return log_data + + +@celery.task(soft_time_limit=3600) +def enable_autorotate_for_certs_attached_to_endpoint(): + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "task_id": task_id, + } + + eligible_certs = get_all_certs_attached_to_endpoint_without_rotate() + for cert in eligible_certs: + log_data["certificate"] = cert.name + log_data["certificate_id"] = cert.id + log_data["message"] = "Enabling auto-rotate for certificate" + current_app.logger.info(log_data) + cert.rotation = True + database.update(cert) From 863af7a3e5f509b984ff2adba31515ebb7187624 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 12:16:46 -0700 Subject: [PATCH 123/150] Making CLI command ; Running black --- lemur/certificates/cli.py | 50 +++++++++++++++++++++++------------ lemur/certificates/service.py | 2 +- lemur/common/celery.py | 21 +++++++-------- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index b57ff175..ca6b0248 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -5,29 +5,18 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import sys import multiprocessing -from tabulate import tabulate -from sqlalchemy import or_ - +import sys from flask import current_app - -from flask_script import Manager from flask_principal import Identity, identity_changed - +from flask_script import Manager +from sqlalchemy import or_ +from tabulate import tabulate from lemur import database -from lemur.extensions import sentry -from lemur.extensions import metrics -from lemur.plugins.base import plugins -from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS -from lemur.deployment import service as deployment_service -from lemur.endpoints import service as endpoint_service -from lemur.notifications.messaging import send_rotation_notification -from lemur.domains.models import Domain from lemur.authorities.models import Authority -from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.models import Certificate +from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.service import ( reissue_certificate, get_certificate_primitives, @@ -35,9 +24,16 @@ from lemur.certificates.service import ( get_by_name, get_all_certs, get, + get_all_certs_attached_to_endpoint_without_autorotate, ) - from lemur.certificates.verify import verify_string +from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS +from lemur.deployment import service as deployment_service +from lemur.domains.models import Domain +from lemur.endpoints import service as endpoint_service +from lemur.extensions import sentry, metrics +from lemur.notifications.messaging import send_rotation_notification +from lemur.plugins.base import plugins manager = Manager(usage="Handles all certificate related tasks.") @@ -482,3 +478,23 @@ def check_revoked(): cert.status = "unknown" database.update(cert) + + +@manager.command +def automatically_enable_autorotate(): + """ + This function automatically enables autorotation for unexpired certificates that are + attached to an endpoint but do not have autorotate enabled. + """ + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + } + + eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate() + for cert in eligible_certs: + log_data["certificate"] = cert.name + log_data["certificate_id"] = cert.id + log_data["message"] = "Enabling auto-rotate for certificate" + current_app.logger.info(log_data) + cert.rotation = True + database.update(cert) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b031d86b..5d1e6e63 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -118,7 +118,7 @@ def get_all_pending_cleaning_expired(source): ) -def get_all_certs_attached_to_endpoint_without_rotate(): +def get_all_certs_attached_to_endpoint_without_autorotate(): """ Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled. diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a5f608b2..7701b82d 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -17,10 +17,8 @@ from celery.signals import task_failure, task_received, task_revoked, task_succe from datetime import datetime, timezone, timedelta from flask import current_app -from lemur import database from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate -from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_rotate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers @@ -818,21 +816,20 @@ def notify_expirations(): @celery.task(soft_time_limit=3600) def enable_autorotate_for_certs_attached_to_endpoint(): - function = f"{__name__}.{sys._getframe().f_code.co_name}" + """ + This celery task automatically enables autorotation for unexpired certificates that are + attached to an endpoint but do not have autorotate enabled. + :return: + """ task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { - "function": function, + "function": f"{__name__}.{sys._getframe().f_code.co_name}", "task_id": task_id, + "message": "Enabling autorotate to eligible certificates", } + current_app.logger.debug(log_data) - eligible_certs = get_all_certs_attached_to_endpoint_without_rotate() - for cert in eligible_certs: - log_data["certificate"] = cert.name - log_data["certificate_id"] = cert.id - log_data["message"] = "Enabling auto-rotate for certificate" - current_app.logger.info(log_data) - cert.rotation = True - database.update(cert) + cli_certificate.automatically_enable_autorotate() From 7e97d885dfea1b86d7274c472f106a318ddc4738 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 13:16:27 -0700 Subject: [PATCH 124/150] Address comments --- lemur/common/celery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 7701b82d..5df470ab 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -821,15 +821,18 @@ def enable_autorotate_for_certs_attached_to_endpoint(): attached to an endpoint but do not have autorotate enabled. :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { - "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "function": function, "task_id": task_id, "message": "Enabling autorotate to eligible certificates", } current_app.logger.debug(log_data) cli_certificate.automatically_enable_autorotate() + metrics.send(f"{function}.success", "counter", 1) + return log_data From 4c40e806bcb0be7006f20654c39803dd0b4c8ab0 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 29 Apr 2020 08:41:42 -0700 Subject: [PATCH 125/150] Requirements fix for Pip 20.1 --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index fa5a23bc..f73ebb6f 100644 --- a/setup.py +++ b/setup.py @@ -45,16 +45,16 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f: exec(f.read(), about) # nosec: about file is benign install_requires_g = parse_requirements("requirements.txt", session=PipSession()) -install_requires = [str(ir.req) for ir in install_requires_g] +install_requires = [str(ir.requirement) for ir in install_requires_g] tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession()) -tests_require = [str(ir.req) for ir in tests_require_g] +tests_require = [str(ir.requirement) for ir in tests_require_g] -docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession()) -docs_require = [str(ir.req) for ir in docs_require_g] +docs_require_g = parse_requirements("require20ments-docs.txt", session=PipSession()) +docs_require = [str(ir.requirement) for ir in docs_require_g] dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession()) -dev_requires = [str(ir.req) for ir in dev_requires_g] +dev_requires = [str(ir.requirement) for ir in dev_requires_g] class SmartInstall(install): From ba8184c87492413dfed09a3405e0ae560ed9b9aa Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 29 Apr 2020 08:51:09 -0700 Subject: [PATCH 126/150] Fix requirement parsing for pip 20.1 --- setup.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index f73ebb6f..4ce03d70 100644 --- a/setup.py +++ b/setup.py @@ -45,16 +45,20 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f: exec(f.read(), about) # nosec: about file is benign install_requires_g = parse_requirements("requirements.txt", session=PipSession()) -install_requires = [str(ir.requirement) for ir in install_requires_g] - tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession()) -tests_require = [str(ir.requirement) for ir in tests_require_g] - -docs_require_g = parse_requirements("require20ments-docs.txt", session=PipSession()) -docs_require = [str(ir.requirement) for ir in docs_require_g] - +docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession()) dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession()) -dev_requires = [str(ir.requirement) for ir in dev_requires_g] + +if tuple(map(int, pip.__version__.split('.'))) >= (20, 1): + install_requires = [str(ir.requirement) for ir in install_requires_g] + tests_require = [str(ir.requirement) for ir in tests_require_g] + docs_require = [str(ir.requirement) for ir in docs_require_g] + dev_requires = [str(ir.requirement) for ir in dev_requires_g] +else: + install_requires = [str(ir.req) for ir in install_requires_g] + tests_require = [str(ir.req) for ir in tests_require_g] + docs_require = [str(ir.req) for ir in docs_require_g] + dev_requires = [str(ir.req) for ir in dev_requires_g] class SmartInstall(install): From d469700cbf46c820f9f73366fb524c6232787c6f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 00:23:52 +0000 Subject: [PATCH 127/150] Bump fakeredis from 1.4.0 to 1.4.1 Bumps [fakeredis](https://github.com/jamesls/fakeredis) from 1.4.0 to 1.4.1. - [Release notes](https://github.com/jamesls/fakeredis/releases) - [Commits](https://github.com/jamesls/fakeredis/compare/1.4.0...1.4.1) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 5f9cafcc..7ffb2d5f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -26,14 +26,14 @@ docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in faker==4.0.3 # via -r requirements-tests.in, factory-boy -fakeredis==1.4.0 # via -r requirements-tests.in +fakeredis==1.4.1 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk gitdb==4.0.4 # via gitpython gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle itsdangerous==1.1.0 # via flask jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore @@ -72,7 +72,7 @@ requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 -six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto From 7f6eae7213db19d62c0963fea6eb68396fa22f54 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 00:25:47 +0000 Subject: [PATCH 128/150] Bump cryptography from 2.9 to 2.9.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9 to 2.9.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9...2.9.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 4 +--- requirements-docs.txt | 7 ++++--- requirements-tests.txt | 6 +++--- requirements.txt | 4 +--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cb2edc22..36a0bd7f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,14 +10,13 @@ certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests -cryptography==2.9 # via secretstorage +cryptography==2.9.2 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv flake8==3.5.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -importlib-metadata==1.6.0 # via keyring, pre-commit, twine, virtualenv invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine @@ -41,7 +40,6 @@ twine==3.1.1 # via -r requirements-dev.in urllib3==1.25.8 # via requests virtualenv==20.0.17 # via pre-commit webencodings==0.5.1 # via bleach -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 54bde14e..c1abe7e0 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -25,7 +25,7 @@ cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynac chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask cloudflare==2.6.5 # via -r requirements.txt -cryptography==2.9 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +cryptography==2.9.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 docutils==0.15.2 # via -r requirements.txt, botocore, sphinx @@ -45,7 +45,6 @@ gunicorn==20.0.4 # via -r requirements.txt hvac==0.10.1 # via -r requirements.txt idna==2.9 # via -r requirements.txt, requests imagesize==1.2.0 # via sphinx -importlib-metadata==1.6.0 # via -r requirements.txt, kombu inflection==0.4.0 # via -r requirements.txt itsdangerous==1.1.0 # via -r requirements.txt, flask javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks @@ -66,6 +65,8 @@ packaging==20.3 # via sphinx paramiko==2.7.1 # via -r requirements.txt pem==20.1.0 # via -r requirements.txt psycopg2==2.8.5 # via -r requirements.txt +pyasn1-modules==0.2.8 # via -r requirements.txt, pyjks, python-ldap +pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx @@ -78,6 +79,7 @@ pyrfc3339==1.1 # via -r requirements.txt, acme python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore python-editor==1.0.4 # via -r requirements.txt, alembic python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +python-ldap==3.2.0 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt @@ -105,7 +107,6 @@ urllib3==1.25.8 # via -r requirements.txt, botocore, requests vine==1.3.0 # via -r requirements.txt, amqp, celery werkzeug==1.0.1 # via -r requirements.txt, flask xmltodict==0.12.0 # via -r requirements.txt -zipp==3.1.0 # via -r requirements.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 5f9cafcc..494ac1d4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -19,7 +19,7 @@ cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.1 # via black, flask coverage==5.1 # via -r requirements-tests.in -cryptography==2.9 # via moto, sshpubkeys +cryptography==2.9.2 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore @@ -33,7 +33,7 @@ future==0.18.2 # via aws-xray-sdk gitdb==4.0.4 # via gitpython gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle itsdangerous==1.1.0 # via flask jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore @@ -72,7 +72,7 @@ requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 -six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto diff --git a/requirements.txt b/requirements.txt index bdad75b0..70a04ee1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask cloudflare==2.6.5 # via -r requirements.in -cryptography==2.9 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +cryptography==2.9.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore @@ -42,7 +42,6 @@ future==0.18.2 # via -r requirements.in, cloudflare gunicorn==20.0.4 # via -r requirements.in hvac==0.10.1 # via -r requirements.in idna==2.9 # via requests -importlib-metadata==1.6.0 # via kombu inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks @@ -92,7 +91,6 @@ urllib3==1.25.8 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==1.0.1 # via flask xmltodict==0.12.0 # via -r requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From f68900d2b355660f495f9243cdab19cdc5338a30 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 May 2020 18:28:01 -0700 Subject: [PATCH 129/150] improving logging and the possibility of defining which Authorities qualify for auto-rotation --- lemur/certificates/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index ca6b0248..3be794e7 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -483,18 +483,35 @@ def check_revoked(): @manager.command def automatically_enable_autorotate(): """ - This function automatically enables autorotation for unexpired certificates that are + This function automatically enables auto-rotation for unexpired certificates that are attached to an endpoint but do not have autorotate enabled. + + WARNING: This will overwrite the Auto-rotate toggle! """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", } + permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", []) + eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate() for cert in eligible_certs: + + if cert.authority_id not in permitted_authorities: + continue + log_data["certificate"] = cert.name log_data["certificate_id"] = cert.id log_data["message"] = "Enabling auto-rotate for certificate" current_app.logger.info(log_data) cert.rotation = True database.update(cert) + # TODO: add the cert destination to the logging + metrics.send("automatically_enable_autorotate", + "counter", 1, + metric_tags={"certificate": cert.name, + "certificate_id": cert.id, + "authority_id": cert.authority_id, + "authority_name": Authority.get(cert.authority_id).name}) + cert.rotation = True + database.update(cert) From 529ee04ae748f1b138ca90929bdf1b72a1e9b5f2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 8 May 2020 09:15:18 -0700 Subject: [PATCH 130/150] removing duplicate line --- lemur/certificates/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 3be794e7..54455eec 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -504,8 +504,6 @@ def automatically_enable_autorotate(): log_data["certificate_id"] = cert.id log_data["message"] = "Enabling auto-rotate for certificate" current_app.logger.info(log_data) - cert.rotation = True - database.update(cert) # TODO: add the cert destination to the logging metrics.send("automatically_enable_autorotate", "counter", 1, From fa13bda99e2ff56355ee5aa6ab4d313282ec019c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 16:45:06 +0000 Subject: [PATCH 131/150] Bump sphinx from 3.0.1 to 3.0.3 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.0.1 to 3.0.3. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.0.1...v3.0.3) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index c1abe7e0..7dc714a9 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -91,7 +91,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in -sphinx==3.0.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.0.3 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 8e2226180a38af17e2589cd60fe235374f6c3946 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 18:35:37 +0000 Subject: [PATCH 132/150] Bump pre-commit from 2.2.0 to 2.3.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.2.0...v2.3.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 36a0bd7f..68e28b73 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.2.0 # via -r requirements-dev.in +pre-commit==2.3.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From 5ad9c11716565bf67c81fdb98d849b44b6dada44 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 18:42:41 +0000 Subject: [PATCH 133/150] Bump arrow from 0.15.5 to 0.15.6 Bumps [arrow](https://github.com/crsmithdev/arrow) from 0.15.5 to 0.15.6. - [Release notes](https://github.com/crsmithdev/arrow/releases) - [Changelog](https://github.com/crsmithdev/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/crsmithdev/arrow/compare/0.15.5...0.15.6) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 7dc714a9..db255de3 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -10,7 +10,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate amqp==2.5.2 # via -r requirements.txt, kombu aniso8601==8.0.0 # via -r requirements.txt, flask-restful -arrow==0.15.5 # via -r requirements.txt +arrow==0.15.6 # via -r requirements.txt asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko diff --git a/requirements.txt b/requirements.txt index 70a04ee1..f8d553d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.5 # via -r requirements.in +arrow==0.15.6 # via -r requirements.in asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery From dac95313b84c4db66d179531e7e8e2f7f56dc35f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:29:51 +0000 Subject: [PATCH 134/150] Bump pyjks from 19.0.0 to 20.0.0 Bumps [pyjks](https://github.com/kurtbrose/pyjks) from 19.0.0 to 20.0.0. - [Release notes](https://github.com/kurtbrose/pyjks/releases) - [Changelog](https://github.com/kurtbrose/pyjks/blob/master/CHANGELOG.md) - [Commits](https://github.com/kurtbrose/pyjks/compare/v19.0.0...v20.0.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index db255de3..14e54b59 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -70,7 +70,7 @@ pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-mod pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx -pyjks==19.0.0 # via -r requirements.txt +pyjks==20.0.0 # via -r requirements.txt pyjwt==1.7.1 # via -r requirements.txt pynacl==1.3.0 # via -r requirements.txt, paramiko pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests diff --git a/requirements.txt b/requirements.txt index f8d553d4..83585828 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,7 @@ pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via cffi pycryptodomex==3.9.7 # via pyjks -pyjks==19.0.0 # via -r requirements.in +pyjks==20.0.0 # via -r requirements.in pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests From 6bef8fb9d7f86e7e76b9ef4489367f03a6d72489 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:36:54 +0000 Subject: [PATCH 135/150] Bump acme from 1.3.0 to 1.4.0 Bumps [acme](https://github.com/letsencrypt/letsencrypt) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/letsencrypt/letsencrypt/releases) - [Commits](https://github.com/letsencrypt/letsencrypt/compare/v1.3.0...v1.4.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 3 +-- requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 14e54b59..6c2a37e4 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.3.0 # via -r requirements.txt +acme==1.4.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate @@ -59,7 +59,6 @@ mako==1.1.2 # via -r requirements.txt, alembic markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy -mock==4.0.2 # via -r requirements.txt, acme ndg-httpsclient==0.5.1 # via -r requirements.txt packaging==20.3 # via sphinx paramiko==2.7.1 # via -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 83585828..ad92d95f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.3.0 # via -r requirements.in +acme==1.4.0 # via -r requirements.in alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu @@ -56,7 +56,6 @@ mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.22.3 # via -r requirements.in marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy -mock==4.0.2 # via acme ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.1 # via -r requirements.in pem==20.1.0 # via -r requirements.in From 5ac65062c93e52bcac58209337018aaae3db3e19 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:49:15 +0000 Subject: [PATCH 136/150] Bump sqlalchemy-utils from 0.36.3 to 0.36.5 Bumps [sqlalchemy-utils](https://github.com/kvesteri/sqlalchemy-utils) from 0.36.3 to 0.36.5. - [Release notes](https://github.com/kvesteri/sqlalchemy-utils/releases) - [Changelog](https://github.com/kvesteri/sqlalchemy-utils/blob/master/CHANGES.rst) - [Commits](https://github.com/kvesteri/sqlalchemy-utils/compare/0.36.3...0.36.5) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 6c2a37e4..d8bc26d7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -98,7 +98,7 @@ sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sqlalchemy-utils==0.36.3 # via -r requirements.txt +sqlalchemy-utils==0.36.5 # via -r requirements.txt sqlalchemy==1.3.16 # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.7 # via -r requirements.txt twofish==0.3.0 # via -r requirements.txt, pyjks diff --git a/requirements.txt b/requirements.txt index ad92d95f..60b38553 100644 --- a/requirements.txt +++ b/requirements.txt @@ -82,7 +82,7 @@ requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, retrying==1.3.3 # via -r requirements.in s3transfer==0.3.3 # via boto3 six==1.14.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils -sqlalchemy-utils==0.36.3 # via -r requirements.in +sqlalchemy-utils==0.36.5 # via -r requirements.in sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.7 # via -r requirements.in twofish==0.3.0 # via pyjks From 5afa32fa226ff972eb4ca2d078058e5258ea6874 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:57:55 +0000 Subject: [PATCH 137/150] Bump boto3 from 1.12.39 to 1.13.6 Bumps [boto3](https://github.com/boto/boto3) from 1.12.39 to 1.13.6. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.12.39...1.13.6) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d8bc26d7..a468419c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,8 +16,8 @@ babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.12.39 # via -r requirements.txt -botocore==1.15.39 # via -r requirements.txt, boto3, s3transfer +boto3==1.13.6 # via -r requirements.txt +botocore==1.16.6 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 9c9bbff0..5bb54cc6 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in -boto3==1.12.39 # via aws-sam-translator, moto +boto3==1.13.6 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.16.6 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 60b38553..c2e269e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,8 @@ asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.12.39 # via -r requirements.in -botocore==1.15.39 # via -r requirements.in, boto3, s3transfer +boto3==1.13.6 # via -r requirements.in +botocore==1.16.6 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 201a96ff72d7c518a37b5497fbb920c61d5a6144 Mon Sep 17 00:00:00 2001 From: Luka Matijevic Date: Mon, 11 May 2020 13:09:23 +0200 Subject: [PATCH 138/150] Improve periodic tasks docs --- docs/production/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index cd044ca4..b91ed6bd 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -390,6 +390,10 @@ Here are the Celery configuration variables that should be set:: CELERY_IMPORTS = ('lemur.common.celery') CELERY_TIMEZONE = 'UTC' +Do not forget to import crontab module in your configuration file:: + + from celery.task.schedules import crontab + You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks. The scheduler can be started with:: From 5d75204a4f81ec30bdebfba277b8def133dc2a1f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 13:45:08 +0000 Subject: [PATCH 139/150] Bump faker from 4.0.3 to 4.1.0 Bumps [faker](https://github.com/joke2k/faker) from 4.0.3 to 4.1.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.0.3...v4.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 5bb54cc6..32b4d72b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in -faker==4.0.3 # via -r requirements-tests.in, factory-boy +faker==4.1.0 # via -r requirements-tests.in, factory-boy fakeredis==1.4.1 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in From c0829478fdf62d46b7ba006129ec21b8489c5ba0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:14:27 +0000 Subject: [PATCH 140/150] Bump requests-mock from 1.7.0 to 1.8.0 Bumps [requests-mock](https://github.com/jamielennox/requests-mock) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/jamielennox/requests-mock/releases) - [Commits](https://github.com/jamielennox/requests-mock/compare/1.7.0...1.8.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 32b4d72b..cac0f42a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -67,7 +67,7 @@ pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto redis==3.4.1 # via fakeredis regex==2020.4.4 # via black -requests-mock==1.7.0 # via -r requirements-tests.in +requests-mock==1.8.0 # via -r requirements-tests.in requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose From 9e1737edc49b307a8526b278ddec8010620ac6d5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:25:47 +0000 Subject: [PATCH 141/150] Bump pytest from 5.4.1 to 5.4.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.1 to 5.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.1...5.4.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index cac0f42a..369408dd 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -60,7 +60,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.0.0 # via -r requirements-tests.in -pytest==5.4.1 # via -r requirements-tests.in, pytest-flask, pytest-mock +pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto From dfd9acb0d7f0b604162c57ebf791c7a060a569a3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:37:21 +0000 Subject: [PATCH 142/150] Bump pytest-mock from 3.0.0 to 3.1.0 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.0.0...v3.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 369408dd..3b5f83db 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -59,7 +59,7 @@ pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in -pytest-mock==3.0.0 # via -r requirements-tests.in +pytest-mock==3.1.0 # via -r requirements-tests.in pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto From 23cd7d3d625a8919ebeb9d8497ad76e0e30154ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:47:18 +0000 Subject: [PATCH 143/150] Bump marshmallow-sqlalchemy from 0.22.3 to 0.23.0 Bumps [marshmallow-sqlalchemy](https://github.com/marshmallow-code/marshmallow-sqlalchemy) from 0.22.3 to 0.23.0. - [Release notes](https://github.com/marshmallow-code/marshmallow-sqlalchemy/releases) - [Changelog](https://github.com/marshmallow-code/marshmallow-sqlalchemy/blob/dev/CHANGELOG.rst) - [Commits](https://github.com/marshmallow-code/marshmallow-sqlalchemy/compare/0.22.3...0.23.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index a468419c..8b22bb16 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -57,7 +57,7 @@ lockfile==0.12.2 # via -r requirements.txt logmatic-python==0.1.7 # via -r requirements.txt mako==1.1.2 # via -r requirements.txt, alembic markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako -marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt +marshmallow-sqlalchemy==0.23.0 # via -r requirements.txt marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy ndg-httpsclient==0.5.1 # via -r requirements.txt packaging==20.3 # via sphinx diff --git a/requirements.txt b/requirements.txt index c2e269e5..40fff126 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ lockfile==0.12.2 # via -r requirements.in logmatic-python==0.1.7 # via -r requirements.in mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.22.3 # via -r requirements.in +marshmallow-sqlalchemy==0.23.0 # via -r requirements.in marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.1 # via -r requirements.in From d3a0fe74911bbe891bc7cd3b7d4d91b448229454 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:58:47 +0000 Subject: [PATCH 144/150] Bump cloudflare from 2.6.5 to 2.7.1 Bumps [cloudflare](https://github.com/cloudflare/python-cloudflare) from 2.6.5 to 2.7.1. - [Release notes](https://github.com/cloudflare/python-cloudflare/releases) - [Changelog](https://github.com/cloudflare/python-cloudflare/blob/master/CHANGELOG.md) - [Commits](https://github.com/cloudflare/python-cloudflare/compare/2.6.5...2.7.1) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 8b22bb16..33b53872 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,7 +24,7 @@ certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask -cloudflare==2.6.5 # via -r requirements.txt +cloudflare==2.7.1 # via -r requirements.txt cryptography==2.9.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 diff --git a/requirements.txt b/requirements.txt index 40fff126..d413c566 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask -cloudflare==2.6.5 # via -r requirements.in +cloudflare==2.7.1 # via -r requirements.in cryptography==2.9.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 From 9bc017c9bf4e2d81411750f06ce00a537f703d01 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:11:39 +0000 Subject: [PATCH 145/150] Bump botocore from 1.16.6 to 1.16.11 Bumps [botocore](https://github.com/boto/botocore) from 1.16.6 to 1.16.11. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.16.6...1.16.11) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 33b53872..7d49fe56 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.13.6 # via -r requirements.txt -botocore==1.16.6 # via -r requirements.txt, boto3, s3transfer +botocore==1.16.11 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 3b5f83db..5ca1f238 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in boto3==1.13.6 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.16.6 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.16.11 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d413c566..72f3938b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.13.6 # via -r requirements.in -botocore==1.16.6 # via -r requirements.in, boto3, s3transfer +botocore==1.16.11 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From d010553d23b1166122dab68394973f0ffc5c8847 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:23:23 +0000 Subject: [PATCH 146/150] Bump redis from 3.4.1 to 3.5.2 Bumps [redis](https://github.com/andymccurdy/redis-py) from 3.4.1 to 3.5.2. - [Release notes](https://github.com/andymccurdy/redis-py/releases) - [Changelog](https://github.com/andymccurdy/redis-py/blob/master/CHANGES) - [Commits](https://github.com/andymccurdy/redis-py/compare/3.4.1...3.5.2) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 7d49fe56..d3e61970 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -82,7 +82,7 @@ python-ldap==3.2.0 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt -redis==3.4.1 # via -r requirements.txt, celery +redis==3.5.2 # via -r requirements.txt, celery requests-toolbelt==0.9.1 # via -r requirements.txt, acme requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx retrying==1.3.3 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 5ca1f238..93d992e5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -65,7 +65,7 @@ python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto -redis==3.4.1 # via fakeredis +redis==3.5.2 # via fakeredis regex==2020.4.4 # via black requests-mock==1.8.0 # via -r requirements-tests.in requests==2.23.0 # via docker, moto, requests-mock, responses diff --git a/requirements.txt b/requirements.txt index 72f3938b..0ad0708c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,7 +76,7 @@ python-ldap==3.2.0 # via -r requirements.in pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.in, cloudflare raven[flask]==6.10.0 # via -r requirements.in -redis==3.4.1 # via -r requirements.in, celery +redis==3.5.2 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt retrying==1.3.3 # via -r requirements.in From 2bf03a0bc245c20524034b72477104bbd3f0a4b2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:35:43 +0000 Subject: [PATCH 147/150] Bump boto3 from 1.13.6 to 1.13.11 Bumps [boto3](https://github.com/boto/boto3) from 1.13.6 to 1.13.11. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.13.6...1.13.11) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d3e61970..16d97413 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,7 +16,7 @@ babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.13.6 # via -r requirements.txt +boto3==1.13.11 # via -r requirements.txt botocore==1.16.11 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index 93d992e5..79340e51 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in -boto3==1.13.6 # via aws-sam-translator, moto +boto3==1.13.11 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.16.11 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests diff --git a/requirements.txt b/requirements.txt index 0ad0708c..315f39b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.13.6 # via -r requirements.in +boto3==1.13.11 # via -r requirements.in botocore==1.16.11 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests From 4d4ff8509062dc82369c00b107120a4c5bffb88d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:44:22 +0000 Subject: [PATCH 148/150] Bump pre-commit from 2.3.0 to 2.4.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.3.0...v2.4.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 68e28b73..785d3f29 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.3.0 # via -r requirements-dev.in +pre-commit==2.4.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From fd444403bb8d97f7c72fe0c434ac15f8b7b8bcba Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 21 May 2020 15:32:38 -0700 Subject: [PATCH 149/150] improved logging. - adding destination name, fixing broken metric. --- lemur/certificates/cli.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 54455eec..d007e458 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -15,6 +15,7 @@ from tabulate import tabulate from lemur import database from lemur.authorities.models import Authority +from lemur.authorities.service import get as authorities_get_by_id from lemur.certificates.models import Certificate from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.service import ( @@ -490,6 +491,7 @@ def automatically_enable_autorotate(): """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Enabling auto-rotate for certificate" } permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", []) @@ -502,14 +504,20 @@ def automatically_enable_autorotate(): log_data["certificate"] = cert.name log_data["certificate_id"] = cert.id - log_data["message"] = "Enabling auto-rotate for certificate" + log_data["authority_id"] = cert.authority_id + log_data["authority_name"] = authorities_get_by_id(cert.authority_id).name + if cert.destinations: + log_data["destination_names"] = ', '.join([d.label for d in cert.destinations]) + else: + log_data["destination_names"] = "NONE" current_app.logger.info(log_data) - # TODO: add the cert destination to the logging metrics.send("automatically_enable_autorotate", "counter", 1, - metric_tags={"certificate": cert.name, - "certificate_id": cert.id, - "authority_id": cert.authority_id, - "authority_name": Authority.get(cert.authority_id).name}) + metric_tags={"certificate": log_data["certificate"], + "certificate_id": log_data["certificate_id"], + "authority_id": log_data["authority_id"], + "authority_name": log_data["authority_name"], + "destination_names": log_data["destination_names"] + }) cert.rotation = True database.update(cert) From ecf940bb8f769b6f53677537c994e7b89f5dde40 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 22 May 2020 10:44:02 -0700 Subject: [PATCH 150/150] Update README.rst updating python --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d42bc810..16b14d79 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ Lemur Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults. -It works on CPython 3.5. We deploy on Ubuntu and develop on OS X. +It works on Python 3.7. We deploy on Ubuntu and develop on OS X. Project resources