From cdb83c48c5504130a6370703845b812af7bf6c51 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 17 Jun 2019 10:41:11 -0700 Subject: [PATCH 01/66] API additions for viewing expired certs as well. Default behavior modified to show only valid certs and those which have expired less than 1 month ago. --- lemur/certificates/service.py | 12 +++++++----- lemur/certificates/views.py | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 544c03d8..d9370232 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -329,12 +329,14 @@ def render(args): """ query = database.session_query(Certificate) - time_range = args.pop("time_range") - if not time_range: - six_month_old = arrow.now()\ - .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -6))\ + show_expired = args.pop("showExpired") + if show_expired != 1: + one_month_old = arrow.now()\ + .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\ .format("YYYY-MM-DD") - query = query.filter(Certificate.not_after > six_month_old) + query = query.filter(Certificate.not_after > one_month_old) + + time_range = args.pop("time_range") destination_id = args.pop("destination_id") notification_id = args.pop("notification_id", None) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 61a74a59..1a003e78 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -347,6 +347,7 @@ class CertificatesList(AuthenticatedResource): ) parser.add_argument("creator", type=str, location="args") parser.add_argument("show", type=str, location="args") + parser.add_argument("showExpired", type=int, location="args") args = parser.parse_args() args["user"] = g.user From 41c781318cdec7a1ac1b793ee790481141c6b63b Mon Sep 17 00:00:00 2001 From: arnydo Date: Wed, 10 Jul 2019 10:08:14 -0400 Subject: [PATCH 02/66] Add ADCS in Docs Add info regarding the ADCS plugin created by "https://github.com/sirferl/lemur". "lemur_adcs" plugin is part of Lemur by default so I added to main plugins section within Docs. --- docs/administration.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 491edcf1..ac8fd542 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1086,6 +1086,18 @@ progress and the API is not frozen. Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec. +Active Directory Certificate Services +-------- + +:Authors: + sirferl +:Type: + Issuer +:Description: + Enables the creation of certificates by ADCS (Active Directory Certificate Services) +:Links: + https://github.com/sirferl/lemur + Verisign/Symantec ----------------- From 66bff57c04345facf7d43064f91331f498a25e20 Mon Sep 17 00:00:00 2001 From: arnydo Date: Wed, 10 Jul 2019 12:10:47 -0400 Subject: [PATCH 03/66] Add ADCS Plugin Configuration to Docs Add configuration options based on https://github.com/Netflix/lemur/pull/2255#issue-240136873 --- docs/administration.rst | 74 +++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index ac8fd542..e292ae03 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -593,8 +593,60 @@ If you are not using a metric provider you do not need to configure any of these Plugin Specific Options ----------------------- +Active Directory Certificate Services Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. data:: ADCS_SERVER + :noindex: + + FQDN of your ADCS Server + + +.. data:: ADCS_AUTH_METHOD + :noindex: + + The chosen authentication method. Either ‘basic’ (the default), ‘ntlm’ or ‘cert’ (SSL client certificate). The next 2 variables are interpreted differently for different methods. + + +.. data:: ADCS_USER + :noindex: + + The username (basic) or the path to the public cert (cert) of the user accessing PKI + + +.. data:: ADCS_PWD + :noindex: + + The passwd (basic) or the path to the private key (cert) of the user accessing PKI + + +.. data:: ADCS_TEMPLATE + :noindex: + + Template to be used for certificate issuing. Usually display name w/o spaces + + +.. data:: ADCS_START + :noindex: + +.. data:: ADCS_STOP + :noindex: + +.. data:: ADCS_ISSUING + :noindex: + + Contains the issuing cert of the CA + + +.. data:: ADCS_ROOT + :noindex: + + Contains the root cert of the CA + + Verisign Issuer Plugin -^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~ Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur, Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation @@ -642,7 +694,7 @@ for those plugins. Digicert Issuer Plugin -^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~ The following configuration properties are required to use the Digicert issuer plugin. @@ -690,7 +742,7 @@ The following configuration properties are required to use the Digicert issuer p CFSSL Issuer Plugin -^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~ The following configuration properties are required to use the CFSSL issuer plugin. @@ -716,7 +768,7 @@ The following configuration properties are required to use the CFSSL issuer plug Hashicorp Vault Source/Destination Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination. @@ -738,7 +790,7 @@ Vault Destination supports a regex filter to prevent certificates with SAN that AWS Source/Destination Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions. @@ -1086,18 +1138,6 @@ progress and the API is not frozen. Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec. -Active Directory Certificate Services --------- - -:Authors: - sirferl -:Type: - Issuer -:Description: - Enables the creation of certificates by ADCS (Active Directory Certificate Services) -:Links: - https://github.com/sirferl/lemur - Verisign/Symantec ----------------- From 2628ed1a8272e100c2c36e852cf502349c9b7e96 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Jul 2019 23:00:35 -0700 Subject: [PATCH 04/66] better alerting --- lemur/common/celery.py | 65 ++++++++++++++++++++++++++++++++++++++++-- lemur/common/redis.py | 33 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 lemur/common/redis.py diff --git a/lemur/common/celery.py b/lemur/common/celery.py index d3cc7621..b775396a 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -9,6 +9,7 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B """ import copy import sys +import time from datetime import datetime, timezone, timedelta from celery import Celery @@ -16,6 +17,7 @@ from celery.exceptions import SoftTimeLimitExceeded from flask import current_app from lemur.authorities.service import get as get_authority +from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.extensions import metrics, sentry from lemur.factory import create_app @@ -30,6 +32,9 @@ if current_app: else: flask_app = create_app() +red = RedisHandler(host=current_app.config.get('REDIS_HOST', 'localhost'), + port=current_app.config.get('REDIS_PORT', 6379), + db=current_app.config.get('REDIS_DB', 0)).redis() def make_celery(app): celery = Celery( @@ -68,6 +73,30 @@ def is_task_active(fun, task_id, args): return False +@celery.task() +def report_celery_last_success_metrics(): + """ + For each celery task, this will determine the number of seconds since it has last been successful. + + Celery tasks should be emitting redis stats with a deterministic key (In our case, `f"{task}.last_success"`. + report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful. + Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted + from this function. + + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_time = int(time.time()) + schedule = current_app.config.get('CELERYBEAT_SCHEDULE') + for _, t in schedule.items(): + task = t.get("task") + last_success = int(red.get(f"{task}.last_success") or 0) + metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success) + red.set( + f"{function}.last_success", int(time.time()) + ) # Alert if this metric is not seen + metrics.send(f"{function}.success", 'counter', 1) + + @celery.task(soft_time_limit=600) def fetch_acme_cert(id): """ @@ -80,8 +109,9 @@ def fetch_acme_cert(id): if celery.current_task: task_id = celery.current_task.request.id + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) log_data = { - "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "function": function, "message": "Resolving pending certificate {}".format(id), "task_id": task_id, "id": id, @@ -165,11 +195,15 @@ def fetch_acme_cert(id): log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) + metrics.send(f"{function}.resolved", 'gauge', new) + metrics.send(f"{function}.failed", 'gauge', failed) + metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, failed=failed, wrong_issuer=wrong_issuer ) ) + red.set(f'{function}.last_success', int(time.time())) @celery.task() @@ -177,8 +211,9 @@ def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" pending_certs = pending_certificate_service.get_unresolved_pending_certs() + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) log_data = { - "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "function": function, "message": "Starting job.", } @@ -195,11 +230,18 @@ def fetch_all_pending_acme_certs(): current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task() def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" - log_data = {"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)} + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + log_data = { + "function": function, + "message": "Starting job.", + } pending_certs = pending_certificate_service.get_pending_certs("all") # Delete pending certs more than a week old @@ -211,6 +253,9 @@ def remove_old_acme_certs(): current_app.logger.debug(log_data) pending_certificate_service.delete(cert) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task() def clean_all_sources(): @@ -218,6 +263,7 @@ def clean_all_sources(): This function will clean unused certificates from sources. This is a destructive operation and should only be ran periodically. This function triggers one celery task per source. """ + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) sources = validate_sources("all") for source in sources: current_app.logger.debug( @@ -225,6 +271,9 @@ def clean_all_sources(): ) clean_source.delay(source.label) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task() def clean_source(source): @@ -244,6 +293,7 @@ def sync_all_sources(): """ This function will sync certificates from all sources. This function triggers one celery task per source. """ + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) sources = validate_sources("all") for source in sources: current_app.logger.debug( @@ -251,6 +301,9 @@ def sync_all_sources(): ) sync_source.delay(source.label) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task(soft_time_limit=7200) def sync_source(source): @@ -279,6 +332,7 @@ def sync_source(source): return try: sync([source]) + metrics.send(f"{function}.success", 'counter', '1', metric_tags={"source": source}) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) @@ -290,6 +344,8 @@ def sync_source(source): log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) + metrics.send(f"{function}.success", 'counter', 1, metric_tags=source) + red.set(f'{function}.last_success', int(time.time())) @celery.task() @@ -302,9 +358,12 @@ def sync_source_destination(): We rely on account numbers to avoid duplicates. """ current_app.logger.debug("Syncing AWS destinations and sources") + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) for dst in destinations_service.get_all(): if add_aws_destination_to_sources(dst): current_app.logger.debug("Source: %s added", dst.label) current_app.logger.debug("Completed Syncing AWS destinations and sources") + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) diff --git a/lemur/common/redis.py b/lemur/common/redis.py new file mode 100644 index 00000000..a996ad67 --- /dev/null +++ b/lemur/common/redis.py @@ -0,0 +1,33 @@ +""" +Helper Class for Redis + +""" +import redis +#from flask import current_app + + +class RedisHandler: + #def __init__(self, host=current_app.config.get('REDIS_HOST', 'localhost'), + # port=current_app.config.get('REDIS_PORT', 6379), + # db=current_app.config.get('REDIS_DB', 0)): + def __init__(self, host, port, db): + self.host = host + self.port = port + self.db = db + + def redis(self, db=0): + # The decode_responses flag here directs the client to convert the responses from Redis into Python strings + # using the default encoding utf-8. This is client specific. + red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, charset="utf-8", decode_responses=True) + return red + + +def redis_get(key, default=None): + red = RedisHandler().redis() + try: + v = red.get(key) + except redis.exceptions.ConnectionError: + v = None + if not v: + return default + return v From 97d74bfa1d4a946e8002042eb2c20032353dc1e7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Jul 2019 08:47:39 -0700 Subject: [PATCH 05/66] fixing the app context issue. we will create an app if no current_app available --- lemur/common/celery.py | 5 ++--- lemur/common/redis.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b775396a..05e66926 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -32,9 +32,8 @@ if current_app: else: flask_app = create_app() -red = RedisHandler(host=current_app.config.get('REDIS_HOST', 'localhost'), - port=current_app.config.get('REDIS_PORT', 6379), - db=current_app.config.get('REDIS_DB', 0)).redis() +red = RedisHandler().redis() + def make_celery(app): celery = Celery( diff --git a/lemur/common/redis.py b/lemur/common/redis.py index a996ad67..4af39aef 100644 --- a/lemur/common/redis.py +++ b/lemur/common/redis.py @@ -3,14 +3,18 @@ Helper Class for Redis """ import redis -#from flask import current_app +from flask import current_app +from lemur.factory import create_app +if current_app: + flask_app = current_app +else: + flask_app = create_app() class RedisHandler: - #def __init__(self, host=current_app.config.get('REDIS_HOST', 'localhost'), - # port=current_app.config.get('REDIS_PORT', 6379), - # db=current_app.config.get('REDIS_DB', 0)): - def __init__(self, host, port, db): + def __init__(self, host=flask_app.config.get('REDIS_HOST', 'localhost'), + port=flask_app.config.get('REDIS_PORT', 6379), + db=flask_app.config.get('REDIS_DB', 0)): self.host = host self.port = port self.db = db From 1b1bdbb261c3f6b03bf317cf91a154c3b19d06c9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Jul 2019 10:25:37 -0700 Subject: [PATCH 06/66] spacing --- lemur/common/redis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/common/redis.py b/lemur/common/redis.py index 4af39aef..0bddf9b4 100644 --- a/lemur/common/redis.py +++ b/lemur/common/redis.py @@ -11,6 +11,7 @@ if current_app: else: flask_app = create_app() + class RedisHandler: def __init__(self, host=flask_app.config.get('REDIS_HOST', 'localhost'), port=flask_app.config.get('REDIS_PORT', 6379), From ae1633b0f2afc29a1ff618c2d7b2944bb2b6cf66 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Jul 2019 10:38:47 -0700 Subject: [PATCH 07/66] updating requirements has been a while since last update, more testing in deployment needed --- requirements-dev.txt | 19 ++++++++------- requirements-docs.txt | 47 +++++++++++++++++++------------------ requirements-tests.txt | 53 ++++++++++++++++++++++++------------------ requirements.txt | 45 ++++++++++++++++++----------------- 4 files changed, 90 insertions(+), 74 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 030c3f93..5e7b36f2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,31 +6,34 @@ # aspy.yaml==1.3.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2019.3.9 # via requests +certifi==2019.6.16 # via requests cfgv==2.0.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.4.3 # via pre-commit +identify==1.4.5 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.17 # via pre-commit +importlib-metadata==0.18 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.16.1 +pre-commit==1.17.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.4.2 # via readme-renderer -pyyaml==5.1 +pyyaml==5.1.1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.32.1 # via twine +tqdm==4.32.2 # via twine twine==1.13.0 urllib3==1.25.3 # via requests -virtualenv==16.6.0 # via pre-commit +virtualenv==16.6.1 # via pre-commit webencodings==0.5.1 # via bleach -zipp==0.5.1 # via importlib-metadata +zipp==0.5.2 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index c0fe427e..6222687e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,23 +4,23 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.34.2 +acme==0.36.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.10 +alembic==1.0.11 amqp==2.5.0 -aniso8601==6.0.0 +aniso8601==7.0.0 arrow==0.14.2 asn1crypto==0.24.0 asyncpool==1.0 babel==2.7.0 # via sphinx -bcrypt==3.1.6 +bcrypt==3.1.7 billiard==3.6.0.0 blinker==1.4 -boto3==1.9.160 -botocore==1.12.160 +boto3==1.9.187 +botocore==1.12.187 celery[redis]==4.3.0 -certifi==2019.3.9 +certifi==2019.6.16 certsrv==2.1.1 cffi==1.12.3 chardet==3.0.4 @@ -32,7 +32,7 @@ dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.7 +flask-cors==3.0.8 flask-mail==0.9.1 flask-migrate==2.5.2 flask-principal==0.4.0 @@ -40,10 +40,10 @@ flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.0 -flask==1.0.3 +flask==1.1.1 future==0.17.1 gunicorn==19.9.0 -hvac==0.9.1 +hvac==0.9.3 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 @@ -51,21 +51,21 @@ itsdangerous==1.1.0 javaobj-py3==0.3.0 jinja2==2.10.1 jmespath==0.9.4 -josepy==1.1.0 +josepy==1.2.0 jsonlines==1.2.0 kombu==4.5.0 lockfile==0.12.2 logmatic-python==0.1.7 -mako==1.0.11 +mako==1.0.13 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.3 -marshmallow==2.19.2 +marshmallow-sqlalchemy==0.17.0 +marshmallow==2.19.5 mock==3.0.5 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx -paramiko==2.4.2 +paramiko==2.6.0 pem==19.1.0 -psycopg2==2.8.2 +psycopg2==2.8.3 pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 @@ -81,17 +81,17 @@ python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.11 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.1 raven[flask]==6.10.0 redis==3.2.1 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 -s3transfer==0.2.0 +s3transfer==0.2.1 six==1.12.0 -snowballstemmer==1.2.1 # via sphinx +snowballstemmer==1.9.0 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==2.1.0 +sphinx==2.1.2 sphinxcontrib-applehelp==1.0.1 # via sphinx sphinxcontrib-devhelp==1.0.1 # via sphinx sphinxcontrib-htmlhelp==1.0.2 # via sphinx @@ -99,11 +99,14 @@ sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.4 +sqlalchemy-utils==0.34.0 +sqlalchemy==1.3.5 tabulate==0.8.3 twofish==0.3.0 urllib3==1.25.3 vine==1.3.0 werkzeug==0.15.4 xmltodict==0.12.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via acme, josepy, sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 77bc92af..f88d44cf 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,33 +7,34 @@ appdirs==1.4.3 # via black asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==19.1.0 # via black, pytest -aws-sam-translator==1.11.0 # via cfn-lint +attrs==19.1.0 # via black, jsonschema, pytest +aws-sam-translator==1.12.0 # via cfn-lint aws-xray-sdk==2.4.2 # via moto -bandit==1.6.0 +bandit==1.6.2 black==19.3b0 -boto3==1.9.160 # via aws-sam-translator, moto +boto3==1.9.187 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.160 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.3.9 # via requests +botocore==1.12.187 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2019.6.16 # via requests cffi==1.12.3 # via cryptography -cfn-lint==0.21.4 # via moto +cfn-lint==0.22.2 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask coverage==4.5.3 -cryptography==2.7 # via moto -docker==4.0.1 # via moto +cryptography==2.7 # via moto, sshpubkeys +datetime==4.3 # via moto +docker==4.0.2 # via moto docutils==0.14 # via botocore -ecdsa==0.13.2 # via python-jose +ecdsa==0.13.2 # via python-jose, sshpubkeys factory-boy==2.12.0 faker==1.0.7 -flask==1.0.3 # via pytest-flask +flask==1.1.1 # via pytest-flask freezegun==0.3.12 future==0.17.1 # via aws-xray-sdk, python-jose gitdb2==2.0.5 # via gitpython gitpython==2.1.11 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.17 # via pluggy, pytest +importlib-metadata==0.18 # via pluggy, pytest itsdangerous==1.1.0 # via flask jinja2==2.10.1 # via flask, moto jmespath==0.9.4 # via boto3, botocore @@ -41,34 +42,36 @@ jsondiff==1.1.2 # via moto jsonpatch==1.23 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==2.6.0 # via aws-sam-translator, cfn-lint +jsonschema==3.0.1 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto -more-itertools==7.0.0 # via pytest -moto==1.3.8 +more-itertools==7.1.0 # via pytest +moto==1.3.11 nose==1.3.7 packaging==19.0 # via pytest -pbr==5.2.1 # via stevedore +pbr==5.4.0 # via stevedore pluggy==0.12.0 # via pytest py==1.8.0 # via pytest pyasn1==0.4.5 # via rsa pycparser==2.19 # via cffi pyflakes==2.1.1 pyparsing==2.4.0 # via packaging +pyrsistent==0.15.3 # via jsonschema pytest-flask==0.15.0 pytest-mock==1.10.4 -pytest==4.6.2 +pytest==5.0.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto -pytz==2019.1 # via moto -pyyaml==5.1 +pytz==2019.1 # via datetime, moto +pyyaml==5.1.1 requests-mock==1.6.0 requests==2.22.0 # via cfn-lint, docker, moto, requests-mock, responses responses==0.10.6 # via moto rsa==4.0 # via python-jose -s3transfer==0.2.0 # via boto3 -six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, mock, moto, packaging, pytest, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +s3transfer==0.2.1 # via boto3 +six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap2==2.0.5 # via gitdb2 +sshpubkeys==3.1.0 # via moto stevedore==1.30.1 # via bandit text-unidecode==1.2 # via faker toml==0.10.0 # via black @@ -76,6 +79,10 @@ urllib3==1.25.3 # via botocore, requests wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker werkzeug==0.15.4 # via flask, moto, pytest-flask -wrapt==1.11.1 # via aws-xray-sdk +wrapt==1.11.2 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==0.5.1 # via importlib-metadata +zipp==0.5.2 # via importlib-metadata +zope.interface==4.6.0 # via datetime + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via cfn-lint, jsonschema, zope.interface diff --git a/requirements.txt b/requirements.txt index c19c7b6e..7635c29d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,21 +4,21 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.34.2 +acme==0.36.0 alembic-autogenerate-enums==0.0.2 -alembic==1.0.10 # via flask-migrate +alembic==1.0.11 # via flask-migrate amqp==2.5.0 # via kombu -aniso8601==6.0.0 # via flask-restful +aniso8601==7.0.0 # via flask-restful arrow==0.14.2 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 -bcrypt==3.1.6 # via flask-bcrypt, paramiko +bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.0.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.160 -botocore==1.12.160 +boto3==1.9.187 +botocore==1.12.187 celery[redis]==4.3.0 -certifi==2019.3.9 +certifi==2019.6.16 certsrv==2.1.1 cffi==1.12.3 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -30,7 +30,7 @@ dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.7 +flask-cors==3.0.8 flask-mail==0.9.1 flask-migrate==2.5.2 flask-principal==0.4.0 @@ -38,32 +38,32 @@ flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.0 -flask==1.0.3 +flask==1.1.1 future==0.17.1 gunicorn==19.9.0 -hvac==0.9.1 +hvac==0.9.3 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask javaobj-py3==0.3.0 # via pyjks jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore -josepy==1.1.0 # via acme +josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.5.0 lockfile==0.12.2 logmatic-python==0.1.7 -mako==1.0.11 # via alembic +mako==1.0.13 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.3 -marshmallow==2.19.2 +marshmallow-sqlalchemy==0.17.0 +marshmallow==2.19.5 mock==3.0.5 # via acme ndg-httpsclient==0.5.1 -paramiko==2.4.2 +paramiko==2.6.0 pem==19.1.0 -psycopg2==2.8.2 +psycopg2==2.8.3 pyasn1-modules==0.2.5 # via pyjks, python-ldap -pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, pyjks, python-ldap +pyasn1==0.4.5 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi pycryptodomex==3.8.2 # via pyjks pyjks==19.0.0 @@ -76,19 +76,22 @@ python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1 +pyyaml==5.1.1 raven[flask]==6.10.0 redis==3.2.1 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 -s3transfer==0.2.0 # via boto3 +s3transfer==0.2.1 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.4 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.34.0 +sqlalchemy==1.3.5 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 twofish==0.3.0 # via pyjks urllib3==1.25.3 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.15.4 # via flask xmltodict==0.12.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via acme, josepy From cd1aeb15f179061ab02adb3b02c01909f6f4b19b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Jul 2019 11:50:12 -0700 Subject: [PATCH 08/66] adding testing for redis --- lemur/common/redis.py | 2 +- lemur/tests/test_redis.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 lemur/tests/test_redis.py diff --git a/lemur/common/redis.py b/lemur/common/redis.py index 0bddf9b4..34a8778f 100644 --- a/lemur/common/redis.py +++ b/lemur/common/redis.py @@ -23,7 +23,7 @@ class RedisHandler: def redis(self, db=0): # The decode_responses flag here directs the client to convert the responses from Redis into Python strings # using the default encoding utf-8. This is client specific. - red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, charset="utf-8", decode_responses=True) + red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True) return red diff --git a/lemur/tests/test_redis.py b/lemur/tests/test_redis.py new file mode 100644 index 00000000..aab2e397 --- /dev/null +++ b/lemur/tests/test_redis.py @@ -0,0 +1,13 @@ +import fakeredis +import time +import sys + + +def test_write_and_read_from_redis(): + function = f"{__name__}.{sys._getframe().f_code.co_name}" + + red = fakeredis.FakeStrictRedis() + key = f"{function}.last_success" + value = int(time.time()) + assert red.set(key, value) is True + assert (int(red.get(key)) == value) is True From 0ed00c5011665b316ff66180a3cbd58dd9f29a8b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 16 Jul 2019 09:01:04 -0700 Subject: [PATCH 09/66] updating test requirement --- requirements-tests.in | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-tests.in b/requirements-tests.in index d315cf7a..5d152fce 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -5,6 +5,7 @@ black coverage factory-boy Faker +fakeredis-1.0.3 freezegun moto nose From 54ecda4e1a129abbe03440da8ab051e7cca679b4 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 16 Jul 2019 09:09:12 -0700 Subject: [PATCH 10/66] updating fakeredis --- requirements-tests.in | 2 +- requirements-tests.txt | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements-tests.in b/requirements-tests.in index 5d152fce..610f26f9 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -5,7 +5,7 @@ black coverage factory-boy Faker -fakeredis-1.0.3 +fakeredis freezegun moto nose diff --git a/requirements-tests.txt b/requirements-tests.txt index 77bc92af..1c4b276e 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -27,6 +27,7 @@ docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose factory-boy==2.12.0 faker==1.0.7 +fakeredis==1.0.3 flask==1.0.3 # via pytest-flask freezegun==0.3.12 future==0.17.1 # via aws-xray-sdk, python-jose @@ -62,13 +63,15 @@ python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto pytz==2019.1 # via moto pyyaml==5.1 +redis==3.2.1 # via fakeredis requests-mock==1.6.0 requests==2.22.0 # via cfn-lint, docker, moto, requests-mock, responses responses==0.10.6 # via moto rsa==4.0 # via python-jose s3transfer==0.2.0 # via boto3 -six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, mock, moto, packaging, pytest, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, fakeredis, freezegun, mock, moto, packaging, pytest, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap2==2.0.5 # via gitdb2 +sortedcontainers==2.1.0 # via fakeredis stevedore==1.30.1 # via bandit text-unidecode==1.2 # via faker toml==0.10.0 # via black @@ -79,3 +82,6 @@ werkzeug==0.15.4 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto zipp==0.5.1 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via cfn-lint From 09c0fa0f940a9d4706f704362b7f3e2ee2692d7d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 16 Jul 2019 17:21:01 -0700 Subject: [PATCH 11/66] updating the function declaration --- lemur/common/celery.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 05e66926..67780957 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -108,7 +108,7 @@ def fetch_acme_cert(id): if celery.current_task: task_id = celery.current_task.request.id - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "Resolving pending certificate {}".format(id), @@ -210,7 +210,7 @@ def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" pending_certs = pending_certificate_service.get_unresolved_pending_certs() - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "Starting job.", @@ -236,7 +236,7 @@ def fetch_all_pending_acme_certs(): @celery.task() def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "Starting job.", @@ -262,7 +262,7 @@ def clean_all_sources(): This function will clean unused certificates from sources. This is a destructive operation and should only be ran periodically. This function triggers one celery task per source. """ - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" sources = validate_sources("all") for source in sources: current_app.logger.debug( @@ -292,7 +292,7 @@ def sync_all_sources(): """ This function will sync certificates from all sources. This function triggers one celery task per source. """ - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" sources = validate_sources("all") for source in sources: current_app.logger.debug( @@ -313,7 +313,7 @@ def sync_source(source): :return: """ - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id @@ -357,7 +357,7 @@ def sync_source_destination(): We rely on account numbers to avoid duplicates. """ current_app.logger.debug("Syncing AWS destinations and sources") - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" for dst in destinations_service.get_all(): if add_aws_destination_to_sources(dst): From e37a7c775ea78730d8f5c68bd82d9fbef9e1aa9b Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Thu, 18 Jul 2019 14:29:54 -0700 Subject: [PATCH 12/66] Initial commit for the UltraDNS plugin to support Lets Encrypt --- lemur/dns_providers/service.py | 1 + lemur/plugins/lemur_acme/plugin.py | 6 +- lemur/plugins/lemur_acme/ultradns.py | 221 +++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 lemur/plugins/lemur_acme/ultradns.py diff --git a/lemur/dns_providers/service.py b/lemur/dns_providers/service.py index ec9fa0de..29f98a5b 100644 --- a/lemur/dns_providers/service.py +++ b/lemur/dns_providers/service.py @@ -98,6 +98,7 @@ def get_types(): ], }, {"name": "dyn"}, + {"name": "ultradns"}, ] }, ) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index c734923a..b0774cbe 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -31,7 +31,7 @@ from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProv from lemur.extensions import metrics, sentry from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin -from lemur.plugins.lemur_acme import cloudflare, dyn, route53 +from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns class AuthorizationRecord(object): @@ -370,7 +370,7 @@ class AcmeHandler(object): pass def get_dns_provider(self, type): - provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53} + provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53, "ultradns": ultradns} provider = provider_types.get(type) if not provider: raise UnknownProvider("No such DNS provider: {}".format(type)) @@ -424,7 +424,7 @@ class ACMEIssuerPlugin(IssuerPlugin): def get_dns_provider(self, type): self.acme = AcmeHandler() - provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53} + provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53, "ultradns": ultradns} provider = provider_types.get(type) if not provider: raise UnknownProvider("No such DNS provider: {}".format(type)) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py new file mode 100644 index 00000000..de65b47f --- /dev/null +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -0,0 +1,221 @@ +import time +import requests +import json + +import dns +import dns.exception +import dns.name +import dns.query +import dns.resolver + +from flask import current_app +from lemur.extensions import metrics, sentry + +use_http = False + + +def get_ultradns_token(): + path = "/v2/authorization/token" + data = { + "grant_type": "password", + "username": current_app.config.get("ACME_ULTRADNS_USERNAME", ""), + "password": current_app.config.get("ACME_ULTRADNS_PASSWORD", ""), + } + base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") + resp = requests.post("{0}{1}".format(base_uri, path), data=data, verify=True) + return resp.json()["access_token"] + + +def _generate_header(): + access_token = get_ultradns_token() + return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"} + + +def _paginate(path, key): + limit = 100 + params = {"offset": 0, "limit": 1} + # params["offset"] = 0 + # params["limit"] = 1 + resp = _get(path, params) + for index in range(0, resp["resultInfo"]["totalCount"], limit): + params["offset"] = index + params["limit"] = limit + resp = _get(path, params) + yield resp[key] + + +def _get(path, params=None): + base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") + resp = requests.get( + "{0}{1}".format(base_uri, path), + headers=_generate_header(), + params=params, + verify=True, + ) + resp.raise_for_status() + return resp.json() + + +def _delete(path): + base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") + resp = requests.delete( + "{0}{1}".format(base_uri, path), + headers=_generate_header(), + verify=True, + ) + resp.raise_for_status() + + +def _post(path, params): + base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") + resp = requests.post( + "{0}{1}".format(base_uri, path), + headers=_generate_header(), + data=json.dumps(params), + verify=True, + ) + resp.raise_for_status() + + +def _has_dns_propagated(name, token): + txt_records = [] + try: + dns_resolver = dns.resolver.Resolver() + dns_resolver.nameservers = [get_authoritative_nameserver(name)] + dns_response = dns_resolver.query(name, "TXT") + for rdata in dns_response: + for txt_record in rdata.strings: + txt_records.append(txt_record.decode("utf-8")) + except dns.exception.DNSException: + metrics.send("has_dns_propagated_fail", "counter", 1) + return False + + for txt_record in txt_records: + if txt_record == token: + metrics.send("has_dns_propagated_success", "counter", 1) + return True + + return False + + +def wait_for_dns_change(change_id, account_number=None): + fqdn, token = change_id + number_of_attempts = 20 + for attempts in range(0, number_of_attempts): + status = _has_dns_propagated(fqdn, token) + current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) + if status: + metrics.send("wait_for_dns_change_success", "counter", 1) + break + time.sleep(10) + if not status: + # TODO: Delete associated DNS text record here + metrics.send("wait_for_dns_change_fail", "counter", 1) + sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)}) + metrics.send( + "wait_for_dns_change_error", + "counter", + 1, + metric_tags={"fqdn": fqdn, "txt_record": token}, + ) + return + + +def get_zones(account_number): + path = "/v2/zones/" + zones = [] + for page in _paginate(path, "zones"): + for elem in page: + zones.append(elem["properties"]["name"][:-1]) + + return zones + + +def get_zone_name(domain, account_number): + zones = get_zones(account_number) + + zone_name = "" + + for z in zones: + if domain.endswith(z): + # Find the most specific zone possible for the domain + # Ex: If fqdn is a.b.c.com, there is a zone for c.com, + # and a zone for b.c.com, we want to use b.c.com. + if z.count(".") > zone_name.count("."): + zone_name = z + if not zone_name: + metrics.send("ultradns_no_zone_name", "counter", 1) + raise Exception("No UltraDNS zone found for domain: {}".format(domain)) + return zone_name + + +def create_txt_record(domain, token, account_number): + zone_name = get_zone_name(domain, account_number) + zone_parts = len(zone_name.split(".")) + node_name = ".".join(domain.split(".")[:-zone_parts]) + fqdn = "{0}.{1}".format(node_name, zone_name) + path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name) + # zone = Zone(zone_name) + params = { + "ttl": 300, + "rdata": [ + "{}".format(token) + ], + } + + try: + _post(path, params) + current_app.logger.debug( + "TXT record created: {0}, token: {1}".format(fqdn, token) + ) + except Exception as e: + current_app.logger.debug( + "Unable to add record. Domain: {}. Token: {}. " + "Record already exists: {}".format(domain, token, e), + exc_info=True, + ) + + change_id = (fqdn, token) + return change_id + + +def delete_txt_record(change_id, account_number, domain, token): + # client = get_ultradns_client() + if not domain: + current_app.logger.debug("delete_txt_record: No domain passed") + return + + zone_name = get_zone_name(domain, account_number) + zone_parts = len(zone_name.split(".")) + node_name = ".".join(domain.split(".")[:-zone_parts]) + fqdn = "{0}.{1}".format(node_name, zone_name) + path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name) + + try: + # rrsets = client.get_rrsets_by_type_owner(zone_name, "TXT", node_name) + rrsets = _get(path) + except Exception as e: + metrics.send("delete_txt_record_geterror", "counter", 1) + # No Text Records remain or host is not in the zone anymore because all records have been deleted. + return + try: + rrsets["rrSets"][0]["rdata"].remove("{}".format(token)) + except ValueError: + current_app.logger.debug("Token not found") + return + + #client.delete_rrset(zone_name, "TXT", node_name) + _delete(path) + + if len(rrsets["rrSets"][0]["rdata"]) > 0: + #client.create_rrset(zone_name, "TXT", node_name, 300, rrsets["rrSets"][0]["rdata"]) + params = { + "ttl": 300, + "rdata": rrsets["rrSets"][0]["rdata"], + } + _post(path, params) + + +def get_authoritative_nameserver(domain): + # return "8.8.8.8" + return "156.154.64.154" From 36ebba64916e87391bf5f1ad73f51b88bed799b1 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 18 Jul 2019 15:16:01 -0700 Subject: [PATCH 13/66] source is not dict --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 67780957..b701a9eb 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -343,7 +343,7 @@ def sync_source(source): log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) - metrics.send(f"{function}.success", 'counter', 1, metric_tags=source) + metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) red.set(f'{function}.last_success', int(time.time())) From 0b52aa8c59984f0e9579a2b5285e4d276a0bc53c Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 22 Jul 2019 11:47:48 -0700 Subject: [PATCH 14/66] Added Zone class to handle ultradns zones --- lemur/plugins/lemur_acme/ultradns.py | 96 +++++++++++++++++++---- lemur/plugins/lemur_acme/ultradns_zone.py | 33 ++++++++ 2 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 lemur/plugins/lemur_acme/ultradns_zone.py diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index de65b47f..eb595789 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -1,6 +1,7 @@ import time import requests import json +from .ultradns_zone import Zone import dns import dns.exception @@ -11,10 +12,11 @@ import dns.resolver from flask import current_app from lemur.extensions import metrics, sentry -use_http = False - def get_ultradns_token(): + # Function to call the UltraDNS Authorization API. Returns the Authorization access_token + # which is valid for 1 hour. Each request calls this function and we generate a new token + # every time. path = "/v2/authorization/token" data = { "grant_type": "password", @@ -27,6 +29,8 @@ def get_ultradns_token(): def _generate_header(): + # Function to generate the header for a request. Contains the Authorization access_key + # obtained from the get_ultradns_token() function. access_token = get_ultradns_token() return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"} @@ -34,8 +38,6 @@ def _generate_header(): def _paginate(path, key): limit = 100 params = {"offset": 0, "limit": 1} - # params["offset"] = 0 - # params["limit"] = 1 resp = _get(path, params) for index in range(0, resp["resultInfo"]["totalCount"], limit): params["offset"] = index @@ -45,6 +47,7 @@ def _paginate(path, key): def _get(path, params=None): + # Function to execute a GET request on the given URL (base_uri + path) with given params base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.get( "{0}{1}".format(base_uri, path), @@ -57,6 +60,7 @@ def _get(path, params=None): def _delete(path): + # Function to execute a DELETE request on the given URL base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.delete( "{0}{1}".format(base_uri, path), @@ -67,6 +71,7 @@ def _delete(path): def _post(path, params): + # Executes a POST request on given URL. Body is sent in JSON format base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.post( "{0}{1}".format(base_uri, path), @@ -78,6 +83,8 @@ def _post(path, params): def _has_dns_propagated(name, token): + # Check whether the DNS change made by Lemur have propagated to the public DNS or not. + # Invoked by wait_for_dns_change() function txt_records = [] try: dns_resolver = dns.resolver.Resolver() @@ -99,6 +106,7 @@ def _has_dns_propagated(name, token): def wait_for_dns_change(change_id, account_number=None): + # Waits and checks if the DNS changes have propagated or not. fqdn, token = change_id number_of_attempts = 20 for attempts in range(0, number_of_attempts): @@ -122,20 +130,26 @@ def wait_for_dns_change(change_id, account_number=None): def get_zones(account_number): + # Get zones from the UltraDNS path = "/v2/zones/" zones = [] for page in _paginate(path, "zones"): for elem in page: - zones.append(elem["properties"]["name"][:-1]) + # UltraDNS zone names end with a "." - Example - lemur.example.com. + # We pick out the names minus the "." at the end while returning the list + zone = Zone(elem) + # TODO : Check for active & Primary + # if elem["properties"]["type"] == "PRIMARY" and elem["properties"]["status"] == "ACTIVE": + if zone.authoritative_type == "PRIMARY" and zone.status == "ACTIVE": + zones.append(zone.name) return zones def get_zone_name(domain, account_number): + # Get the matching zone for the given domain zones = get_zones(account_number) - zone_name = "" - for z in zones: if domain.endswith(z): # Find the most specific zone possible for the domain @@ -150,12 +164,20 @@ def get_zone_name(domain, account_number): def create_txt_record(domain, token, account_number): + # Create a TXT record for the given domain. + # The part of the domain that matches with the zone becomes the zone name. + # The remainder becomes the owner name (referred to as node name here) + # Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we + # get a request to create a cert for lemur.example.com + # Domain - _acme-challenge.lemur.example.com + # Matching zone - example.com + # Owner name - _acme-challenge.lemur + zone_name = get_zone_name(domain, account_number) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) fqdn = "{0}.{1}".format(node_name, zone_name) path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name) - # zone = Zone(zone_name) params = { "ttl": 300, "rdata": [ @@ -180,7 +202,16 @@ def create_txt_record(domain, token, account_number): def delete_txt_record(change_id, account_number, domain, token): - # client = get_ultradns_client() + # Delete the TXT record that was created in the create_txt_record() function. + # UltraDNS handles records differently compared to Dyn. It creates an RRSet + # which is a set of records of the same type and owner. This means + # that while deleting the record, we cannot delete any individual record from + # the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are + # being created for the same domain at the same time, the challenge TXT records + # that are created will be added under the same RRSet. If the RRSet had more + # than 1 record, then we create a new RRSet on UltraDNS minus the record that + # has to be deleted. + if not domain: current_app.logger.debug("delete_txt_record: No domain passed") return @@ -188,27 +219,26 @@ def delete_txt_record(change_id, account_number, domain, token): zone_name = get_zone_name(domain, account_number) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = "{0}.{1}".format(node_name, zone_name) path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name) try: - # rrsets = client.get_rrsets_by_type_owner(zone_name, "TXT", node_name) rrsets = _get(path) except Exception as e: metrics.send("delete_txt_record_geterror", "counter", 1) # No Text Records remain or host is not in the zone anymore because all records have been deleted. return try: + # Remove the record from the RRSet locally rrsets["rrSets"][0]["rdata"].remove("{}".format(token)) except ValueError: current_app.logger.debug("Token not found") return - #client.delete_rrset(zone_name, "TXT", node_name) + # Delete the RRSet from UltraDNS _delete(path) + # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS if len(rrsets["rrSets"][0]["rdata"]) > 0: - #client.create_rrset(zone_name, "TXT", node_name, 300, rrsets["rrSets"][0]["rdata"]) params = { "ttl": 300, "rdata": rrsets["rrSets"][0]["rdata"], @@ -216,6 +246,42 @@ def delete_txt_record(change_id, account_number, domain, token): _post(path, params) +def delete_acme_txt_records(domain): + + if not domain: + current_app.logger.debug("delete_acme_txt_records: No domain passed") + return + acme_challenge_string = "_acme-challenge" + if not domain.startswith(acme_challenge_string): + current_app.logger.debug( + "delete_acme_txt_records: Domain {} doesn't start with string {}. " + "Cowardly refusing to delete TXT records".format( + domain, acme_challenge_string + ) + ) + return + + zone_name = get_zone_name(domain) + zone_parts = len(zone_name.split(".")) + node_name = ".".join(domain.split(".")[:-zone_parts]) + path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name) + + _delete(path) + + def get_authoritative_nameserver(domain): - # return "8.8.8.8" - return "156.154.64.154" + """ + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + REMEMBER TO CHANGE THE RETURN VALUE + """ + return "8.8.8.8" + # return "156.154.64.154" diff --git a/lemur/plugins/lemur_acme/ultradns_zone.py b/lemur/plugins/lemur_acme/ultradns_zone.py new file mode 100644 index 00000000..c6d90422 --- /dev/null +++ b/lemur/plugins/lemur_acme/ultradns_zone.py @@ -0,0 +1,33 @@ +class Zone: + """ + This class implements an Ultra DNS zone. + """ + + def __init__(self, _data, _client="Client"): + self._data = _data + self._client = _client + + @property + def name(self): + """ + Zone name, has a trailing "." at the end, which we manually remove. + """ + return self._data["properties"]["name"][:-1] + + @property + def authoritative_type(self): + """ + Indicates whether the zone is setup as a PRIMARY or SECONDARY + """ + return self._data["properties"]["type"] + + @property + def record_count(self): + return self._data["properties"]["resourceRecordCount"] + + @property + def status(self): + """ + Returns the status of the zone - ACTIVE, SUSPENDED, etc + """ + return self._data["properties"]["status"] From 51f3b7dde0ff14eebc4d06d8eb09d6fcccd53a2d Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 22 Jul 2019 14:23:40 -0700 Subject: [PATCH 15/66] Added the Record class for UltraDNS --- lemur/plugins/lemur_acme/ultradns.py | 10 +++++--- lemur/plugins/lemur_acme/ultradns_record.py | 26 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 lemur/plugins/lemur_acme/ultradns_record.py diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index eb595789..95adc77a 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -2,6 +2,7 @@ import time import requests import json from .ultradns_zone import Zone +from .ultradns_record import Record import dns import dns.exception @@ -223,13 +224,15 @@ def delete_txt_record(change_id, account_number, domain, token): try: rrsets = _get(path) + record = Record(rrsets) except Exception as e: metrics.send("delete_txt_record_geterror", "counter", 1) # No Text Records remain or host is not in the zone anymore because all records have been deleted. return try: # Remove the record from the RRSet locally - rrsets["rrSets"][0]["rdata"].remove("{}".format(token)) + # rrsets["rrSets"][0]["rdata"].remove("{}".format(token)) + record.rdata.remove("{}".format(token)) except ValueError: current_app.logger.debug("Token not found") return @@ -238,10 +241,11 @@ def delete_txt_record(change_id, account_number, domain, token): _delete(path) # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS - if len(rrsets["rrSets"][0]["rdata"]) > 0: + # if len(rrsets["rrSets"][0]["rdata"]) > 0: + if len(record.rdata) > 0: params = { "ttl": 300, - "rdata": rrsets["rrSets"][0]["rdata"], + "rdata": record.rdata, } _post(path, params) diff --git a/lemur/plugins/lemur_acme/ultradns_record.py b/lemur/plugins/lemur_acme/ultradns_record.py new file mode 100644 index 00000000..9ec8d4d8 --- /dev/null +++ b/lemur/plugins/lemur_acme/ultradns_record.py @@ -0,0 +1,26 @@ +class Record: + """ + This class implements an Ultra DNS record. + Accepts the response from the API call as the argument. + """ + + def __init__(self, _data): + # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response. + # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response. + self._data = _data["rrSets"][0] + + @property + def name(self): + return self._data["ownerName"] + + @property + def rrtype(self): + return self._data["rrtype"] + + @property + def rdata(self): + return self._data["rdata"] + + @property + def ttl(self): + return self._data["ttl"] From 252410c6e9529a10926a0f6b23768a1f322c163e Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 22 Jul 2019 16:00:20 -0700 Subject: [PATCH 16/66] Updated TTL from 300 to 5 --- lemur/plugins/lemur_acme/ultradns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index 95adc77a..d6ad64af 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -180,7 +180,7 @@ def create_txt_record(domain, token, account_number): fqdn = "{0}.{1}".format(node_name, zone_name) path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name) params = { - "ttl": 300, + "ttl": 5, "rdata": [ "{}".format(token) ], @@ -244,7 +244,7 @@ def delete_txt_record(change_id, account_number, domain, token): # if len(rrsets["rrSets"][0]["rdata"]) > 0: if len(record.rdata) > 0: params = { - "ttl": 300, + "ttl": 5, "rdata": record.rdata, } _post(path, params) From 429e6a967c4133e2da0b0e576abb9537b810f33d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 25 Jul 2019 18:49:19 -0700 Subject: [PATCH 17/66] better error handling for redis --- lemur/common/redis.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lemur/common/redis.py b/lemur/common/redis.py index 34a8778f..ca15734f 100644 --- a/lemur/common/redis.py +++ b/lemur/common/redis.py @@ -3,7 +3,9 @@ Helper Class for Redis """ import redis +import sys from flask import current_app +from lemur.extensions import sentry from lemur.factory import create_app if current_app: @@ -23,7 +25,19 @@ class RedisHandler: def redis(self, db=0): # The decode_responses flag here directs the client to convert the responses from Redis into Python strings # using the default encoding utf-8. This is client specific. - red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True) + function = f"{__name__}.{sys._getframe().f_code.co_name}" + try: + red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True) + red.set("test", 0) + except redis.ConnectionError: + log_data = { + "function": function, + "message": "Redis Connection error", + "host": self.host, + "port": self.port + } + current_app.logger.error(log_data) + sentry.captureException() return red From adabe18c905cb4a9fe4a44f3a9e710e07cdf63d4 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 25 Jul 2019 18:56:28 -0700 Subject: [PATCH 18/66] metric tags, to be able to track which domains where failing during the LetsEncrypt domain validation --- lemur/plugins/lemur_acme/dyn.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 00a48eb6..fff2e632 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -33,22 +33,22 @@ def get_dynect_session(): return dynect_session -def _has_dns_propagated(name, token): +def _has_dns_propagated(fqdn, token): txt_records = [] try: dns_resolver = dns.resolver.Resolver() - dns_resolver.nameservers = [get_authoritative_nameserver(name)] - dns_response = dns_resolver.query(name, "TXT") + dns_resolver.nameservers = [get_authoritative_nameserver(fqdn)] + dns_response = dns_resolver.query(fqdn, "TXT") for rdata in dns_response: for txt_record in rdata.strings: txt_records.append(txt_record.decode("utf-8")) except dns.exception.DNSException: - metrics.send("has_dns_propagated_fail", "counter", 1) + metrics.send("has_dns_propagated_fail", "counter", 1, metric_tags={"dns": fqdn}) return False for txt_record in txt_records: if txt_record == token: - metrics.send("has_dns_propagated_success", "counter", 1) + metrics.send("has_dns_propagated_success", "counter", 1, metric_tags={"dns": fqdn}) return True return False @@ -61,12 +61,12 @@ def wait_for_dns_change(change_id, account_number=None): status = _has_dns_propagated(fqdn, token) current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) if status: - metrics.send("wait_for_dns_change_success", "counter", 1) + metrics.send("wait_for_dns_change_success", "counter", 1, metric_tags={"dns": fqdn}) break time.sleep(10) if not status: # TODO: Delete associated DNS text record here - metrics.send("wait_for_dns_change_fail", "counter", 1) + metrics.send("wait_for_dns_change_fail", "counter", 1, metric_tags={"dns": fqdn}) sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)}) metrics.send( "wait_for_dns_change_error", From e993194b4f30262a86880a86097d968f7cf9fc2e Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 29 Jul 2019 14:59:28 -0700 Subject: [PATCH 19/66] Check ultraDNS authoritative server first. Upon success, check Googles DNS server. --- lemur/plugins/lemur_acme/ultradns.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index d6ad64af..1a520f2e 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -83,13 +83,15 @@ def _post(path, params): resp.raise_for_status() -def _has_dns_propagated(name, token): +def _has_dns_propagated(name, token, domain="8.8.8.8"): # Check whether the DNS change made by Lemur have propagated to the public DNS or not. # Invoked by wait_for_dns_change() function txt_records = [] try: dns_resolver = dns.resolver.Resolver() - dns_resolver.nameservers = [get_authoritative_nameserver(name)] + # dns_resolver.nameservers = [get_authoritative_nameserver(name)] + # dns_resolver.nameservers = ["156.154.64.154"] + dns_resolver.nameservers = [domain] dns_response = dns_resolver.query(name, "TXT") for rdata in dns_response: for txt_record in rdata.strings: @@ -111,12 +113,21 @@ def wait_for_dns_change(change_id, account_number=None): fqdn, token = change_id number_of_attempts = 20 for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token) + status = _has_dns_propagated(fqdn, token, "156.154.64.154") current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) if status: - metrics.send("wait_for_dns_change_success", "counter", 1) + # metrics.send("wait_for_dns_change_success", "counter", 1) + time.sleep(10) break time.sleep(10) + if status: + for attempts in range(0, number_of_attempts): + status = _has_dns_propagated(fqdn, token, "8.8.8.8") + current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) + if status: + metrics.send("wait_for_dns_change_success", "counter", 1) + break + time.sleep(10) if not status: # TODO: Delete associated DNS text record here metrics.send("wait_for_dns_change_fail", "counter", 1) @@ -132,7 +143,7 @@ def wait_for_dns_change(change_id, account_number=None): def get_zones(account_number): # Get zones from the UltraDNS - path = "/v2/zones/" + path = "/v2/zones" zones = [] for page in _paginate(path, "zones"): for elem in page: @@ -287,5 +298,5 @@ def get_authoritative_nameserver(domain): REMEMBER TO CHANGE THE RETURN VALUE REMEMBER TO CHANGE THE RETURN VALUE """ - return "8.8.8.8" - # return "156.154.64.154" + # return "8.8.8.8" + return "156.154.64.154" From 3ad791e1ec634646edc78aaa1b4e4d40bbb67936 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 29 Jul 2019 18:01:28 -0700 Subject: [PATCH 20/66] Dynamically obtain the authoritative nameserver for the domain --- lemur/plugins/lemur_acme/ultradns.py | 72 ++++++++++++++++++---------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index 1a520f2e..c7f853d0 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -83,14 +83,12 @@ def _post(path, params): resp.raise_for_status() -def _has_dns_propagated(name, token, domain="8.8.8.8"): +def _has_dns_propagated(name, token, domain): # Check whether the DNS change made by Lemur have propagated to the public DNS or not. # Invoked by wait_for_dns_change() function txt_records = [] try: dns_resolver = dns.resolver.Resolver() - # dns_resolver.nameservers = [get_authoritative_nameserver(name)] - # dns_resolver.nameservers = ["156.154.64.154"] dns_resolver.nameservers = [domain] dns_response = dns_resolver.query(name, "TXT") for rdata in dns_response: @@ -110,19 +108,21 @@ def _has_dns_propagated(name, token, domain="8.8.8.8"): def wait_for_dns_change(change_id, account_number=None): # Waits and checks if the DNS changes have propagated or not. + # First check the domains authoritative server. Once this succeeds, + # we ask a public DNS server (Google <8.8.8.8> in our case). fqdn, token = change_id number_of_attempts = 20 + nameserver = get_authoritative_nameserver(fqdn) for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token, "156.154.64.154") + status = _has_dns_propagated(fqdn, token, nameserver) current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) if status: - # metrics.send("wait_for_dns_change_success", "counter", 1) time.sleep(10) break time.sleep(10) if status: for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token, "8.8.8.8") + status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver()) current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) if status: metrics.send("wait_for_dns_change_success", "counter", 1) @@ -150,8 +150,6 @@ def get_zones(account_number): # UltraDNS zone names end with a "." - Example - lemur.example.com. # We pick out the names minus the "." at the end while returning the list zone = Zone(elem) - # TODO : Check for active & Primary - # if elem["properties"]["type"] == "PRIMARY" and elem["properties"]["status"] == "ACTIVE": if zone.authoritative_type == "PRIMARY" and zone.status == "ACTIVE": zones.append(zone.name) @@ -242,7 +240,6 @@ def delete_txt_record(change_id, account_number, domain, token): return try: # Remove the record from the RRSet locally - # rrsets["rrSets"][0]["rdata"].remove("{}".format(token)) record.rdata.remove("{}".format(token)) except ValueError: current_app.logger.debug("Token not found") @@ -252,7 +249,6 @@ def delete_txt_record(change_id, account_number, domain, token): _delete(path) # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS - # if len(rrsets["rrSets"][0]["rdata"]) > 0: if len(record.rdata) > 0: params = { "ttl": 5, @@ -285,18 +281,44 @@ def delete_acme_txt_records(domain): def get_authoritative_nameserver(domain): - """ - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - REMEMBER TO CHANGE THE RETURN VALUE - """ - # return "8.8.8.8" - return "156.154.64.154" + n = dns.name.from_text(domain) + + depth = 2 + default = dns.resolver.get_default_resolver() + nameserver = default.nameservers[0] + + last = False + while not last: + s = n.split(depth) + + last = s[0].to_unicode() == u"@" + sub = s[1] + + query = dns.message.make_query(sub, dns.rdatatype.NS) + response = dns.query.udp(query, nameserver) + + rcode = response.rcode() + if rcode != dns.rcode.NOERROR: + metrics.send("get_authoritative_nameserver_error", "counter", 1) + if rcode == dns.rcode.NXDOMAIN: + raise Exception("%s does not exist." % sub) + else: + raise Exception("Error %s" % dns.rcode.to_text(rcode)) + + if len(response.authority) > 0: + rrset = response.authority[0] + else: + rrset = response.answer[0] + + rr = rrset[0] + if rr.rdtype != dns.rdatatype.SOA: + authority = rr.target + nameserver = default.query(authority).rrset[0].to_text() + + depth += 1 + + return nameserver + + +def get_public_authoritative_nameserver(): + return "8.8.8.8" From a89cbe933224575fd8c3354b644f60750b2894b2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 30 Jul 2019 09:57:15 -0700 Subject: [PATCH 21/66] moving all cron jobs to become celery jobs --- lemur/common/celery.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b701a9eb..b889a80a 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -26,6 +26,11 @@ from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources from lemur.sources.service import add_aws_destination_to_sources +from lemur.certificates import cli as cli_certificate +from lemur.dns_providers import cli as cli_dns_providers +from lemur.notifications import cli as cli_notification +from lemur.endpoints import cli as cli_endpoints + if current_app: flask_app = current_app @@ -366,3 +371,83 @@ def sync_source_destination(): current_app.logger.debug("Completed Syncing AWS destinations and sources") red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) + + +@celery.task() +def certificate_reissue(): + """ + This celery task reissues certificates which are pending reissue + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_app.logger.debug(f"{function}: reissuing certificates") + cli_certificate.reissue(None, True) + current_app.logger.debug(f"{function}: reissuance completed") + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + + +@celery.task() +def certificate_rotate(): + """ + This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_app.logger.debug(f"{function}: rotating certificates") + cli_certificate.rotate(None, None, None, None, True) + current_app.logger.debug(f"{function}: rotation completed") + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + + +@celery.task() +def endpoints_expire(): + """ + This celery task removes all endpoints that have not been recently updated + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_app.logger.debug(f"{function}: endpoints expire") + cli_endpoints.expire(2) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + + +@celery.task() +def get_all_zones(): + """ + This celery syncs all zones from the available dns providers + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_app.logger.debug(f"{function}: get_all_zones") + cli_dns_providers.get_all_zones() + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + + +@celery.task() +def check_revoked(): + """ + This celery task attempts to check if any certs are expired + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_app.logger.debug(f"{function}: check if any certificates are revoked revoked") + cli_certificate.check_revoked() + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + + +@celery.task() +def notify_expirations(): + """ + This celery task notifies about expiring certs + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_app.logger.debug(f"{function}: Cert Expiration Notifcation") + cli_notification.expirations(["MetatronUserCertfor", "Metatron-User-Cert-for"]) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) From 244aa069f04f9b6787bfc41162ff370a9110fbfb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 30 Jul 2019 10:32:09 -0700 Subject: [PATCH 22/66] lemur is one level deep than makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1ca94e42..069eb29b 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ reset-db: @echo "--> Enabling pg_trgm extension" psql lemur -c "create extension IF NOT EXISTS pg_trgm;" @echo "--> Applying migrations" - lemur db upgrade + cd lemur && lemur db upgrade setup-git: @echo "--> Installing git hooks" From 3d48b422b5ee3a23e6df83b5ca9208ffebb4621f Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Tue, 30 Jul 2019 11:39:35 -0700 Subject: [PATCH 23/66] Removed TODO --- lemur/plugins/lemur_acme/ultradns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index c7f853d0..c43840e4 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -129,7 +129,6 @@ def wait_for_dns_change(change_id, account_number=None): break time.sleep(10) if not status: - # TODO: Delete associated DNS text record here metrics.send("wait_for_dns_change_fail", "counter", 1) sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)}) metrics.send( From 44bc562e8b2e2339d420af0a42647fa4efe91e0c Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Tue, 30 Jul 2019 13:08:16 -0700 Subject: [PATCH 24/66] Update ultradns.py Minor logging changes in wait_for_dns_change --- lemur/plugins/lemur_acme/ultradns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index c43840e4..24b98e66 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -115,7 +115,7 @@ def wait_for_dns_change(change_id, account_number=None): nameserver = get_authoritative_nameserver(fqdn) for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token, nameserver) - current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) + current_app.logger.debug("Record status on ultraDNS authoritative server for fqdn: {}: {}".format(fqdn, status)) if status: time.sleep(10) break @@ -123,7 +123,7 @@ def wait_for_dns_change(change_id, account_number=None): if status: for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver()) - current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) + current_app.logger.debug("Record status on Google DNS for fqdn: {}: {}".format(fqdn, status)) if status: metrics.send("wait_for_dns_change_success", "counter", 1) break From 3ba7fdbd494401331e9da084c1e8b4f0b147e559 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 31 Jul 2019 11:11:39 -0700 Subject: [PATCH 25/66] Updated logger to log a dictionary instead of a string --- lemur/plugins/lemur_acme/ultradns.py | 80 +++++++++++++++++++++------- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index 24b98e66..d3b68afc 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -1,6 +1,7 @@ import time import requests import json +import sys from .ultradns_zone import Zone from .ultradns_record import Record @@ -115,7 +116,14 @@ def wait_for_dns_change(change_id, account_number=None): nameserver = get_authoritative_nameserver(fqdn) for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token, nameserver) - current_app.logger.debug("Record status on ultraDNS authoritative server for fqdn: {}: {}".format(fqdn, status)) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": fqdn, + "status": status, + "message": "Record status on ultraDNS authoritative server" + } + current_app.logger.debug(log_data) if status: time.sleep(10) break @@ -123,7 +131,14 @@ def wait_for_dns_change(change_id, account_number=None): if status: for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver()) - current_app.logger.debug("Record status on Google DNS for fqdn: {}: {}".format(fqdn, status)) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": fqdn, + "status": status, + "message": "Record status on Public DNS" + } + current_app.logger.debug(log_data) if status: metrics.send("wait_for_dns_change_success", "counter", 1) break @@ -196,15 +211,24 @@ def create_txt_record(domain, token, account_number): try: _post(path, params) - current_app.logger.debug( - "TXT record created: {0}, token: {1}".format(fqdn, token) - ) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": fqdn, + "token": token, + "message": "TXT record created" + } + current_app.logger.debug(log_data) except Exception as e: - current_app.logger.debug( - "Unable to add record. Domain: {}. Token: {}. " - "Record already exists: {}".format(domain, token, e), - exc_info=True, - ) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "domain": domain, + "token": token, + "Exception": e, + "message": "Unable to add record. Record already exists." + } + current_app.logger.debug(log_data) change_id = (fqdn, token) return change_id @@ -222,7 +246,12 @@ def delete_txt_record(change_id, account_number, domain, token): # has to be deleted. if not domain: - current_app.logger.debug("delete_txt_record: No domain passed") + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "message": "No domain passed" + } + current_app.logger.debug(log_data) return zone_name = get_zone_name(domain, account_number) @@ -241,7 +270,13 @@ def delete_txt_record(change_id, account_number, domain, token): # Remove the record from the RRSet locally record.rdata.remove("{}".format(token)) except ValueError: - current_app.logger.debug("Token not found") + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "token": token, + "message": "Token not found" + } + current_app.logger.debug(log_data) return # Delete the RRSet from UltraDNS @@ -259,16 +294,23 @@ def delete_txt_record(change_id, account_number, domain, token): def delete_acme_txt_records(domain): if not domain: - current_app.logger.debug("delete_acme_txt_records: No domain passed") + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "message": "No domain passed" + } + current_app.logger.debug(log_data) return acme_challenge_string = "_acme-challenge" if not domain.startswith(acme_challenge_string): - current_app.logger.debug( - "delete_acme_txt_records: Domain {} doesn't start with string {}. " - "Cowardly refusing to delete TXT records".format( - domain, acme_challenge_string - ) - ) + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "domain": domain, + "acme_challenge_string": acme_challenge_string, + "message": "Domain does not start with the acme challenge string" + } + current_app.logger.debug(log_data) return zone_name = get_zone_name(domain) From 11cd09513154a2d505755ccc266d33b2e6d4127a Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 31 Jul 2019 11:12:28 -0700 Subject: [PATCH 26/66] Reduced the number of calls to get_public_authoritative_nameserver by using a variable --- lemur/plugins/lemur_acme/ultradns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index d3b68afc..d516be1a 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -129,8 +129,9 @@ def wait_for_dns_change(change_id, account_number=None): break time.sleep(10) if status: + nameserver = get_public_authoritative_nameserver() for attempts in range(0, number_of_attempts): - status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver()) + status = _has_dns_propagated(fqdn, token, nameserver) function = sys._getframe().f_code.co_name log_data = { "function": function, From 503df999fa1d569475898290f50bc9ce3d1ac2ba Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 31 Jul 2019 11:32:04 -0700 Subject: [PATCH 27/66] Updated metrics.send to send function named, followed by status, separated by a period --- lemur/plugins/lemur_acme/ultradns.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index d516be1a..0229a59e 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -96,12 +96,14 @@ def _has_dns_propagated(name, token, domain): for txt_record in rdata.strings: txt_records.append(txt_record.decode("utf-8")) except dns.exception.DNSException: - metrics.send("has_dns_propagated_fail", "counter", 1) + function = sys._getframe().f_code.co_name + metrics.send("{}.fail".format(function), "counter", 1) return False for txt_record in txt_records: if txt_record == token: - metrics.send("has_dns_propagated_success", "counter", 1) + function = sys._getframe().f_code.co_name + metrics.send("{}.success".format(function), "counter", 1) return True return False @@ -132,7 +134,6 @@ def wait_for_dns_change(change_id, account_number=None): nameserver = get_public_authoritative_nameserver() for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token, nameserver) - function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": fqdn, @@ -141,18 +142,12 @@ def wait_for_dns_change(change_id, account_number=None): } current_app.logger.debug(log_data) if status: - metrics.send("wait_for_dns_change_success", "counter", 1) + metrics.send("{}.success".format(function), "counter", 1) break time.sleep(10) if not status: - metrics.send("wait_for_dns_change_fail", "counter", 1) + metrics.send("{}.fail".format, "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token}) sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)}) - metrics.send( - "wait_for_dns_change_error", - "counter", - 1, - metric_tags={"fqdn": fqdn, "txt_record": token}, - ) return @@ -183,7 +178,8 @@ def get_zone_name(domain, account_number): if z.count(".") > zone_name.count("."): zone_name = z if not zone_name: - metrics.send("ultradns_no_zone_name", "counter", 1) + function = sys._getframe().f_code.co_name + metrics.send("{}.fail".format(function), "counter", 1) raise Exception("No UltraDNS zone found for domain: {}".format(domain)) return zone_name @@ -264,7 +260,8 @@ def delete_txt_record(change_id, account_number, domain, token): rrsets = _get(path) record = Record(rrsets) except Exception as e: - metrics.send("delete_txt_record_geterror", "counter", 1) + function = sys._getframe().f_code.co_name + metrics.send("{}.geterror".format(function), "counter", 1) # No Text Records remain or host is not in the zone anymore because all records have been deleted. return try: @@ -341,7 +338,8 @@ def get_authoritative_nameserver(domain): rcode = response.rcode() if rcode != dns.rcode.NOERROR: - metrics.send("get_authoritative_nameserver_error", "counter", 1) + function = sys._getframe().f_code.co_name + metrics.send("{}.error".format(function), "counter", 1) if rcode == dns.rcode.NXDOMAIN: raise Exception("%s does not exist." % sub) else: From fe075dc9f56985d2b81a75d7b4db12d87e3f6023 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 31 Jul 2019 12:00:31 -0700 Subject: [PATCH 28/66] Changed function comments to doc strings. --- lemur/plugins/lemur_acme/ultradns.py | 83 +++++++++++++++++----------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index 0229a59e..1c3aa961 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -16,9 +16,12 @@ from lemur.extensions import metrics, sentry def get_ultradns_token(): - # Function to call the UltraDNS Authorization API. Returns the Authorization access_token - # which is valid for 1 hour. Each request calls this function and we generate a new token - # every time. + """ + Function to call the UltraDNS Authorization API. + + Returns the Authorization access_token which is valid for 1 hour. + Each request calls this function and we generate a new token every time. + """ path = "/v2/authorization/token" data = { "grant_type": "password", @@ -31,8 +34,11 @@ def get_ultradns_token(): def _generate_header(): - # Function to generate the header for a request. Contains the Authorization access_key - # obtained from the get_ultradns_token() function. + """ + Function to generate the header for a request. + + Contains the Authorization access_key obtained from the get_ultradns_token() function. + """ access_token = get_ultradns_token() return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"} @@ -49,7 +55,7 @@ def _paginate(path, key): def _get(path, params=None): - # Function to execute a GET request on the given URL (base_uri + path) with given params + """Function to execute a GET request on the given URL (base_uri + path) with given params""" base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.get( "{0}{1}".format(base_uri, path), @@ -62,7 +68,7 @@ def _get(path, params=None): def _delete(path): - # Function to execute a DELETE request on the given URL + """Function to execute a DELETE request on the given URL""" base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.delete( "{0}{1}".format(base_uri, path), @@ -73,7 +79,7 @@ def _delete(path): def _post(path, params): - # Executes a POST request on given URL. Body is sent in JSON format + """Executes a POST request on given URL. Body is sent in JSON format""" base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.post( "{0}{1}".format(base_uri, path), @@ -85,8 +91,11 @@ def _post(path, params): def _has_dns_propagated(name, token, domain): - # Check whether the DNS change made by Lemur have propagated to the public DNS or not. - # Invoked by wait_for_dns_change() function + """ + Check whether the DNS change made by Lemur have propagated to the public DNS or not. + + Invoked by wait_for_dns_change() function + """ txt_records = [] try: dns_resolver = dns.resolver.Resolver() @@ -110,9 +119,12 @@ def _has_dns_propagated(name, token, domain): def wait_for_dns_change(change_id, account_number=None): - # Waits and checks if the DNS changes have propagated or not. - # First check the domains authoritative server. Once this succeeds, - # we ask a public DNS server (Google <8.8.8.8> in our case). + """ + Waits and checks if the DNS changes have propagated or not. + + First check the domains authoritative server. Once this succeeds, + we ask a public DNS server (Google <8.8.8.8> in our case). + """ fqdn, token = change_id number_of_attempts = 20 nameserver = get_authoritative_nameserver(fqdn) @@ -152,7 +164,7 @@ def wait_for_dns_change(change_id, account_number=None): def get_zones(account_number): - # Get zones from the UltraDNS + """Get zones from the UltraDNS""" path = "/v2/zones" zones = [] for page in _paginate(path, "zones"): @@ -167,7 +179,7 @@ def get_zones(account_number): def get_zone_name(domain, account_number): - # Get the matching zone for the given domain + """Get the matching zone for the given domain""" zones = get_zones(account_number) zone_name = "" for z in zones: @@ -185,14 +197,17 @@ def get_zone_name(domain, account_number): def create_txt_record(domain, token, account_number): - # Create a TXT record for the given domain. - # The part of the domain that matches with the zone becomes the zone name. - # The remainder becomes the owner name (referred to as node name here) - # Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we - # get a request to create a cert for lemur.example.com - # Domain - _acme-challenge.lemur.example.com - # Matching zone - example.com - # Owner name - _acme-challenge.lemur + """ + Create a TXT record for the given domain. + + The part of the domain that matches with the zone becomes the zone name. + The remainder becomes the owner name (referred to as node name here) + Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we + get a request to create a cert for lemur.example.com + Domain - _acme-challenge.lemur.example.com + Matching zone - example.com + Owner name - _acme-challenge.lemur + """ zone_name = get_zone_name(domain, account_number) zone_parts = len(zone_name.split(".")) @@ -232,15 +247,18 @@ def create_txt_record(domain, token, account_number): def delete_txt_record(change_id, account_number, domain, token): - # Delete the TXT record that was created in the create_txt_record() function. - # UltraDNS handles records differently compared to Dyn. It creates an RRSet - # which is a set of records of the same type and owner. This means - # that while deleting the record, we cannot delete any individual record from - # the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are - # being created for the same domain at the same time, the challenge TXT records - # that are created will be added under the same RRSet. If the RRSet had more - # than 1 record, then we create a new RRSet on UltraDNS minus the record that - # has to be deleted. + """ + Delete the TXT record that was created in the create_txt_record() function. + + UltraDNS handles records differently compared to Dyn. It creates an RRSet + which is a set of records of the same type and owner. This means + that while deleting the record, we cannot delete any individual record from + the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are + being created for the same domain at the same time, the challenge TXT records + that are created will be added under the same RRSet. If the RRSet had more + than 1 record, then we create a new RRSet on UltraDNS minus the record that + has to be deleted. + """ if not domain: function = sys._getframe().f_code.co_name @@ -320,6 +338,7 @@ def delete_acme_txt_records(domain): def get_authoritative_nameserver(domain): + """Get the authoritative nameserver for the given domain""" n = dns.name.from_text(domain) depth = 2 From 5a401b2d87526f3ef88405a21c0978a15c6d1895 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 31 Jul 2019 12:04:42 -0700 Subject: [PATCH 29/66] Added the Zone class and Record class to ultradns.py and removed the respective files --- lemur/plugins/lemur_acme/ultradns.py | 66 ++++++++++++++++++++- lemur/plugins/lemur_acme/ultradns_record.py | 26 -------- lemur/plugins/lemur_acme/ultradns_zone.py | 33 ----------- 3 files changed, 64 insertions(+), 61 deletions(-) delete mode 100644 lemur/plugins/lemur_acme/ultradns_record.py delete mode 100644 lemur/plugins/lemur_acme/ultradns_zone.py diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index 1c3aa961..40661740 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -2,8 +2,6 @@ import time import requests import json import sys -from .ultradns_zone import Zone -from .ultradns_record import Record import dns import dns.exception @@ -15,6 +13,70 @@ from flask import current_app from lemur.extensions import metrics, sentry +class Record: + """ + This class implements an Ultra DNS record. + + Accepts the response from the API call as the argument. + """ + + def __init__(self, _data): + # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response. + # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response. + self._data = _data["rrSets"][0] + + @property + def name(self): + return self._data["ownerName"] + + @property + def rrtype(self): + return self._data["rrtype"] + + @property + def rdata(self): + return self._data["rdata"] + + @property + def ttl(self): + return self._data["ttl"] + + +class Zone: + """ + This class implements an Ultra DNS zone. + """ + + def __init__(self, _data, _client="Client"): + self._data = _data + self._client = _client + + @property + def name(self): + """ + Zone name, has a trailing "." at the end, which we manually remove. + """ + return self._data["properties"]["name"][:-1] + + @property + def authoritative_type(self): + """ + Indicates whether the zone is setup as a PRIMARY or SECONDARY + """ + return self._data["properties"]["type"] + + @property + def record_count(self): + return self._data["properties"]["resourceRecordCount"] + + @property + def status(self): + """ + Returns the status of the zone - ACTIVE, SUSPENDED, etc + """ + return self._data["properties"]["status"] + + def get_ultradns_token(): """ Function to call the UltraDNS Authorization API. diff --git a/lemur/plugins/lemur_acme/ultradns_record.py b/lemur/plugins/lemur_acme/ultradns_record.py deleted file mode 100644 index 9ec8d4d8..00000000 --- a/lemur/plugins/lemur_acme/ultradns_record.py +++ /dev/null @@ -1,26 +0,0 @@ -class Record: - """ - This class implements an Ultra DNS record. - Accepts the response from the API call as the argument. - """ - - def __init__(self, _data): - # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response. - # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response. - self._data = _data["rrSets"][0] - - @property - def name(self): - return self._data["ownerName"] - - @property - def rrtype(self): - return self._data["rrtype"] - - @property - def rdata(self): - return self._data["rdata"] - - @property - def ttl(self): - return self._data["ttl"] diff --git a/lemur/plugins/lemur_acme/ultradns_zone.py b/lemur/plugins/lemur_acme/ultradns_zone.py deleted file mode 100644 index c6d90422..00000000 --- a/lemur/plugins/lemur_acme/ultradns_zone.py +++ /dev/null @@ -1,33 +0,0 @@ -class Zone: - """ - This class implements an Ultra DNS zone. - """ - - def __init__(self, _data, _client="Client"): - self._data = _data - self._client = _client - - @property - def name(self): - """ - Zone name, has a trailing "." at the end, which we manually remove. - """ - return self._data["properties"]["name"][:-1] - - @property - def authoritative_type(self): - """ - Indicates whether the zone is setup as a PRIMARY or SECONDARY - """ - return self._data["properties"]["type"] - - @property - def record_count(self): - return self._data["properties"]["resourceRecordCount"] - - @property - def status(self): - """ - Returns the status of the zone - ACTIVE, SUSPENDED, etc - """ - return self._data["properties"]["status"] From e8e4f826eab7ffda070d368f2f7be3b17fbb3b7a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 31 Jul 2019 13:08:59 -0700 Subject: [PATCH 30/66] updating logging format --- lemur/common/celery.py | 86 +++++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b889a80a..f5edb9ab 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -268,11 +268,14 @@ def clean_all_sources(): be ran periodically. This function triggers one celery task per source. """ function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "Creating celery task to clean source", + } sources = validate_sources("all") for source in sources: - current_app.logger.debug( - "Creating celery task to clean source {}".format(source.label) - ) + log_data["source"] = source.label + current_app.logger.debug(log_data) clean_source.delay(source.label) red.set(f'{function}.last_success', int(time.time())) @@ -288,7 +291,13 @@ def clean_source(source): :param source: :return: """ - current_app.logger.debug("Cleaning source {}".format(source)) + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "Cleaning source", + "source": source, + } + current_app.logger.debug(log_data) clean([source], True) @@ -298,11 +307,14 @@ def sync_all_sources(): This function will sync certificates from all sources. This function triggers one celery task per source. """ function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "creating celery task to sync source", + } sources = validate_sources("all") for source in sources: - current_app.logger.debug( - "Creating celery task to sync source {}".format(source.label) - ) + log_data["source"] = source.label + current_app.logger.debug(log_data) sync_source.delay(source.label) red.set(f'{function}.last_success', int(time.time())) @@ -361,14 +373,20 @@ def sync_source_destination(): The destination sync_as_source_name reveals the name of the suitable source-plugin. We rely on account numbers to avoid duplicates. """ - current_app.logger.debug("Syncing AWS destinations and sources") function = f"{__name__}.{sys._getframe().f_code.co_name}" - + log_data = { + "function": function, + "message": "syncing AWS destinations and sources", + } + current_app.logger.debug(log_data) for dst in destinations_service.get_all(): if add_aws_destination_to_sources(dst): - current_app.logger.debug("Source: %s added", dst.label) + log_data["message"] = "new source added" + log_data["source"] = dst.label + current_app.logger.debug(log_data) - current_app.logger.debug("Completed Syncing AWS destinations and sources") + log_data["message"] = "completed Syncing AWS destinations and sources" + current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -380,9 +398,14 @@ def certificate_reissue(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - current_app.logger.debug(f"{function}: reissuing certificates") + log_data = { + "function": function, + "message": "reissuing certificates", + } + current_app.logger.debug(log_data) cli_certificate.reissue(None, True) - current_app.logger.debug(f"{function}: reissuance completed") + log_data["message"] = "reissuance completed" + current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -394,9 +417,14 @@ def certificate_rotate(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - current_app.logger.debug(f"{function}: rotating certificates") + log_data = { + "function": function, + "message": "rotating certificates", + } + current_app.logger.debug(log_data) cli_certificate.rotate(None, None, None, None, True) - current_app.logger.debug(f"{function}: rotation completed") + log_data["message"] = "rotation completed" + current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -408,8 +436,12 @@ def endpoints_expire(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - current_app.logger.debug(f"{function}: endpoints expire") - cli_endpoints.expire(2) + log_data = { + "function": function, + "message": "endpoints expire", + } + current_app.logger.debug(log_data) + cli_endpoints.expire(2) # Time in hours red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -421,7 +453,11 @@ def get_all_zones(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - current_app.logger.debug(f"{function}: get_all_zones") + log_data = { + "function": function, + "message": "refresh all zones from available DNS providers", + } + current_app.logger.debug(log_data) cli_dns_providers.get_all_zones() red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -434,7 +470,11 @@ def check_revoked(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - current_app.logger.debug(f"{function}: check if any certificates are revoked revoked") + log_data = { + "function": function, + "message": "check if any certificates are revoked revoked", + } + current_app.logger.debug(log_data) cli_certificate.check_revoked() red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -447,7 +487,11 @@ def notify_expirations(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - current_app.logger.debug(f"{function}: Cert Expiration Notifcation") - cli_notification.expirations(["MetatronUserCertfor", "Metatron-User-Cert-for"]) + log_data = { + "function": function, + "message": "notify for cert expiration", + } + current_app.logger.debug(log_data) + cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) From 2903799b85e3c2db11d9a9b46a2d05cd850b266b Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 31 Jul 2019 14:19:49 -0700 Subject: [PATCH 31/66] Changed string formatting from "{}".format() to f"{}" for consistency --- lemur/plugins/lemur_acme/ultradns.py | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py index 40661740..dcf3e3c6 100644 --- a/lemur/plugins/lemur_acme/ultradns.py +++ b/lemur/plugins/lemur_acme/ultradns.py @@ -91,7 +91,7 @@ def get_ultradns_token(): "password": current_app.config.get("ACME_ULTRADNS_PASSWORD", ""), } base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") - resp = requests.post("{0}{1}".format(base_uri, path), data=data, verify=True) + resp = requests.post(f"{base_uri}{path}", data=data, verify=True) return resp.json()["access_token"] @@ -102,7 +102,7 @@ def _generate_header(): Contains the Authorization access_key obtained from the get_ultradns_token() function. """ access_token = get_ultradns_token() - return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"} + return {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} def _paginate(path, key): @@ -120,7 +120,7 @@ def _get(path, params=None): """Function to execute a GET request on the given URL (base_uri + path) with given params""" base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.get( - "{0}{1}".format(base_uri, path), + f"{base_uri}{path}", headers=_generate_header(), params=params, verify=True, @@ -133,7 +133,7 @@ def _delete(path): """Function to execute a DELETE request on the given URL""" base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.delete( - "{0}{1}".format(base_uri, path), + f"{base_uri}{path}", headers=_generate_header(), verify=True, ) @@ -144,7 +144,7 @@ def _post(path, params): """Executes a POST request on given URL. Body is sent in JSON format""" base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "") resp = requests.post( - "{0}{1}".format(base_uri, path), + f"{base_uri}{path}", headers=_generate_header(), data=json.dumps(params), verify=True, @@ -168,13 +168,13 @@ def _has_dns_propagated(name, token, domain): txt_records.append(txt_record.decode("utf-8")) except dns.exception.DNSException: function = sys._getframe().f_code.co_name - metrics.send("{}.fail".format(function), "counter", 1) + metrics.send(f"{function}.fail", "counter", 1) return False for txt_record in txt_records: if txt_record == token: function = sys._getframe().f_code.co_name - metrics.send("{}.success".format(function), "counter", 1) + metrics.send(f"{function}.success", "counter", 1) return True return False @@ -216,11 +216,11 @@ def wait_for_dns_change(change_id, account_number=None): } current_app.logger.debug(log_data) if status: - metrics.send("{}.success".format(function), "counter", 1) + metrics.send(f"{function}.success", "counter", 1) break time.sleep(10) if not status: - metrics.send("{}.fail".format, "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token}) + metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token}) sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)}) return @@ -253,8 +253,8 @@ def get_zone_name(domain, account_number): zone_name = z if not zone_name: function = sys._getframe().f_code.co_name - metrics.send("{}.fail".format(function), "counter", 1) - raise Exception("No UltraDNS zone found for domain: {}".format(domain)) + metrics.send(f"{function}.fail", "counter", 1) + raise Exception(f"No UltraDNS zone found for domain: {domain}") return zone_name @@ -274,12 +274,12 @@ def create_txt_record(domain, token, account_number): zone_name = get_zone_name(domain, account_number) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - fqdn = "{0}.{1}".format(node_name, zone_name) - path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name) + fqdn = f"{node_name}.{zone_name}" + path = f"/v2/zones/{zone_name}/rrsets/TXT/{node_name}" params = { "ttl": 5, "rdata": [ - "{}".format(token) + f"{token}" ], } @@ -334,19 +334,19 @@ def delete_txt_record(change_id, account_number, domain, token): zone_name = get_zone_name(domain, account_number) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name) + path = f"/v2/zones/{zone_name}/rrsets/16/{node_name}" try: rrsets = _get(path) record = Record(rrsets) except Exception as e: function = sys._getframe().f_code.co_name - metrics.send("{}.geterror".format(function), "counter", 1) + metrics.send(f"{function}.geterror", "counter", 1) # No Text Records remain or host is not in the zone anymore because all records have been deleted. return try: # Remove the record from the RRSet locally - record.rdata.remove("{}".format(token)) + record.rdata.remove(f"{token}") except ValueError: function = sys._getframe().f_code.co_name log_data = { @@ -394,7 +394,7 @@ def delete_acme_txt_records(domain): zone_name = get_zone_name(domain) zone_parts = len(zone_name.split(".")) node_name = ".".join(domain.split(".")[:-zone_parts]) - path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name) + path = f"/v2/zones/{zone_name}/rrsets/16/{node_name}" _delete(path) @@ -420,7 +420,7 @@ def get_authoritative_nameserver(domain): rcode = response.rcode() if rcode != dns.rcode.NOERROR: function = sys._getframe().f_code.co_name - metrics.send("{}.error".format(function), "counter", 1) + metrics.send(f"{function}.error", "counter", 1) if rcode == dns.rcode.NXDOMAIN: raise Exception("%s does not exist." % sub) else: From a7c2b970b0b04fc3a1b15b1d7f0acc1ca1bc8a6b Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Mon, 5 Aug 2019 13:59:59 -0700 Subject: [PATCH 32/66] Unit testing Part 1 --- lemur/plugins/lemur_acme/tests/test_acme.py | 63 ++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 3bf1d05c..29c9534e 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,8 +1,10 @@ +import json import unittest +from requests.models import Response from mock import MagicMock, Mock, patch -from lemur.plugins.lemur_acme import plugin +from lemur.plugins.lemur_acme import plugin, ultradns class TestAcme(unittest.TestCase): @@ -360,3 +362,62 @@ class TestAcme(unittest.TestCase): mock_request_certificate.return_value = ("pem_certificate", "chain") result = provider.create_certificate(csr, issuer_options) assert result + + @patch("lemur.plugins.lemur_acme.ultradns.requests") + @patch("lemur.plugins.lemur_acme.ultradns.current_app") + def test_get_ultradns_token(self, mock_current_app, mock_requests): + # ret_val = json.dumps({"access_token": "access"}) + the_response = Response() + the_response._content = b'{"access_token": "access"}' + mock_requests.post = Mock(return_value=the_response) + mock_current_app.config.get = Mock(return_value="Test") + result = ultradns.get_ultradns_token() + self.assertTrue(len(result) > 0) + + @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name") + @patch("lemur.plugins.lemur_acme.ultradns._post") + @patch("lemur.plugins.lemur_acme.ultradns.current_app") + def test_create_txt_record(self, mock_current_app, mock__post, mock_get_zone_name): + domain = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + mock_current_app.logger.debug = Mock() + mock_get_zone_name = Mock(domain, account_number, return_value="example.com") + path = "a/b/c" + params = { + "test": "Test" + } + mock__post = Mock(path, params) + result = ultradns.create_txt_record(domain, token, account_number) + self.assertEqual(type(change_id), type(result)) + + # @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name") + # @patch("lemur.plugins.lemur_acme.ultradns._get") + # @patch("lemur.plugins.lemur_acme.ultradns._delete") + # @patch("lemur.plugins.lemur_acme.ultradns._post") + # @patch("lemur.plugins.lemur_acme.ultradns.current_app") + # def test_delete_txt_record(self, mock_get_zone_name): + # domain = "test.example.com" + # token = "ABCDEFGHIJ" + # account_number = "1234567890" + # change_id = (domain, token) + # mock_get_zone_name = Mock(domain, account_number, return_value="example.com") + + # @patch("lemur.plugins.lemur_acme.ultradns.get_authoritative_nameserver") + # @patch("lemur.plugins.lemur_acme.ultradns._has_dns_propagated") + # @patch("lemur.plugins.lemur_acme.ultradns.current_app") + # def test_wait_for_dns_change(self, mock_current_app, mock_has_dns_propagated, mock_get_authoritative_nameserver): + # domain = "test.example.com" + # token = "ABCDEFGHIJ" + # account_number = "1234567890" + # change_id = (domain, token) + # mock_current_app.logger.debug = Mock() + # result = ultradns.wait_for_dns_change(change_id, token) + # self.assertEqual(result, true) + + # def test_has_dns_propagated(self): + + + + From b885cdf9d0cf9515a1351774a1eb4929297c2604 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Aug 2019 10:24:38 -0700 Subject: [PATCH 33/66] adding multi profile name support with DigiCert plug. This requires that the configs are a dict, with multiple entries, where the key is the name of the Authority used to issue certs with. DIGICERT_CIS_PROFILE_NAMES = {"sha2-rsa-ecc-root": "ssl_plus"} DIGICERT_CIS_ROOTS = {"root": "ROOT"} DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"} Hence, in DB one need to add 1) the corresponding authority table, with digicert-cis-issuer. Note the names here are used to mapping in the above config 2) the corresponding intermediary in the certificate table , with root_aurhority_id set to the id of the new authority_id --- lemur/plugins/lemur_digicert/plugin.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index c5b01cc4..5e104094 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -158,7 +158,7 @@ def map_cis_fields(options, csr): ) data = { - "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAME"), + "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES")[options['authority'].name], "common_name": options["common_name"], "additional_dns_names": get_additional_names(options), "csr": csr, @@ -423,9 +423,9 @@ class DigiCertCISSourcePlugin(SourcePlugin): required_vars = [ "DIGICERT_CIS_API_KEY", "DIGICERT_CIS_URL", - "DIGICERT_CIS_ROOT", - "DIGICERT_CIS_INTERMEDIATE", - "DIGICERT_CIS_PROFILE_NAME", + "DIGICERT_CIS_ROOTS", + "DIGICERT_CIS_INTERMEDIATES", + "DIGICERT_CIS_PROFILE_NAMES", ] validate_conf(current_app, required_vars) @@ -498,9 +498,9 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): required_vars = [ "DIGICERT_CIS_API_KEY", "DIGICERT_CIS_URL", - "DIGICERT_CIS_ROOT", - "DIGICERT_CIS_INTERMEDIATE", - "DIGICERT_CIS_PROFILE_NAME", + "DIGICERT_CIS_ROOTS", + "DIGICERT_CIS_INTERMEDIATES", + "DIGICERT_CIS_PROFILE_NAMES", ] validate_conf(current_app, required_vars) @@ -537,14 +537,14 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): if "ECC" in issuer_options["key_type"]: return ( "\n".join(str(end_entity).splitlines()), - current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATE"), + current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES")[issuer_options['authority'].name], data["id"], ) # By default return RSA return ( "\n".join(str(end_entity).splitlines()), - current_app.config.get("DIGICERT_CIS_INTERMEDIATE"), + current_app.config.get("DIGICERT_CIS_INTERMEDIATES")[issuer_options['authority'].name], data["id"], ) @@ -577,4 +577,4 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): :return: """ role = {"username": "", "password": "", "name": "digicert"} - return current_app.config.get("DIGICERT_CIS_ROOT"), "", [role] + return current_app.config.get("DIGICERT_CIS_ROOTS")[options['authority'].name], "", [role] From e2ea2ca4d1663820894caac1bc86c962bffac010 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Aug 2019 11:05:07 -0700 Subject: [PATCH 34/66] providing sample config --- lemur/tests/conf.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index 6d0d6967..af0c09ce 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -80,6 +80,13 @@ DIGICERT_API_KEY = "api-key" DIGICERT_ORG_ID = 111111 DIGICERT_ROOT = "ROOT" +DIGICERT_CIS_URL = "mock://www.digicert.com" +DIGICERT_CIS_PROFILE_NAMES = {"sha2-rsa-ecc-root": "ssl_plus"} +DIGICERT_CIS_API_KEY = "api-key" +DIGICERT_CIS_ROOTS = {"root": "ROOT"} +DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"} + + VERISIGN_URL = "http://example.com" VERISIGN_PEM_PATH = "~/" VERISIGN_FIRST_NAME = "Jim" From bbda9b1d6f4bec461e1653ea8f9825f2d22d0fcc Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Aug 2019 12:05:13 -0700 Subject: [PATCH 35/66] making sure to handle when no config file provided, though we do a check for that --- lemur/plugins/lemur_digicert/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 5e104094..6f137281 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -158,7 +158,7 @@ def map_cis_fields(options, csr): ) data = { - "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES")[options['authority'].name], + "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES", {}).get(options['authority'].name), "common_name": options["common_name"], "additional_dns_names": get_additional_names(options), "csr": csr, @@ -537,14 +537,14 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): if "ECC" in issuer_options["key_type"]: return ( "\n".join(str(end_entity).splitlines()), - current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES")[issuer_options['authority'].name], + current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name), data["id"], ) # By default return RSA return ( "\n".join(str(end_entity).splitlines()), - current_app.config.get("DIGICERT_CIS_INTERMEDIATES")[issuer_options['authority'].name], + current_app.config.get("DIGICERT_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name), data["id"], ) @@ -577,4 +577,4 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): :return: """ role = {"username": "", "password": "", "name": "digicert"} - return current_app.config.get("DIGICERT_CIS_ROOTS")[options['authority'].name], "", [role] + return current_app.config.get("DIGICERT_CIS_ROOTS", {}).get(options['authority'].name), "", [role] From ff1f73f985df4258f46aba9f7076059c8d2a2ed0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Aug 2019 12:05:36 -0700 Subject: [PATCH 36/66] fixing the plugin test to include authority --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 71efbad4..77b0a1fa 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -66,7 +66,7 @@ def test_map_fields_with_validity_years(app): } -def test_map_cis_fields(app): +def test_map_cis_fields(app, authority): from lemur.plugins.lemur_digicert.plugin import map_cis_fields names = [u"one.example.com", u"two.example.com", u"three.example.com"] @@ -80,6 +80,7 @@ def test_map_cis_fields(app): "organizational_unit": "Example Org", "validity_end": arrow.get(2017, 5, 7), "validity_start": arrow.get(2016, 10, 30), + "authority": authority, } data = map_cis_fields(options, CSR_STR) @@ -104,6 +105,7 @@ def test_map_cis_fields(app): "organization": "Example, Inc.", "organizational_unit": "Example Org", "validity_years": 2, + "authority": authority, } with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime): From 6e84e1fd59f682ec0efa6603835809cadfb741ca Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:04:38 -0700 Subject: [PATCH 37/66] Unit Tests for create_txt_record, delete_txt_record, wait_for_dns_change --- lemur/plugins/lemur_acme/tests/test_acme.py | 117 +++++++++++++------- 1 file changed, 76 insertions(+), 41 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 29c9534e..d0535718 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -374,50 +374,85 @@ class TestAcme(unittest.TestCase): result = ultradns.get_ultradns_token() self.assertTrue(len(result) > 0) - @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name") - @patch("lemur.plugins.lemur_acme.ultradns._post") @patch("lemur.plugins.lemur_acme.ultradns.current_app") - def test_create_txt_record(self, mock_current_app, mock__post, mock_get_zone_name): - domain = "test.example.com" + def test_create_txt_record(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + path = "a/b/c" + paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.', + 'pdns154.ultradns.org.']}}, 'inherit': 'ALL'}, + {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}, + {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}] + ultradns._paginate = Mock(path, "zones") + ultradns._paginate.side_effect = [[paginate_response]] + mock_current_app.logger.debug = Mock() + ultradns._post = Mock() + log_data = { + "function": "create_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record created" + } + result = ultradns.create_txt_record(domain, token, account_number) + mock_current_app.logger.debug.assert_called_with(log_data) + + @patch("lemur.plugins.lemur_acme.ultradns.current_app") + @patch("lemur.extensions.metrics") + def test_delete_txt_record(self, mock_metrics, mock_current_app): + domain = "_acme_challenge.test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) - mock_current_app.logger.debug = Mock() - mock_get_zone_name = Mock(domain, account_number, return_value="example.com") path = "a/b/c" - params = { - "test": "Test" - } - mock__post = Mock(path, params) - result = ultradns.create_txt_record(domain, token, account_number) - self.assertEqual(type(change_id), type(result)) - - # @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name") - # @patch("lemur.plugins.lemur_acme.ultradns._get") - # @patch("lemur.plugins.lemur_acme.ultradns._delete") - # @patch("lemur.plugins.lemur_acme.ultradns._post") - # @patch("lemur.plugins.lemur_acme.ultradns.current_app") - # def test_delete_txt_record(self, mock_get_zone_name): - # domain = "test.example.com" - # token = "ABCDEFGHIJ" - # account_number = "1234567890" - # change_id = (domain, token) - # mock_get_zone_name = Mock(domain, account_number, return_value="example.com") - - # @patch("lemur.plugins.lemur_acme.ultradns.get_authoritative_nameserver") - # @patch("lemur.plugins.lemur_acme.ultradns._has_dns_propagated") - # @patch("lemur.plugins.lemur_acme.ultradns.current_app") - # def test_wait_for_dns_change(self, mock_current_app, mock_has_dns_propagated, mock_get_authoritative_nameserver): - # domain = "test.example.com" - # token = "ABCDEFGHIJ" - # account_number = "1234567890" - # change_id = (domain, token) - # mock_current_app.logger.debug = Mock() - # result = ultradns.wait_for_dns_change(change_id, token) - # self.assertEqual(result, true) - - # def test_has_dns_propagated(self): - - - + paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.', + 'pdns154.ultradns.org.']}}, 'inherit': 'ALL'}, + {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}, + {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}] + ultradns._paginate = Mock(path, "zones") + ultradns._paginate.side_effect = [[paginate_response]] + mock_current_app.logger.debug = Mock() + ultradns._post = Mock() + ultradns._get = Mock() + ultradns._get.return_value = {'zoneName': 'test.example.com.com', + 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', + 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], + 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, + 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} + ultradns._delete = Mock() + mock_metrics.send = Mock() + mock_current_app.logger.debug.assert_not_called() + mock_metrics.send.assert_not_called() + @patch("lemur.extensions.metrics") + def test_wait_for_dns_change(self, mock_metrics): + ultradns._has_dns_propagated = Mock(return_value=True) + ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0") + mock_metrics.send = Mock() + mock_metrics.send.assert_not_called() From f2cbddf9e21e3360f010bb6f6a8f473c54d51b0b Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:17:16 -0700 Subject: [PATCH 38/66] Unit tests for get_zone_name, get_zones --- lemur/plugins/lemur_acme/tests/test_acme.py | 40 +++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index d0535718..a5e7c3e7 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -383,8 +383,9 @@ class TestAcme(unittest.TestCase): paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.', - 'pdns154.ultradns.org.']}}, 'inherit': 'ALL'}, + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}, {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { @@ -456,3 +457,38 @@ class TestAcme(unittest.TestCase): ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0") mock_metrics.send = Mock() mock_metrics.send.assert_not_called() + + def test_get_zone_name(self): + zones = ['example.com', 'test.example.com'] + zone = "test.example.com" + domain = "_acme-challenge.test.example.com" + account_number = "1234567890" + ultradns.get_zones = Mock(return_value=zones) + result = ultradns.get_zone_name(domain, account_number) + self.assertEqual(result, zone) + + def test_get_zones(self): + account_number = "1234567890" + path = "a/b/c" + zones = ['example.com', 'test.example.com'] + paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.', + 'pdns154.ultradns.org.']}}, 'inherit': 'ALL'}, + {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}, + {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { + 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', + 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'inherit': 'ALL'}] + ultradns._paginate = Mock(path, "zones") + ultradns._paginate.side_effect = [[paginate_response]] + result = ultradns.get_zones(account_number) + self.assertEqual(result, zones) From 785c1ca73ec18ed7c4bdbd806513d063c296f5c4 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:20:24 -0700 Subject: [PATCH 39/66] test_create_txt_record modified - get_zone_name mocked to return the zone name directly, instead of actually running the function. --- lemur/plugins/lemur_acme/tests/test_acme.py | 23 ++------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index a5e7c3e7..da935a46 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -377,29 +377,10 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.ultradns.current_app") def test_create_txt_record(self, mock_current_app): domain = "_acme_challenge.test.example.com" + zone = "test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" - path = "a/b/c" - paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, - 'inherit': 'ALL'}, - {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, - 'inherit': 'ALL'}, - {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, - 'inherit': 'ALL'}] - ultradns._paginate = Mock(path, "zones") - ultradns._paginate.side_effect = [[paginate_response]] + ultradns.get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() ultradns._post = Mock() log_data = { From 31c2d207a2bafd83676cba7b94e68d9f23fe61b7 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:23:05 -0700 Subject: [PATCH 40/66] test_delete_txt_record fixed. Function call was missing earlier --- lemur/plugins/lemur_acme/tests/test_acme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index da935a46..ae78f911 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -429,6 +429,7 @@ class TestAcme(unittest.TestCase): 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} ultradns._delete = Mock() mock_metrics.send = Mock() + ultradns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_not_called() mock_metrics.send.assert_not_called() From 37a1b55b0832b85bf2ccfbbd22f2fa92851bb2f9 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:27:21 -0700 Subject: [PATCH 41/66] test_delete_txt_record changed to mock get_zone_name and return the value directly instead of executing the function. --- lemur/plugins/lemur_acme/tests/test_acme.py | 22 ++------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index ae78f911..f6fe5b2f 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -396,30 +396,12 @@ class TestAcme(unittest.TestCase): @patch("lemur.extensions.metrics") def test_delete_txt_record(self, mock_metrics, mock_current_app): domain = "_acme_challenge.test.example.com" + zone = "test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) - path = "a/b/c" - paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.', - 'pdns154.ultradns.org.']}}, 'inherit': 'ALL'}, - {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, - 'inherit': 'ALL'}, - {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, - 'inherit': 'ALL'}] - ultradns._paginate = Mock(path, "zones") - ultradns._paginate.side_effect = [[paginate_response]] mock_current_app.logger.debug = Mock() + ultradns.get_zone_name = Mock(return_value=zone) ultradns._post = Mock() ultradns._get = Mock() ultradns._get.return_value = {'zoneName': 'test.example.com.com', From 894502644c03aa0dc2fcbac12ed34360f8d8d9e0 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:39:20 -0700 Subject: [PATCH 42/66] test_wait_for_dns_change fixed! --- lemur/plugins/lemur_acme/tests/test_acme.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index f6fe5b2f..199b4a05 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -415,12 +415,26 @@ class TestAcme(unittest.TestCase): mock_current_app.logger.debug.assert_not_called() mock_metrics.send.assert_not_called() + @patch("lemur.plugins.lemur_acme.ultradns.current_app") @patch("lemur.extensions.metrics") - def test_wait_for_dns_change(self, mock_metrics): + def test_wait_for_dns_change(self, mock_metrics, mock_current_app): ultradns._has_dns_propagated = Mock(return_value=True) ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0") mock_metrics.send = Mock() - mock_metrics.send.assert_not_called() + domain = "_acme-challenge.test.example.com" + token = "ABCDEFGHIJ" + change_id = (domain, token) + mock_current_app.logger.debug = Mock() + ultradns.wait_for_dns_change(change_id) + # mock_metrics.send.assert_not_called() + log_data = { + "function": "wait_for_dns_change", + "fqdn": domain, + "status": True, + "message": "Record status on Public DNS" + } + mock_current_app.logger.debug.assert_called_with(log_data) + def test_get_zone_name(self): zones = ['example.com', 'test.example.com'] From 3ff56fc5950ffd7ecb236a450f15b8218e5eedb5 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:42:11 -0700 Subject: [PATCH 43/66] Blank line removed --- lemur/plugins/lemur_acme/tests/test_acme.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 199b4a05..61a738bc 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -435,7 +435,6 @@ class TestAcme(unittest.TestCase): } mock_current_app.logger.debug.assert_called_with(log_data) - def test_get_zone_name(self): zones = ['example.com', 'test.example.com'] zone = "test.example.com" From fa7f71d8599800fa5b54c2a2696f9793865a7481 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:53:10 -0700 Subject: [PATCH 44/66] Modified paginate response to dummy values --- lemur/plugins/lemur_acme/tests/test_acme.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 61a738bc..2d2055d8 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -390,6 +390,7 @@ class TestAcme(unittest.TestCase): "message": "TXT record created" } result = ultradns.create_txt_record(domain, token, account_number) + # TODO: check change_id mock_current_app.logger.debug.assert_called_with(log_data) @patch("lemur.plugins.lemur_acme.ultradns.current_app") @@ -448,22 +449,22 @@ class TestAcme(unittest.TestCase): account_number = "1234567890" path = "a/b/c" zones = ['example.com', 'test.example.com'] - paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.', - 'pdns154.ultradns.org.']}}, 'inherit': 'ALL'}, - {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY', + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', 'example.ultradns.biz.', + 'example.ultradns.org.']}}, 'inherit': 'ALL'}, + {'properties': {'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', + 'example.ultradns.biz.', 'example.ultradns.org.']}}, 'inherit': 'ALL'}, - {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY', + {'properties': {'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', - 'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}}, + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', + 'example.ultradns.biz.', 'example.ultradns.org.']}}, 'inherit': 'ALL'}] ultradns._paginate = Mock(path, "zones") ultradns._paginate.side_effect = [[paginate_response]] From b4f4e4dc241bb182b87fa97584dc2f399e3aa71c Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 13:55:02 -0700 Subject: [PATCH 45/66] Added extra check for return value to test_create_txt_record --- lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 2d2055d8..58857b75 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -380,6 +380,7 @@ class TestAcme(unittest.TestCase): zone = "test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" + change_id = (domain, token) ultradns.get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() ultradns._post = Mock() @@ -390,8 +391,8 @@ class TestAcme(unittest.TestCase): "message": "TXT record created" } result = ultradns.create_txt_record(domain, token, account_number) - # TODO: check change_id mock_current_app.logger.debug.assert_called_with(log_data) + self.assertEqual(result, change_id) @patch("lemur.plugins.lemur_acme.ultradns.current_app") @patch("lemur.extensions.metrics") From cadf372f7b5ca909a142ad732fdd92aaa56d0399 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 14:02:10 -0700 Subject: [PATCH 46/66] Removed hardcoded value from function call --- lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 58857b75..b66e6d58 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -421,7 +421,8 @@ class TestAcme(unittest.TestCase): @patch("lemur.extensions.metrics") def test_wait_for_dns_change(self, mock_metrics, mock_current_app): ultradns._has_dns_propagated = Mock(return_value=True) - ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0") + nameserver = "0.0.0.0" + ultradns.get_authoritative_nameserver = Mock(return_value=nameserver) mock_metrics.send = Mock() domain = "_acme-challenge.test.example.com" token = "ABCDEFGHIJ" From 43f5c8b34e74648bab47c63bebd783c1bd5a1410 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 14:08:06 -0700 Subject: [PATCH 47/66] Fixed indentation --- lemur/plugins/lemur_acme/tests/test_acme.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index b66e6d58..c4d10039 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -454,8 +454,9 @@ class TestAcme(unittest.TestCase): paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', 'example.ultradns.biz.', - 'example.ultradns.org.']}}, 'inherit': 'ALL'}, + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', + 'example.ultradns.biz.', 'example.ultradns.org.']}}, + 'inherit': 'ALL'}, {'properties': {'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY', 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { From a6bf081bec357fde6a17153fa57c1f51f0a621a6 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 14:08:27 -0700 Subject: [PATCH 48/66] Remove unused import --- lemur/plugins/lemur_acme/tests/test_acme.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index c4d10039..31a9e370 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,4 +1,3 @@ -import json import unittest from requests.models import Response From a97283f0a43bd3203dc8a00220e6d65a524bc13b Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 14:23:09 -0700 Subject: [PATCH 49/66] Fixed indentation --- lemur/plugins/lemur_acme/tests/test_acme.py | 43 ++++++++++++--------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 31a9e370..f49141a8 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -450,24 +450,31 @@ class TestAcme(unittest.TestCase): account_number = "1234567890" path = "a/b/c" zones = ['example.com', 'test.example.com'] - paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', - 'example.ultradns.biz.', 'example.ultradns.org.']}}, - 'inherit': 'ALL'}, - {'properties': {'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', - 'example.ultradns.biz.', 'example.ultradns.org.']}}, - 'inherit': 'ALL'}, - {'properties': {'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY', - 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, - 'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': { - 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', - 'example.ultradns.biz.', 'example.ultradns.org.']}}, - 'inherit': 'ALL'}] + paginate_response = [{ + 'properties': { + 'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, + 'registrarInfo': { + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', + 'example.ultradns.biz.', 'example.ultradns.org.']}}, + 'inherit': 'ALL'}, { + 'properties': { + 'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, + 'registrarInfo': { + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', + 'example.ultradns.biz.', 'example.ultradns.org.']}}, + 'inherit': 'ALL'}, { + 'properties': { + 'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY', + 'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9, + 'lastModifiedDateTime': '2017-06-14T06:45Z'}, + 'registrarInfo': { + 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', + 'example.ultradns.biz.', 'example.ultradns.org.']}}, + 'inherit': 'ALL'}] ultradns._paginate = Mock(path, "zones") ultradns._paginate.side_effect = [[paginate_response]] result = ultradns.get_zones(account_number) From d9aef2da3e51cfe1c20383301b457207b183d3d0 Mon Sep 17 00:00:00 2001 From: Kush Bavishi Date: Wed, 7 Aug 2019 14:38:18 -0700 Subject: [PATCH 50/66] Changed dummy nameserver value --- lemur/plugins/lemur_acme/tests/test_acme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index f49141a8..2f9dd719 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -420,7 +420,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.extensions.metrics") def test_wait_for_dns_change(self, mock_metrics, mock_current_app): ultradns._has_dns_propagated = Mock(return_value=True) - nameserver = "0.0.0.0" + nameserver = "1.1.1.1" ultradns.get_authoritative_nameserver = Mock(return_value=nameserver) mock_metrics.send = Mock() domain = "_acme-challenge.test.example.com" From 9a02230d63419285ef4836784462de3538ce4847 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Aug 2019 17:48:06 -0700 Subject: [PATCH 51/66] adding soft time outs for celery --- lemur/common/celery.py | 91 +++++++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f5edb9ab..e868585a 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -282,7 +282,7 @@ def clean_all_sources(): metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +@celery.task(soft_time_limit=600) def clean_source(source): """ This celery task will clean the specified source. This is a destructive operation that will delete unused @@ -298,7 +298,13 @@ def clean_source(source): "source": source, } current_app.logger.debug(log_data) - clean([source], True) + try: + clean([source], True) + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("clean_source_timeout", "counter", 1) @celery.task() @@ -391,7 +397,7 @@ def sync_source_destination(): metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +@celery.task(soft_time_limit=3600) def certificate_reissue(): """ This celery task reissues certificates which are pending reissue @@ -403,14 +409,21 @@ def certificate_reissue(): "message": "reissuing certificates", } current_app.logger.debug(log_data) - cli_certificate.reissue(None, True) + try: + cli_certificate.reissue(None, True) + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("certificate_reissue_timeout", "counter", 1) + return log_data["message"] = "reissuance completed" current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +@celery.task(soft_time_limit=3600) def certificate_rotate(): """ This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert @@ -422,14 +435,21 @@ def certificate_rotate(): "message": "rotating certificates", } current_app.logger.debug(log_data) - cli_certificate.rotate(None, None, None, None, True) + try: + cli_certificate.rotate(None, None, None, None, True) + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("certificate_rotate_timeout", "counter", 1) + return log_data["message"] = "rotation completed" current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +@celery.task(soft_time_limit=3600) def endpoints_expire(): """ This celery task removes all endpoints that have not been recently updated @@ -441,12 +461,19 @@ def endpoints_expire(): "message": "endpoints expire", } current_app.logger.debug(log_data) - cli_endpoints.expire(2) # Time in hours + try: + cli_endpoints.expire(2) # Time in hours + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("endpoints_expire_timeout", "counter", 1) + return red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +@celery.task(soft_time_limit=600) def get_all_zones(): """ This celery syncs all zones from the available dns providers @@ -458,29 +485,58 @@ def get_all_zones(): "message": "refresh all zones from available DNS providers", } current_app.logger.debug(log_data) - cli_dns_providers.get_all_zones() + try: + cli_dns_providers.get_all_zones() + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("get_all_zones_timeout", "counter", 1) + return red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +@celery.task(soft_time_limit=3600) def check_revoked(): """ This celery task attempts to check if any certs are expired :return: """ + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "check if any certificates are revoked revoked", } + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) - cli_certificate.check_revoked() + try: + cli_certificate.check_revoked() + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("check_revoked_timeout", "counter", 1) + return + red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) -@celery.task() +check_revoked() + + +@celery.task(soft_time_limit=3600) def notify_expirations(): """ This celery task notifies about expiring certs @@ -492,6 +548,13 @@ def notify_expirations(): "message": "notify for cert expiration", } current_app.logger.debug(log_data) - cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) + try: + cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) + except SoftTimeLimitExceeded: + log_data["message"] = "Checking revoked: Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("notify_expirations_timeout", "counter", 1) + return red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) From 3b9b94623fc0571e3777232906049b4c857e058f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Aug 2019 18:06:59 -0700 Subject: [PATCH 52/66] cleaning up --- lemur/common/celery.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index e868585a..dfeb7017 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -533,9 +533,6 @@ def check_revoked(): metrics.send(f"{function}.success", 'counter', 1) -check_revoked() - - @celery.task(soft_time_limit=3600) def notify_expirations(): """ From da9c91afb4f5b6bc6cb4016d5e3e049e4db41f13 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 8 Aug 2019 17:56:22 -0700 Subject: [PATCH 53/66] fixing metric bug --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f5edb9ab..b19a9607 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -348,7 +348,7 @@ def sync_source(source): return try: sync([source]) - metrics.send(f"{function}.success", 'counter', '1', metric_tags={"source": source}) + metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) From bf47f87c215f9c6042374ddf3a43f5f4bbc24d43 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 12 Aug 2019 13:52:01 -0700 Subject: [PATCH 54/66] preventing celery duplicate tasks --- lemur/common/celery.py | 91 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b19a9607..a79ec838 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -248,6 +248,15 @@ def remove_old_acme_certs(): } pending_certs = pending_certificate_service.get_pending_certs("all") + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): @@ -311,6 +320,17 @@ def sync_all_sources(): "function": function, "message": "creating celery task to sync source", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + + sources = validate_sources("all") for source in sources: log_data["source"] = source.label @@ -340,6 +360,17 @@ def sync_source(source): "source": source, "task_id": task_id, } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + + current_app.logger.debug(log_data) if task_id and is_task_active(function, task_id, (source,)): @@ -378,6 +409,16 @@ def sync_source_destination(): "function": function, "message": "syncing AWS destinations and sources", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) for dst in destinations_service.get_all(): if add_aws_destination_to_sources(dst): @@ -402,6 +443,16 @@ def certificate_reissue(): "function": function, "message": "reissuing certificates", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) cli_certificate.reissue(None, True) log_data["message"] = "reissuance completed" @@ -421,6 +472,16 @@ def certificate_rotate(): "function": function, "message": "rotating certificates", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) cli_certificate.rotate(None, None, None, None, True) log_data["message"] = "rotation completed" @@ -440,6 +501,16 @@ def endpoints_expire(): "function": function, "message": "endpoints expire", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) cli_endpoints.expire(2) # Time in hours red.set(f'{function}.last_success', int(time.time())) @@ -457,6 +528,16 @@ def get_all_zones(): "function": function, "message": "refresh all zones from available DNS providers", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) cli_dns_providers.get_all_zones() red.set(f'{function}.last_success', int(time.time())) @@ -491,6 +572,16 @@ def notify_expirations(): "function": function, "message": "notify for cert expiration", } + + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + if task_id and is_task_active(function, task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) red.set(f'{function}.last_success', int(time.time())) From 07a9c56fb86b97c07a04b67fe576540d961cfb7b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 09:35:57 -0700 Subject: [PATCH 55/66] making lint happy --- lemur/common/celery.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a79ec838..bcd7b580 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -31,7 +31,6 @@ from lemur.dns_providers import cli as cli_dns_providers from lemur.notifications import cli as cli_notification from lemur.endpoints import cli as cli_endpoints - if current_app: flask_app = current_app else: @@ -256,7 +255,7 @@ def remove_old_acme_certs(): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return - + # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): @@ -330,7 +329,6 @@ def sync_all_sources(): current_app.logger.debug(log_data) return - sources = validate_sources("all") for source in sources: log_data["source"] = source.label @@ -370,7 +368,6 @@ def sync_source(source): current_app.logger.debug(log_data) return - current_app.logger.debug(log_data) if task_id and is_task_active(function, task_id, (source,)): From 4d728738eeb2bcbc7afa4a30d15f6d7656674873 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 11:42:43 -0700 Subject: [PATCH 56/66] handling celery tasks without any arguments --- lemur/common/celery.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index bcd7b580..fa739029 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -65,6 +65,9 @@ celery = make_celery(flask_app) def is_task_active(fun, task_id, args): from celery.task.control import inspect + if not args: + args = '()' # empty args + i = inspect() active_tasks = i.active() for _, tasks in active_tasks.items(): From c29f2825607df669ff67d3713fcfc4feb3fa96bc Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 11:52:56 -0700 Subject: [PATCH 57/66] improved the flow for checking if the task is active --- lemur/common/celery.py | 169 ++++++++++++++++++++++++++--------------- 1 file changed, 109 insertions(+), 60 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index fa739029..a37f96e5 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -91,6 +91,21 @@ def report_celery_last_success_metrics(): """ function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "message": "recurrent task", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_time = int(time.time()) schedule = current_app.config.get('CELERYBEAT_SCHEDULE') for _, t in schedule.items(): @@ -215,15 +230,25 @@ def fetch_acme_cert(id): @celery.task() def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" - pending_certs = pending_certificate_service.get_unresolved_pending_certs() function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + log_data = { "function": function, "message": "Starting job.", + "task_id": task_id, } + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) + pending_certs = pending_certificate_service.get_unresolved_pending_certs() # We only care about certs using the acme-issuer plugin for cert in pending_certs: @@ -244,21 +269,23 @@ def fetch_all_pending_acme_certs(): def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "Starting job.", - } - pending_certs = pending_certificate_service.get_pending_certs("all") - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "Starting job.", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return + pending_certs = pending_certificate_service.get_pending_certs("all") + # Delete pending certs more than a week old for cert in pending_certs: if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): @@ -279,10 +306,21 @@ def clean_all_sources(): be ran periodically. This function triggers one celery task per source. """ function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + log_data = { "function": function, "message": "Creating celery task to clean source", + "task_id": task_id, } + + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + sources = validate_sources("all") for source in sources: log_data["source"] = source.label @@ -303,11 +341,22 @@ def clean_source(source): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + log_data = { "function": function, "message": "Cleaning source", "source": source, + "task_id": task_id, } + + if task_id and is_task_active(function, task_id, (source,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + current_app.logger.debug(log_data) clean([source], True) @@ -318,16 +367,17 @@ def sync_all_sources(): This function will sync certificates from all sources. This function triggers one celery task per source. """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "creating celery task to sync source", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "creating celery task to sync source", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return @@ -355,6 +405,7 @@ def sync_source(source): task_id = None if celery.current_task: task_id = celery.current_task.request.id + log_data = { "function": function, "message": "Syncing source", @@ -362,21 +413,12 @@ def sync_source(source): "task_id": task_id, } - task_id = None - if celery.current_task: - task_id = celery.current_task.request.id - - if task_id and is_task_active(function, task_id, (id,)): + if task_id and is_task_active(function, task_id, (source,)): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return current_app.logger.debug(log_data) - - if task_id and is_task_active(function, task_id, (source,)): - log_data["message"] = "Skipping task: Task is already active" - current_app.logger.debug(log_data) - return try: sync([source]) metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) @@ -405,16 +447,17 @@ def sync_source_destination(): We rely on account numbers to avoid duplicates. """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "syncing AWS destinations and sources", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "syncing AWS destinations and sources", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return @@ -439,16 +482,17 @@ def certificate_reissue(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "reissuing certificates", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "reissuing certificates", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return @@ -468,16 +512,18 @@ def certificate_rotate(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "rotating certificates", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "rotating certificates", + "task_id": task_id, + + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return @@ -497,16 +543,17 @@ def endpoints_expire(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "endpoints expire", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "endpoints expire", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return @@ -524,16 +571,17 @@ def get_all_zones(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "refresh all zones from available DNS providers", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "refresh all zones from available DNS providers", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return @@ -568,16 +616,17 @@ def notify_expirations(): :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" - log_data = { - "function": function, - "message": "notify for cert expiration", - } - task_id = None if celery.current_task: task_id = celery.current_task.request.id - if task_id and is_task_active(function, task_id, (id,)): + log_data = { + "function": function, + "message": "notify for cert expiration", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return From a3dfc3ef0ad9c43fab1862a6c5b4361096b452ff Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 11:58:58 -0700 Subject: [PATCH 58/66] consistency --- lemur/common/celery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index dfeb7017..acb20081 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -503,18 +503,18 @@ def check_revoked(): This celery task attempts to check if any certs are expired :return: """ - + function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id - function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "message": "check if any certificates are revoked revoked", + "task_id": task_id, } - if task_id and is_task_active(function, task_id, (id,)): + if task_id and is_task_active(function, task_id, None): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return From 22c60fedad9047edf22d4126442e5ed48a338a70 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 12:11:04 -0700 Subject: [PATCH 59/66] cosmetics --- lemur/common/celery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a37f96e5..06b93a31 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -31,6 +31,7 @@ from lemur.dns_providers import cli as cli_dns_providers from lemur.notifications import cli as cli_notification from lemur.endpoints import cli as cli_endpoints + if current_app: flask_app = current_app else: From 6e17d36d76b228d878ecab1c2f983de25237080f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 12:16:23 -0700 Subject: [PATCH 60/66] typos --- lemur/common/celery.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index acb20081..e36d8b35 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -301,7 +301,7 @@ def clean_source(source): try: clean([source], True) except SoftTimeLimitExceeded: - log_data["message"] = "Checking revoked: Time limit exceeded." + log_data["message"] = "Clean source: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() metrics.send("clean_source_timeout", "counter", 1) @@ -412,7 +412,7 @@ def certificate_reissue(): try: cli_certificate.reissue(None, True) except SoftTimeLimitExceeded: - log_data["message"] = "Checking revoked: Time limit exceeded." + log_data["message"] = "Certificate reissue: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() metrics.send("certificate_reissue_timeout", "counter", 1) @@ -438,7 +438,7 @@ def certificate_rotate(): try: cli_certificate.rotate(None, None, None, None, True) except SoftTimeLimitExceeded: - log_data["message"] = "Checking revoked: Time limit exceeded." + log_data["message"] = "Certificate rotate: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() metrics.send("certificate_rotate_timeout", "counter", 1) @@ -464,7 +464,7 @@ def endpoints_expire(): try: cli_endpoints.expire(2) # Time in hours except SoftTimeLimitExceeded: - log_data["message"] = "Checking revoked: Time limit exceeded." + log_data["message"] = "endpoint expire: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() metrics.send("endpoints_expire_timeout", "counter", 1) @@ -488,7 +488,7 @@ def get_all_zones(): try: cli_dns_providers.get_all_zones() except SoftTimeLimitExceeded: - log_data["message"] = "Checking revoked: Time limit exceeded." + log_data["message"] = "get all zones: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() metrics.send("get_all_zones_timeout", "counter", 1) @@ -548,7 +548,7 @@ def notify_expirations(): try: cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) except SoftTimeLimitExceeded: - log_data["message"] = "Checking revoked: Time limit exceeded." + log_data["message"] = "Notify expiring Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() metrics.send("notify_expirations_timeout", "counter", 1) From 2de3f287ab3ace80e6fb750e12d6f4b990bb02c5 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 13 Aug 2019 12:21:27 -0700 Subject: [PATCH 61/66] standardizing the timeouts to easier monitor any timeouts --- lemur/common/celery.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index e36d8b35..a3f9cc5f 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -304,7 +304,7 @@ def clean_source(source): log_data["message"] = "Clean source: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("clean_source_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) @celery.task() @@ -359,9 +359,8 @@ def sync_source(source): log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send( - "sync_source_timeout", "counter", 1, metric_tags={"source": source} - ) + metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source}) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return log_data["message"] = "Done syncing source" @@ -415,8 +414,9 @@ def certificate_reissue(): log_data["message"] = "Certificate reissue: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("certificate_reissue_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return + log_data["message"] = "reissuance completed" current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) @@ -441,8 +441,9 @@ def certificate_rotate(): log_data["message"] = "Certificate rotate: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("certificate_rotate_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return + log_data["message"] = "rotation completed" current_app.logger.debug(log_data) red.set(f'{function}.last_success', int(time.time())) @@ -467,8 +468,9 @@ def endpoints_expire(): log_data["message"] = "endpoint expire: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("endpoints_expire_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return + red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -491,8 +493,9 @@ def get_all_zones(): log_data["message"] = "get all zones: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("get_all_zones_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return + red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) @@ -526,7 +529,7 @@ def check_revoked(): log_data["message"] = "Checking revoked: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("check_revoked_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return red.set(f'{function}.last_success', int(time.time())) @@ -551,7 +554,8 @@ def notify_expirations(): log_data["message"] = "Notify expiring Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("notify_expirations_timeout", "counter", 1) + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return + red.set(f'{function}.last_success', int(time.time())) metrics.send(f"{function}.success", 'counter', 1) From 1c6fee7292b93da2ca2157c2ac340bdd7c7bcb4e Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 15 Aug 2019 10:52:26 -0700 Subject: [PATCH 62/66] Allow better DNS autodetection for domains that directly match a DNS hosted zone --- lemur/plugins/lemur_acme/plugin.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index b0774cbe..f31ffdcb 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -294,7 +294,7 @@ class AcmeHandler(object): if not dns_provider.domains: continue for name in dns_provider.domains: - if domain.endswith("." + name): + if name == domain or domain.endswith("." + name): if len(name) > match_length: self.dns_providers_for_domain[domain] = [dns_provider] match_length = len(name) @@ -370,7 +370,12 @@ class AcmeHandler(object): pass def get_dns_provider(self, type): - provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53, "ultradns": ultradns} + provider_types = { + "cloudflare": cloudflare, + "dyn": dyn, + "route53": route53, + "ultradns": ultradns, + } provider = provider_types.get(type) if not provider: raise UnknownProvider("No such DNS provider: {}".format(type)) @@ -424,7 +429,12 @@ class ACMEIssuerPlugin(IssuerPlugin): def get_dns_provider(self, type): self.acme = AcmeHandler() - provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53, "ultradns": ultradns} + provider_types = { + "cloudflare": cloudflare, + "dyn": dyn, + "route53": route53, + "ultradns": ultradns, + } provider = provider_types.get(type) if not provider: raise UnknownProvider("No such DNS provider: {}".format(type)) From 9b04d901c482d1d8909d7d5d571ddded35235b94 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 15 Aug 2019 19:14:08 -0700 Subject: [PATCH 63/66] metric for missing certificate from an endpoint --- lemur/sources/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index ec988623..b6bdb1be 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -15,6 +15,7 @@ from lemur.sources.models import Source from lemur.certificates.models import Certificate from lemur.certificates import service as certificate_service from lemur.endpoints import service as endpoint_service +from lemur.extensions import metrics from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema @@ -94,6 +95,8 @@ def sync_endpoints(source): certificate_name, endpoint["name"] ) ) + metrics.send("endpoint.certificate.not.found", + "counter", 1, metric_tags={"cert": certificate_name, "endpoint": endpoint["name"]}) continue policy = endpoint.pop("policy") From e5e395f0d964d6fa8b43442576ef0b92979f2a96 Mon Sep 17 00:00:00 2001 From: Javier Ramos Date: Tue, 20 Aug 2019 09:29:58 +0200 Subject: [PATCH 64/66] Show number of found items in pager This commit does not involve any additional query as the data is already in API calls' responses --- lemur/static/app/angular/pager.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/static/app/angular/pager.html b/lemur/static/app/angular/pager.html index 3dc8a7d0..d9ee5204 100644 --- a/lemur/static/app/angular/pager.html +++ b/lemur/static/app/angular/pager.html @@ -4,6 +4,9 @@ +
+  Found +
From db91e48395da9273b16405283bfddf20a27ca43b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 21 Aug 2019 09:54:18 -0700 Subject: [PATCH 65/66] adding account number for better logging, since the endpoint is not available in Lemur DB --- lemur/sources/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index b6bdb1be..d5bd7426 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -96,7 +96,8 @@ def sync_endpoints(source): ) ) metrics.send("endpoint.certificate.not.found", - "counter", 1, metric_tags={"cert": certificate_name, "endpoint": endpoint["name"]}) + "counter", 1, + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) continue policy = endpoint.pop("policy") From 972051a61eb8dd9e1c60c5de9ad5e3aeef860eb6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 20 Sep 2019 10:16:23 -0700 Subject: [PATCH 66/66] removing 3 and 4 years from validity range options --- .../app/angular/certificates/certificate/tracking.tpl.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index 19d8f37f..7ac2107f 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -133,7 +133,7 @@
@@ -141,8 +141,6 @@ - -