From cdb83c48c5504130a6370703845b812af7bf6c51 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 17 Jun 2019 10:41:11 -0700
Subject: [PATCH 01/40] API additions for viewing expired certs as well.
 Default behavior modified to show only valid certs and those which have
 expired less than 1 month ago.

---
 lemur/certificates/service.py | 12 +++++++-----
 lemur/certificates/views.py   |  1 +
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py
index 544c03d8..d9370232 100644
--- a/lemur/certificates/service.py
+++ b/lemur/certificates/service.py
@@ -329,12 +329,14 @@ def render(args):
     """
     query = database.session_query(Certificate)
 
-    time_range = args.pop("time_range")
-    if not time_range:
-        six_month_old = arrow.now()\
-            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -6))\
+    show_expired = args.pop("showExpired")
+    if show_expired != 1:
+        one_month_old = arrow.now()\
+            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\
             .format("YYYY-MM-DD")
-        query = query.filter(Certificate.not_after > six_month_old)
+        query = query.filter(Certificate.not_after > one_month_old)
+
+    time_range = args.pop("time_range")
 
     destination_id = args.pop("destination_id")
     notification_id = args.pop("notification_id", None)
diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py
index 61a74a59..1a003e78 100644
--- a/lemur/certificates/views.py
+++ b/lemur/certificates/views.py
@@ -347,6 +347,7 @@ class CertificatesList(AuthenticatedResource):
         )
         parser.add_argument("creator", type=str, location="args")
         parser.add_argument("show", type=str, location="args")
+        parser.add_argument("showExpired", type=int, location="args")
 
         args = parser.parse_args()
         args["user"] = g.user

From e37a7c775ea78730d8f5c68bd82d9fbef9e1aa9b Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Thu, 18 Jul 2019 14:29:54 -0700
Subject: [PATCH 02/40] Initial commit for the UltraDNS plugin to support Lets
 Encrypt

---
 lemur/dns_providers/service.py       |   1 +
 lemur/plugins/lemur_acme/plugin.py   |   6 +-
 lemur/plugins/lemur_acme/ultradns.py | 221 +++++++++++++++++++++++++++
 3 files changed, 225 insertions(+), 3 deletions(-)
 create mode 100644 lemur/plugins/lemur_acme/ultradns.py

diff --git a/lemur/dns_providers/service.py b/lemur/dns_providers/service.py
index ec9fa0de..29f98a5b 100644
--- a/lemur/dns_providers/service.py
+++ b/lemur/dns_providers/service.py
@@ -98,6 +98,7 @@ def get_types():
                     ],
                 },
                 {"name": "dyn"},
+                {"name": "ultradns"},
             ]
         },
     )
diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py
index c734923a..b0774cbe 100644
--- a/lemur/plugins/lemur_acme/plugin.py
+++ b/lemur/plugins/lemur_acme/plugin.py
@@ -31,7 +31,7 @@ from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProv
 from lemur.extensions import metrics, sentry
 from lemur.plugins import lemur_acme as acme
 from lemur.plugins.bases import IssuerPlugin
-from lemur.plugins.lemur_acme import cloudflare, dyn, route53
+from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns
 
 
 class AuthorizationRecord(object):
@@ -370,7 +370,7 @@ class AcmeHandler(object):
                         pass
 
     def get_dns_provider(self, type):
-        provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53}
+        provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53, "ultradns": ultradns}
         provider = provider_types.get(type)
         if not provider:
             raise UnknownProvider("No such DNS provider: {}".format(type))
@@ -424,7 +424,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
     def get_dns_provider(self, type):
         self.acme = AcmeHandler()
 
-        provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53}
+        provider_types = {"cloudflare": cloudflare, "dyn": dyn, "route53": route53, "ultradns": ultradns}
         provider = provider_types.get(type)
         if not provider:
             raise UnknownProvider("No such DNS provider: {}".format(type))
diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
new file mode 100644
index 00000000..de65b47f
--- /dev/null
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -0,0 +1,221 @@
+import time
+import requests
+import json
+
+import dns
+import dns.exception
+import dns.name
+import dns.query
+import dns.resolver
+
+from flask import current_app
+from lemur.extensions import metrics, sentry
+
+use_http = False
+
+
+def get_ultradns_token():
+    path = "/v2/authorization/token"
+    data = {
+        "grant_type": "password",
+        "username": current_app.config.get("ACME_ULTRADNS_USERNAME", ""),
+        "password": current_app.config.get("ACME_ULTRADNS_PASSWORD", ""),
+    }
+    base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
+    resp = requests.post("{0}{1}".format(base_uri, path), data=data, verify=True)
+    return resp.json()["access_token"]
+
+
+def _generate_header():
+    access_token = get_ultradns_token()
+    return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"}
+
+
+def _paginate(path, key):
+    limit = 100
+    params = {"offset": 0, "limit": 1}
+    # params["offset"] = 0
+    # params["limit"] = 1
+    resp = _get(path, params)
+    for index in range(0, resp["resultInfo"]["totalCount"], limit):
+        params["offset"] = index
+        params["limit"] = limit
+        resp = _get(path, params)
+        yield resp[key]
+
+
+def _get(path, params=None):
+    base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
+    resp = requests.get(
+        "{0}{1}".format(base_uri, path),
+        headers=_generate_header(),
+        params=params,
+        verify=True,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def _delete(path):
+    base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
+    resp = requests.delete(
+        "{0}{1}".format(base_uri, path),
+        headers=_generate_header(),
+        verify=True,
+    )
+    resp.raise_for_status()
+
+
+def _post(path, params):
+    base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
+    resp = requests.post(
+        "{0}{1}".format(base_uri, path),
+        headers=_generate_header(),
+        data=json.dumps(params),
+        verify=True,
+    )
+    resp.raise_for_status()
+
+
+def _has_dns_propagated(name, token):
+    txt_records = []
+    try:
+        dns_resolver = dns.resolver.Resolver()
+        dns_resolver.nameservers = [get_authoritative_nameserver(name)]
+        dns_response = dns_resolver.query(name, "TXT")
+        for rdata in dns_response:
+            for txt_record in rdata.strings:
+                txt_records.append(txt_record.decode("utf-8"))
+    except dns.exception.DNSException:
+        metrics.send("has_dns_propagated_fail", "counter", 1)
+        return False
+
+    for txt_record in txt_records:
+        if txt_record == token:
+            metrics.send("has_dns_propagated_success", "counter", 1)
+            return True
+
+    return False
+
+
+def wait_for_dns_change(change_id, account_number=None):
+    fqdn, token = change_id
+    number_of_attempts = 20
+    for attempts in range(0, number_of_attempts):
+        status = _has_dns_propagated(fqdn, token)
+        current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
+        if status:
+            metrics.send("wait_for_dns_change_success", "counter", 1)
+            break
+        time.sleep(10)
+    if not status:
+        # TODO: Delete associated DNS text record here
+        metrics.send("wait_for_dns_change_fail", "counter", 1)
+        sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)})
+        metrics.send(
+            "wait_for_dns_change_error",
+            "counter",
+            1,
+            metric_tags={"fqdn": fqdn, "txt_record": token},
+        )
+    return
+
+
+def get_zones(account_number):
+    path = "/v2/zones/"
+    zones = []
+    for page in _paginate(path, "zones"):
+        for elem in page:
+            zones.append(elem["properties"]["name"][:-1])
+
+    return zones
+
+
+def get_zone_name(domain, account_number):
+    zones = get_zones(account_number)
+
+    zone_name = ""
+
+    for z in zones:
+        if domain.endswith(z):
+            # Find the most specific zone possible for the domain
+            # Ex: If fqdn is a.b.c.com, there is a zone for c.com,
+            # and a zone for b.c.com, we want to use b.c.com.
+            if z.count(".") > zone_name.count("."):
+                zone_name = z
+    if not zone_name:
+        metrics.send("ultradns_no_zone_name", "counter", 1)
+        raise Exception("No UltraDNS zone found for domain: {}".format(domain))
+    return zone_name
+
+
+def create_txt_record(domain, token, account_number):
+    zone_name = get_zone_name(domain, account_number)
+    zone_parts = len(zone_name.split("."))
+    node_name = ".".join(domain.split(".")[:-zone_parts])
+    fqdn = "{0}.{1}".format(node_name, zone_name)
+    path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name)
+    # zone = Zone(zone_name)
+    params = {
+        "ttl": 300,
+        "rdata": [
+            "{}".format(token)
+        ],
+    }
+
+    try:
+        _post(path, params)
+        current_app.logger.debug(
+            "TXT record created: {0}, token: {1}".format(fqdn, token)
+        )
+    except Exception as e:
+        current_app.logger.debug(
+            "Unable to add record. Domain: {}. Token: {}. "
+            "Record already exists: {}".format(domain, token, e),
+            exc_info=True,
+        )
+
+    change_id = (fqdn, token)
+    return change_id
+
+
+def delete_txt_record(change_id, account_number, domain, token):
+    # client = get_ultradns_client()
+    if not domain:
+        current_app.logger.debug("delete_txt_record: No domain passed")
+        return
+
+    zone_name = get_zone_name(domain, account_number)
+    zone_parts = len(zone_name.split("."))
+    node_name = ".".join(domain.split(".")[:-zone_parts])
+    fqdn = "{0}.{1}".format(node_name, zone_name)
+    path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name)
+
+    try:
+        # rrsets = client.get_rrsets_by_type_owner(zone_name, "TXT", node_name)
+        rrsets = _get(path)
+    except Exception as e:
+        metrics.send("delete_txt_record_geterror", "counter", 1)
+        # No Text Records remain or host is not in the zone anymore because all records have been deleted.
+        return
+    try:
+        rrsets["rrSets"][0]["rdata"].remove("{}".format(token))
+    except ValueError:
+        current_app.logger.debug("Token not found")
+        return
+
+    #client.delete_rrset(zone_name, "TXT", node_name)
+    _delete(path)
+
+    if len(rrsets["rrSets"][0]["rdata"]) > 0:
+        #client.create_rrset(zone_name, "TXT", node_name, 300, rrsets["rrSets"][0]["rdata"])
+        params = {
+            "ttl": 300,
+            "rdata": rrsets["rrSets"][0]["rdata"],
+        }
+        _post(path, params)
+
+
+def get_authoritative_nameserver(domain):
+    # return "8.8.8.8"
+    return "156.154.64.154"

From 0b52aa8c59984f0e9579a2b5285e4d276a0bc53c Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 22 Jul 2019 11:47:48 -0700
Subject: [PATCH 03/40] Added Zone class to handle ultradns zones

---
 lemur/plugins/lemur_acme/ultradns.py      | 96 +++++++++++++++++++----
 lemur/plugins/lemur_acme/ultradns_zone.py | 33 ++++++++
 2 files changed, 114 insertions(+), 15 deletions(-)
 create mode 100644 lemur/plugins/lemur_acme/ultradns_zone.py

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index de65b47f..eb595789 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -1,6 +1,7 @@
 import time
 import requests
 import json
+from .ultradns_zone import Zone
 
 import dns
 import dns.exception
@@ -11,10 +12,11 @@ import dns.resolver
 from flask import current_app
 from lemur.extensions import metrics, sentry
 
-use_http = False
-
 
 def get_ultradns_token():
+    # Function to call the UltraDNS Authorization API. Returns the Authorization access_token
+    # which is valid for 1 hour. Each request calls this function and we generate a new token
+    # every time.
     path = "/v2/authorization/token"
     data = {
         "grant_type": "password",
@@ -27,6 +29,8 @@ def get_ultradns_token():
 
 
 def _generate_header():
+    # Function to generate the header for a request. Contains the Authorization access_key
+    # obtained from the get_ultradns_token() function.
     access_token = get_ultradns_token()
     return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"}
 
@@ -34,8 +38,6 @@ def _generate_header():
 def _paginate(path, key):
     limit = 100
     params = {"offset": 0, "limit": 1}
-    # params["offset"] = 0
-    # params["limit"] = 1
     resp = _get(path, params)
     for index in range(0, resp["resultInfo"]["totalCount"], limit):
         params["offset"] = index
@@ -45,6 +47,7 @@ def _paginate(path, key):
 
 
 def _get(path, params=None):
+    # Function to execute a GET request on the given URL (base_uri + path) with given params
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.get(
         "{0}{1}".format(base_uri, path),
@@ -57,6 +60,7 @@ def _get(path, params=None):
 
 
 def _delete(path):
+    # Function to execute a DELETE request on the given URL
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.delete(
         "{0}{1}".format(base_uri, path),
@@ -67,6 +71,7 @@ def _delete(path):
 
 
 def _post(path, params):
+    # Executes a POST request on given URL. Body is sent in JSON format
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.post(
         "{0}{1}".format(base_uri, path),
@@ -78,6 +83,8 @@ def _post(path, params):
 
 
 def _has_dns_propagated(name, token):
+    # Check whether the DNS change made by Lemur have propagated to the public DNS or not.
+    # Invoked by wait_for_dns_change() function
     txt_records = []
     try:
         dns_resolver = dns.resolver.Resolver()
@@ -99,6 +106,7 @@ def _has_dns_propagated(name, token):
 
 
 def wait_for_dns_change(change_id, account_number=None):
+    # Waits and checks if the DNS changes have propagated or not.
     fqdn, token = change_id
     number_of_attempts = 20
     for attempts in range(0, number_of_attempts):
@@ -122,20 +130,26 @@ def wait_for_dns_change(change_id, account_number=None):
 
 
 def get_zones(account_number):
+    # Get zones from the UltraDNS
     path = "/v2/zones/"
     zones = []
     for page in _paginate(path, "zones"):
         for elem in page:
-            zones.append(elem["properties"]["name"][:-1])
+            # UltraDNS zone names end with a "." - Example - lemur.example.com.
+            # We pick out the names minus the "." at the end while returning the list
+            zone = Zone(elem)
+            # TODO : Check for active & Primary
+            # if elem["properties"]["type"] == "PRIMARY" and elem["properties"]["status"] == "ACTIVE":
+            if zone.authoritative_type == "PRIMARY" and zone.status == "ACTIVE":
+                zones.append(zone.name)
 
     return zones
 
 
 def get_zone_name(domain, account_number):
+    # Get the matching zone for the given domain
     zones = get_zones(account_number)
-
     zone_name = ""
-
     for z in zones:
         if domain.endswith(z):
             # Find the most specific zone possible for the domain
@@ -150,12 +164,20 @@ def get_zone_name(domain, account_number):
 
 
 def create_txt_record(domain, token, account_number):
+    # Create a TXT record for the given domain.
+    # The part of the domain that matches with the zone becomes the zone name.
+    # The remainder becomes the owner name (referred to as node name here)
+    # Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we
+    # get a request to create a cert for lemur.example.com
+    # Domain - _acme-challenge.lemur.example.com
+    # Matching zone - example.com
+    # Owner name - _acme-challenge.lemur
+
     zone_name = get_zone_name(domain, account_number)
     zone_parts = len(zone_name.split("."))
     node_name = ".".join(domain.split(".")[:-zone_parts])
     fqdn = "{0}.{1}".format(node_name, zone_name)
     path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name)
-    # zone = Zone(zone_name)
     params = {
         "ttl": 300,
         "rdata": [
@@ -180,7 +202,16 @@ def create_txt_record(domain, token, account_number):
 
 
 def delete_txt_record(change_id, account_number, domain, token):
-    # client = get_ultradns_client()
+    # Delete the TXT record that was created in the create_txt_record() function.
+    # UltraDNS handles records differently compared to Dyn. It creates an RRSet
+    # which is a set of records of the same type and owner. This means
+    # that while deleting the record, we cannot delete any individual record from
+    # the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are
+    # being created for the same domain at the same time, the challenge TXT records
+    # that are created will be added under the same RRSet. If the RRSet had more
+    # than 1 record, then we create a new RRSet on UltraDNS minus the record that
+    # has to be deleted.
+
     if not domain:
         current_app.logger.debug("delete_txt_record: No domain passed")
         return
@@ -188,27 +219,26 @@ def delete_txt_record(change_id, account_number, domain, token):
     zone_name = get_zone_name(domain, account_number)
     zone_parts = len(zone_name.split("."))
     node_name = ".".join(domain.split(".")[:-zone_parts])
-    fqdn = "{0}.{1}".format(node_name, zone_name)
     path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name)
 
     try:
-        # rrsets = client.get_rrsets_by_type_owner(zone_name, "TXT", node_name)
         rrsets = _get(path)
     except Exception as e:
         metrics.send("delete_txt_record_geterror", "counter", 1)
         # No Text Records remain or host is not in the zone anymore because all records have been deleted.
         return
     try:
+        # Remove the record from the RRSet locally
         rrsets["rrSets"][0]["rdata"].remove("{}".format(token))
     except ValueError:
         current_app.logger.debug("Token not found")
         return
 
-    #client.delete_rrset(zone_name, "TXT", node_name)
+    # Delete the RRSet from UltraDNS
     _delete(path)
 
+    # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS
     if len(rrsets["rrSets"][0]["rdata"]) > 0:
-        #client.create_rrset(zone_name, "TXT", node_name, 300, rrsets["rrSets"][0]["rdata"])
         params = {
             "ttl": 300,
             "rdata": rrsets["rrSets"][0]["rdata"],
@@ -216,6 +246,42 @@ def delete_txt_record(change_id, account_number, domain, token):
         _post(path, params)
 
 
+def delete_acme_txt_records(domain):
+
+    if not domain:
+        current_app.logger.debug("delete_acme_txt_records: No domain passed")
+        return
+    acme_challenge_string = "_acme-challenge"
+    if not domain.startswith(acme_challenge_string):
+        current_app.logger.debug(
+            "delete_acme_txt_records: Domain {} doesn't start with string {}. "
+            "Cowardly refusing to delete TXT records".format(
+                domain, acme_challenge_string
+            )
+        )
+        return
+
+    zone_name = get_zone_name(domain)
+    zone_parts = len(zone_name.split("."))
+    node_name = ".".join(domain.split(".")[:-zone_parts])
+    path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name)
+
+    _delete(path)
+
+
 def get_authoritative_nameserver(domain):
-    # return "8.8.8.8"
-    return "156.154.64.154"
+    """
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    REMEMBER TO CHANGE THE RETURN VALUE
+    """
+    return "8.8.8.8"
+    # return "156.154.64.154"
diff --git a/lemur/plugins/lemur_acme/ultradns_zone.py b/lemur/plugins/lemur_acme/ultradns_zone.py
new file mode 100644
index 00000000..c6d90422
--- /dev/null
+++ b/lemur/plugins/lemur_acme/ultradns_zone.py
@@ -0,0 +1,33 @@
+class Zone:
+    """
+    This class implements an Ultra DNS zone.
+    """
+
+    def __init__(self, _data, _client="Client"):
+        self._data = _data
+        self._client = _client
+
+    @property
+    def name(self):
+        """
+        Zone name, has a trailing "." at the end, which we manually remove.
+        """
+        return self._data["properties"]["name"][:-1]
+
+    @property
+    def authoritative_type(self):
+        """
+        Indicates whether the zone is setup as a PRIMARY or SECONDARY
+        """
+        return self._data["properties"]["type"]
+
+    @property
+    def record_count(self):
+        return self._data["properties"]["resourceRecordCount"]
+
+    @property
+    def status(self):
+        """
+        Returns the status of the zone - ACTIVE, SUSPENDED, etc
+        """
+        return self._data["properties"]["status"]

From 51f3b7dde0ff14eebc4d06d8eb09d6fcccd53a2d Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 22 Jul 2019 14:23:40 -0700
Subject: [PATCH 04/40] Added the Record class for UltraDNS

---
 lemur/plugins/lemur_acme/ultradns.py        | 10 +++++---
 lemur/plugins/lemur_acme/ultradns_record.py | 26 +++++++++++++++++++++
 2 files changed, 33 insertions(+), 3 deletions(-)
 create mode 100644 lemur/plugins/lemur_acme/ultradns_record.py

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index eb595789..95adc77a 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -2,6 +2,7 @@ import time
 import requests
 import json
 from .ultradns_zone import Zone
+from .ultradns_record import Record
 
 import dns
 import dns.exception
@@ -223,13 +224,15 @@ def delete_txt_record(change_id, account_number, domain, token):
 
     try:
         rrsets = _get(path)
+        record = Record(rrsets)
     except Exception as e:
         metrics.send("delete_txt_record_geterror", "counter", 1)
         # No Text Records remain or host is not in the zone anymore because all records have been deleted.
         return
     try:
         # Remove the record from the RRSet locally
-        rrsets["rrSets"][0]["rdata"].remove("{}".format(token))
+        # rrsets["rrSets"][0]["rdata"].remove("{}".format(token))
+        record.rdata.remove("{}".format(token))
     except ValueError:
         current_app.logger.debug("Token not found")
         return
@@ -238,10 +241,11 @@ def delete_txt_record(change_id, account_number, domain, token):
     _delete(path)
 
     # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS
-    if len(rrsets["rrSets"][0]["rdata"]) > 0:
+    # if len(rrsets["rrSets"][0]["rdata"]) > 0:
+    if len(record.rdata) > 0:
         params = {
             "ttl": 300,
-            "rdata": rrsets["rrSets"][0]["rdata"],
+            "rdata": record.rdata,
         }
         _post(path, params)
 
diff --git a/lemur/plugins/lemur_acme/ultradns_record.py b/lemur/plugins/lemur_acme/ultradns_record.py
new file mode 100644
index 00000000..9ec8d4d8
--- /dev/null
+++ b/lemur/plugins/lemur_acme/ultradns_record.py
@@ -0,0 +1,26 @@
+class Record:
+    """
+    This class implements an Ultra DNS record.
+    Accepts the response from the API call as the argument.
+    """
+
+    def __init__(self, _data):
+        # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response.
+        # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response.
+        self._data = _data["rrSets"][0]
+
+    @property
+    def name(self):
+        return self._data["ownerName"]
+
+    @property
+    def rrtype(self):
+        return self._data["rrtype"]
+
+    @property
+    def rdata(self):
+        return self._data["rdata"]
+
+    @property
+    def ttl(self):
+        return self._data["ttl"]

From 252410c6e9529a10926a0f6b23768a1f322c163e Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 22 Jul 2019 16:00:20 -0700
Subject: [PATCH 05/40] Updated TTL from 300 to 5

---
 lemur/plugins/lemur_acme/ultradns.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index 95adc77a..d6ad64af 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -180,7 +180,7 @@ def create_txt_record(domain, token, account_number):
     fqdn = "{0}.{1}".format(node_name, zone_name)
     path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name)
     params = {
-        "ttl": 300,
+        "ttl": 5,
         "rdata": [
             "{}".format(token)
         ],
@@ -244,7 +244,7 @@ def delete_txt_record(change_id, account_number, domain, token):
     # if len(rrsets["rrSets"][0]["rdata"]) > 0:
     if len(record.rdata) > 0:
         params = {
-            "ttl": 300,
+            "ttl": 5,
             "rdata": record.rdata,
         }
         _post(path, params)

From e993194b4f30262a86880a86097d968f7cf9fc2e Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 29 Jul 2019 14:59:28 -0700
Subject: [PATCH 06/40] Check ultraDNS authoritative server first. Upon
 success, check Googles DNS server.

---
 lemur/plugins/lemur_acme/ultradns.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index d6ad64af..1a520f2e 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -83,13 +83,15 @@ def _post(path, params):
     resp.raise_for_status()
 
 
-def _has_dns_propagated(name, token):
+def _has_dns_propagated(name, token, domain="8.8.8.8"):
     # Check whether the DNS change made by Lemur have propagated to the public DNS or not.
     # Invoked by wait_for_dns_change() function
     txt_records = []
     try:
         dns_resolver = dns.resolver.Resolver()
-        dns_resolver.nameservers = [get_authoritative_nameserver(name)]
+        # dns_resolver.nameservers = [get_authoritative_nameserver(name)]
+        # dns_resolver.nameservers = ["156.154.64.154"]
+        dns_resolver.nameservers = [domain]
         dns_response = dns_resolver.query(name, "TXT")
         for rdata in dns_response:
             for txt_record in rdata.strings:
@@ -111,12 +113,21 @@ def wait_for_dns_change(change_id, account_number=None):
     fqdn, token = change_id
     number_of_attempts = 20
     for attempts in range(0, number_of_attempts):
-        status = _has_dns_propagated(fqdn, token)
+        status = _has_dns_propagated(fqdn, token, "156.154.64.154")
         current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
         if status:
-            metrics.send("wait_for_dns_change_success", "counter", 1)
+            # metrics.send("wait_for_dns_change_success", "counter", 1)
+            time.sleep(10)
             break
         time.sleep(10)
+    if status:
+        for attempts in range(0, number_of_attempts):
+            status = _has_dns_propagated(fqdn, token, "8.8.8.8")
+            current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
+            if status:
+                metrics.send("wait_for_dns_change_success", "counter", 1)
+                break
+            time.sleep(10)
     if not status:
         # TODO: Delete associated DNS text record here
         metrics.send("wait_for_dns_change_fail", "counter", 1)
@@ -132,7 +143,7 @@ def wait_for_dns_change(change_id, account_number=None):
 
 def get_zones(account_number):
     # Get zones from the UltraDNS
-    path = "/v2/zones/"
+    path = "/v2/zones"
     zones = []
     for page in _paginate(path, "zones"):
         for elem in page:
@@ -287,5 +298,5 @@ def get_authoritative_nameserver(domain):
     REMEMBER TO CHANGE THE RETURN VALUE
     REMEMBER TO CHANGE THE RETURN VALUE
     """
-    return "8.8.8.8"
-    # return "156.154.64.154"
+    # return "8.8.8.8"
+    return "156.154.64.154"

From 3ad791e1ec634646edc78aaa1b4e4d40bbb67936 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 29 Jul 2019 18:01:28 -0700
Subject: [PATCH 07/40] Dynamically obtain the authoritative nameserver for the
 domain

---
 lemur/plugins/lemur_acme/ultradns.py | 72 ++++++++++++++++++----------
 1 file changed, 47 insertions(+), 25 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index 1a520f2e..c7f853d0 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -83,14 +83,12 @@ def _post(path, params):
     resp.raise_for_status()
 
 
-def _has_dns_propagated(name, token, domain="8.8.8.8"):
+def _has_dns_propagated(name, token, domain):
     # Check whether the DNS change made by Lemur have propagated to the public DNS or not.
     # Invoked by wait_for_dns_change() function
     txt_records = []
     try:
         dns_resolver = dns.resolver.Resolver()
-        # dns_resolver.nameservers = [get_authoritative_nameserver(name)]
-        # dns_resolver.nameservers = ["156.154.64.154"]
         dns_resolver.nameservers = [domain]
         dns_response = dns_resolver.query(name, "TXT")
         for rdata in dns_response:
@@ -110,19 +108,21 @@ def _has_dns_propagated(name, token, domain="8.8.8.8"):
 
 def wait_for_dns_change(change_id, account_number=None):
     # Waits and checks if the DNS changes have propagated or not.
+    # First check the domains authoritative server. Once this succeeds,
+    # we ask a public DNS server (Google <8.8.8.8> in our case).
     fqdn, token = change_id
     number_of_attempts = 20
+    nameserver = get_authoritative_nameserver(fqdn)
     for attempts in range(0, number_of_attempts):
-        status = _has_dns_propagated(fqdn, token, "156.154.64.154")
+        status = _has_dns_propagated(fqdn, token, nameserver)
         current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
         if status:
-            # metrics.send("wait_for_dns_change_success", "counter", 1)
             time.sleep(10)
             break
         time.sleep(10)
     if status:
         for attempts in range(0, number_of_attempts):
-            status = _has_dns_propagated(fqdn, token, "8.8.8.8")
+            status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver())
             current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
             if status:
                 metrics.send("wait_for_dns_change_success", "counter", 1)
@@ -150,8 +150,6 @@ def get_zones(account_number):
             # UltraDNS zone names end with a "." - Example - lemur.example.com.
             # We pick out the names minus the "." at the end while returning the list
             zone = Zone(elem)
-            # TODO : Check for active & Primary
-            # if elem["properties"]["type"] == "PRIMARY" and elem["properties"]["status"] == "ACTIVE":
             if zone.authoritative_type == "PRIMARY" and zone.status == "ACTIVE":
                 zones.append(zone.name)
 
@@ -242,7 +240,6 @@ def delete_txt_record(change_id, account_number, domain, token):
         return
     try:
         # Remove the record from the RRSet locally
-        # rrsets["rrSets"][0]["rdata"].remove("{}".format(token))
         record.rdata.remove("{}".format(token))
     except ValueError:
         current_app.logger.debug("Token not found")
@@ -252,7 +249,6 @@ def delete_txt_record(change_id, account_number, domain, token):
     _delete(path)
 
     # Check if the RRSet has more records. If yes, add the modified RRSet back to UltraDNS
-    # if len(rrsets["rrSets"][0]["rdata"]) > 0:
     if len(record.rdata) > 0:
         params = {
             "ttl": 5,
@@ -285,18 +281,44 @@ def delete_acme_txt_records(domain):
 
 
 def get_authoritative_nameserver(domain):
-    """
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    REMEMBER TO CHANGE THE RETURN VALUE
-    """
-    # return "8.8.8.8"
-    return "156.154.64.154"
+    n = dns.name.from_text(domain)
+
+    depth = 2
+    default = dns.resolver.get_default_resolver()
+    nameserver = default.nameservers[0]
+
+    last = False
+    while not last:
+        s = n.split(depth)
+
+        last = s[0].to_unicode() == u"@"
+        sub = s[1]
+
+        query = dns.message.make_query(sub, dns.rdatatype.NS)
+        response = dns.query.udp(query, nameserver)
+
+        rcode = response.rcode()
+        if rcode != dns.rcode.NOERROR:
+            metrics.send("get_authoritative_nameserver_error", "counter", 1)
+            if rcode == dns.rcode.NXDOMAIN:
+                raise Exception("%s does not exist." % sub)
+            else:
+                raise Exception("Error %s" % dns.rcode.to_text(rcode))
+
+        if len(response.authority) > 0:
+            rrset = response.authority[0]
+        else:
+            rrset = response.answer[0]
+
+        rr = rrset[0]
+        if rr.rdtype != dns.rdatatype.SOA:
+            authority = rr.target
+            nameserver = default.query(authority).rrset[0].to_text()
+
+        depth += 1
+
+    return nameserver
+
+
+def get_public_authoritative_nameserver():
+    return "8.8.8.8"

From 3d48b422b5ee3a23e6df83b5ca9208ffebb4621f Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Tue, 30 Jul 2019 11:39:35 -0700
Subject: [PATCH 08/40] Removed TODO

---
 lemur/plugins/lemur_acme/ultradns.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index c7f853d0..c43840e4 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -129,7 +129,6 @@ def wait_for_dns_change(change_id, account_number=None):
                 break
             time.sleep(10)
     if not status:
-        # TODO: Delete associated DNS text record here
         metrics.send("wait_for_dns_change_fail", "counter", 1)
         sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)})
         metrics.send(

From 44bc562e8b2e2339d420af0a42647fa4efe91e0c Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kush.bavishi@gmail.com>
Date: Tue, 30 Jul 2019 13:08:16 -0700
Subject: [PATCH 09/40] Update ultradns.py

Minor logging changes in wait_for_dns_change
---
 lemur/plugins/lemur_acme/ultradns.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index c43840e4..24b98e66 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -115,7 +115,7 @@ def wait_for_dns_change(change_id, account_number=None):
     nameserver = get_authoritative_nameserver(fqdn)
     for attempts in range(0, number_of_attempts):
         status = _has_dns_propagated(fqdn, token, nameserver)
-        current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
+        current_app.logger.debug("Record status on ultraDNS authoritative server for fqdn: {}: {}".format(fqdn, status))
         if status:
             time.sleep(10)
             break
@@ -123,7 +123,7 @@ def wait_for_dns_change(change_id, account_number=None):
     if status:
         for attempts in range(0, number_of_attempts):
             status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver())
-            current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
+            current_app.logger.debug("Record status on Google DNS for fqdn: {}: {}".format(fqdn, status))
             if status:
                 metrics.send("wait_for_dns_change_success", "counter", 1)
                 break

From 3ba7fdbd494401331e9da084c1e8b4f0b147e559 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 31 Jul 2019 11:11:39 -0700
Subject: [PATCH 10/40] Updated logger to log a dictionary instead of a string

---
 lemur/plugins/lemur_acme/ultradns.py | 80 +++++++++++++++++++++-------
 1 file changed, 61 insertions(+), 19 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index 24b98e66..d3b68afc 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -1,6 +1,7 @@
 import time
 import requests
 import json
+import sys
 from .ultradns_zone import Zone
 from .ultradns_record import Record
 
@@ -115,7 +116,14 @@ def wait_for_dns_change(change_id, account_number=None):
     nameserver = get_authoritative_nameserver(fqdn)
     for attempts in range(0, number_of_attempts):
         status = _has_dns_propagated(fqdn, token, nameserver)
-        current_app.logger.debug("Record status on ultraDNS authoritative server for fqdn: {}: {}".format(fqdn, status))
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "fqdn": fqdn,
+            "status": status,
+            "message": "Record status on ultraDNS authoritative server"
+        }
+        current_app.logger.debug(log_data)
         if status:
             time.sleep(10)
             break
@@ -123,7 +131,14 @@ def wait_for_dns_change(change_id, account_number=None):
     if status:
         for attempts in range(0, number_of_attempts):
             status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver())
-            current_app.logger.debug("Record status on Google DNS for fqdn: {}: {}".format(fqdn, status))
+            function = sys._getframe().f_code.co_name
+            log_data = {
+                "function": function,
+                "fqdn": fqdn,
+                "status": status,
+                "message": "Record status on Public DNS"
+            }
+            current_app.logger.debug(log_data)
             if status:
                 metrics.send("wait_for_dns_change_success", "counter", 1)
                 break
@@ -196,15 +211,24 @@ def create_txt_record(domain, token, account_number):
 
     try:
         _post(path, params)
-        current_app.logger.debug(
-            "TXT record created: {0}, token: {1}".format(fqdn, token)
-        )
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "fqdn": fqdn,
+            "token": token,
+            "message": "TXT record created"
+        }
+        current_app.logger.debug(log_data)
     except Exception as e:
-        current_app.logger.debug(
-            "Unable to add record. Domain: {}. Token: {}. "
-            "Record already exists: {}".format(domain, token, e),
-            exc_info=True,
-        )
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "domain": domain,
+            "token": token,
+            "Exception": e,
+            "message": "Unable to add record. Record already exists."
+        }
+        current_app.logger.debug(log_data)
 
     change_id = (fqdn, token)
     return change_id
@@ -222,7 +246,12 @@ def delete_txt_record(change_id, account_number, domain, token):
     # has to be deleted.
 
     if not domain:
-        current_app.logger.debug("delete_txt_record: No domain passed")
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "message": "No domain passed"
+        }
+        current_app.logger.debug(log_data)
         return
 
     zone_name = get_zone_name(domain, account_number)
@@ -241,7 +270,13 @@ def delete_txt_record(change_id, account_number, domain, token):
         # Remove the record from the RRSet locally
         record.rdata.remove("{}".format(token))
     except ValueError:
-        current_app.logger.debug("Token not found")
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "token": token,
+            "message": "Token not found"
+        }
+        current_app.logger.debug(log_data)
         return
 
     # Delete the RRSet from UltraDNS
@@ -259,16 +294,23 @@ def delete_txt_record(change_id, account_number, domain, token):
 def delete_acme_txt_records(domain):
 
     if not domain:
-        current_app.logger.debug("delete_acme_txt_records: No domain passed")
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "message": "No domain passed"
+        }
+        current_app.logger.debug(log_data)
         return
     acme_challenge_string = "_acme-challenge"
     if not domain.startswith(acme_challenge_string):
-        current_app.logger.debug(
-            "delete_acme_txt_records: Domain {} doesn't start with string {}. "
-            "Cowardly refusing to delete TXT records".format(
-                domain, acme_challenge_string
-            )
-        )
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "domain": domain,
+            "acme_challenge_string": acme_challenge_string,
+            "message": "Domain does not start with the acme challenge string"
+        }
+        current_app.logger.debug(log_data)
         return
 
     zone_name = get_zone_name(domain)

From 11cd09513154a2d505755ccc266d33b2e6d4127a Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 31 Jul 2019 11:12:28 -0700
Subject: [PATCH 11/40] Reduced the number of calls to
 get_public_authoritative_nameserver by using a variable

---
 lemur/plugins/lemur_acme/ultradns.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index d3b68afc..d516be1a 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -129,8 +129,9 @@ def wait_for_dns_change(change_id, account_number=None):
             break
         time.sleep(10)
     if status:
+        nameserver = get_public_authoritative_nameserver()
         for attempts in range(0, number_of_attempts):
-            status = _has_dns_propagated(fqdn, token, get_public_authoritative_nameserver())
+            status = _has_dns_propagated(fqdn, token, nameserver)
             function = sys._getframe().f_code.co_name
             log_data = {
                 "function": function,

From 503df999fa1d569475898290f50bc9ce3d1ac2ba Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 31 Jul 2019 11:32:04 -0700
Subject: [PATCH 12/40] Updated metrics.send to send function named, followed
 by status, separated by a period

---
 lemur/plugins/lemur_acme/ultradns.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index d516be1a..0229a59e 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -96,12 +96,14 @@ def _has_dns_propagated(name, token, domain):
             for txt_record in rdata.strings:
                 txt_records.append(txt_record.decode("utf-8"))
     except dns.exception.DNSException:
-        metrics.send("has_dns_propagated_fail", "counter", 1)
+        function = sys._getframe().f_code.co_name
+        metrics.send("{}.fail".format(function), "counter", 1)
         return False
 
     for txt_record in txt_records:
         if txt_record == token:
-            metrics.send("has_dns_propagated_success", "counter", 1)
+            function = sys._getframe().f_code.co_name
+            metrics.send("{}.success".format(function), "counter", 1)
             return True
 
     return False
@@ -132,7 +134,6 @@ def wait_for_dns_change(change_id, account_number=None):
         nameserver = get_public_authoritative_nameserver()
         for attempts in range(0, number_of_attempts):
             status = _has_dns_propagated(fqdn, token, nameserver)
-            function = sys._getframe().f_code.co_name
             log_data = {
                 "function": function,
                 "fqdn": fqdn,
@@ -141,18 +142,12 @@ def wait_for_dns_change(change_id, account_number=None):
             }
             current_app.logger.debug(log_data)
             if status:
-                metrics.send("wait_for_dns_change_success", "counter", 1)
+                metrics.send("{}.success".format(function), "counter", 1)
                 break
             time.sleep(10)
     if not status:
-        metrics.send("wait_for_dns_change_fail", "counter", 1)
+        metrics.send("{}.fail".format, "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token})
         sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)})
-        metrics.send(
-            "wait_for_dns_change_error",
-            "counter",
-            1,
-            metric_tags={"fqdn": fqdn, "txt_record": token},
-        )
     return
 
 
@@ -183,7 +178,8 @@ def get_zone_name(domain, account_number):
             if z.count(".") > zone_name.count("."):
                 zone_name = z
     if not zone_name:
-        metrics.send("ultradns_no_zone_name", "counter", 1)
+        function = sys._getframe().f_code.co_name
+        metrics.send("{}.fail".format(function), "counter", 1)
         raise Exception("No UltraDNS zone found for domain: {}".format(domain))
     return zone_name
 
@@ -264,7 +260,8 @@ def delete_txt_record(change_id, account_number, domain, token):
         rrsets = _get(path)
         record = Record(rrsets)
     except Exception as e:
-        metrics.send("delete_txt_record_geterror", "counter", 1)
+        function = sys._getframe().f_code.co_name
+        metrics.send("{}.geterror".format(function), "counter", 1)
         # No Text Records remain or host is not in the zone anymore because all records have been deleted.
         return
     try:
@@ -341,7 +338,8 @@ def get_authoritative_nameserver(domain):
 
         rcode = response.rcode()
         if rcode != dns.rcode.NOERROR:
-            metrics.send("get_authoritative_nameserver_error", "counter", 1)
+            function = sys._getframe().f_code.co_name
+            metrics.send("{}.error".format(function), "counter", 1)
             if rcode == dns.rcode.NXDOMAIN:
                 raise Exception("%s does not exist." % sub)
             else:

From fe075dc9f56985d2b81a75d7b4db12d87e3f6023 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 31 Jul 2019 12:00:31 -0700
Subject: [PATCH 13/40] Changed function comments to doc strings.

---
 lemur/plugins/lemur_acme/ultradns.py | 83 +++++++++++++++++-----------
 1 file changed, 51 insertions(+), 32 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index 0229a59e..1c3aa961 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -16,9 +16,12 @@ from lemur.extensions import metrics, sentry
 
 
 def get_ultradns_token():
-    # Function to call the UltraDNS Authorization API. Returns the Authorization access_token
-    # which is valid for 1 hour. Each request calls this function and we generate a new token
-    # every time.
+    """
+    Function to call the UltraDNS Authorization API.
+
+    Returns the Authorization access_token which is valid for 1 hour.
+    Each request calls this function and we generate a new token every time.
+    """
     path = "/v2/authorization/token"
     data = {
         "grant_type": "password",
@@ -31,8 +34,11 @@ def get_ultradns_token():
 
 
 def _generate_header():
-    # Function to generate the header for a request. Contains the Authorization access_key
-    # obtained from the get_ultradns_token() function.
+    """
+    Function to generate the header for a request.
+
+    Contains the Authorization access_key obtained from the get_ultradns_token() function.
+    """
     access_token = get_ultradns_token()
     return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"}
 
@@ -49,7 +55,7 @@ def _paginate(path, key):
 
 
 def _get(path, params=None):
-    # Function to execute a GET request on the given URL (base_uri + path) with given params
+    """Function to execute a GET request on the given URL (base_uri + path) with given params"""
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.get(
         "{0}{1}".format(base_uri, path),
@@ -62,7 +68,7 @@ def _get(path, params=None):
 
 
 def _delete(path):
-    # Function to execute a DELETE request on the given URL
+    """Function to execute a DELETE request on the given URL"""
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.delete(
         "{0}{1}".format(base_uri, path),
@@ -73,7 +79,7 @@ def _delete(path):
 
 
 def _post(path, params):
-    # Executes a POST request on given URL. Body is sent in JSON format
+    """Executes a POST request on given URL. Body is sent in JSON format"""
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.post(
         "{0}{1}".format(base_uri, path),
@@ -85,8 +91,11 @@ def _post(path, params):
 
 
 def _has_dns_propagated(name, token, domain):
-    # Check whether the DNS change made by Lemur have propagated to the public DNS or not.
-    # Invoked by wait_for_dns_change() function
+    """
+    Check whether the DNS change made by Lemur have propagated to the public DNS or not.
+
+    Invoked by wait_for_dns_change() function
+    """
     txt_records = []
     try:
         dns_resolver = dns.resolver.Resolver()
@@ -110,9 +119,12 @@ def _has_dns_propagated(name, token, domain):
 
 
 def wait_for_dns_change(change_id, account_number=None):
-    # Waits and checks if the DNS changes have propagated or not.
-    # First check the domains authoritative server. Once this succeeds,
-    # we ask a public DNS server (Google <8.8.8.8> in our case).
+    """
+    Waits and checks if the DNS changes have propagated or not.
+
+    First check the domains authoritative server. Once this succeeds,
+    we ask a public DNS server (Google <8.8.8.8> in our case).
+    """
     fqdn, token = change_id
     number_of_attempts = 20
     nameserver = get_authoritative_nameserver(fqdn)
@@ -152,7 +164,7 @@ def wait_for_dns_change(change_id, account_number=None):
 
 
 def get_zones(account_number):
-    # Get zones from the UltraDNS
+    """Get zones from the UltraDNS"""
     path = "/v2/zones"
     zones = []
     for page in _paginate(path, "zones"):
@@ -167,7 +179,7 @@ def get_zones(account_number):
 
 
 def get_zone_name(domain, account_number):
-    # Get the matching zone for the given domain
+    """Get the matching zone for the given domain"""
     zones = get_zones(account_number)
     zone_name = ""
     for z in zones:
@@ -185,14 +197,17 @@ def get_zone_name(domain, account_number):
 
 
 def create_txt_record(domain, token, account_number):
-    # Create a TXT record for the given domain.
-    # The part of the domain that matches with the zone becomes the zone name.
-    # The remainder becomes the owner name (referred to as node name here)
-    # Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we
-    # get a request to create a cert for lemur.example.com
-    # Domain - _acme-challenge.lemur.example.com
-    # Matching zone - example.com
-    # Owner name - _acme-challenge.lemur
+    """
+    Create a TXT record for the given domain.
+
+    The part of the domain that matches with the zone becomes the zone name.
+    The remainder becomes the owner name (referred to as node name here)
+    Example: Let's say we have a zone named "exmaple.com" in UltraDNS and we
+    get a request to create a cert for lemur.example.com
+    Domain - _acme-challenge.lemur.example.com
+    Matching zone - example.com
+    Owner name - _acme-challenge.lemur
+    """
 
     zone_name = get_zone_name(domain, account_number)
     zone_parts = len(zone_name.split("."))
@@ -232,15 +247,18 @@ def create_txt_record(domain, token, account_number):
 
 
 def delete_txt_record(change_id, account_number, domain, token):
-    # Delete the TXT record that was created in the create_txt_record() function.
-    # UltraDNS handles records differently compared to Dyn. It creates an RRSet
-    # which is a set of records of the same type and owner. This means
-    # that while deleting the record, we cannot delete any individual record from
-    # the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are
-    # being created for the same domain at the same time, the challenge TXT records
-    # that are created will be added under the same RRSet. If the RRSet had more
-    # than 1 record, then we create a new RRSet on UltraDNS minus the record that
-    # has to be deleted.
+    """
+    Delete the TXT record that was created in the create_txt_record() function.
+
+    UltraDNS handles records differently compared to Dyn. It creates an RRSet
+    which is a set of records of the same type and owner. This means
+    that while deleting the record, we cannot delete any individual record from
+    the RRSet. Instead, we have to delete the entire RRSet. If multiple certs are
+    being created for the same domain at the same time, the challenge TXT records
+    that are created will be added under the same RRSet. If the RRSet had more
+    than 1 record, then we create a new RRSet on UltraDNS minus the record that
+    has to be deleted.
+    """
 
     if not domain:
         function = sys._getframe().f_code.co_name
@@ -320,6 +338,7 @@ def delete_acme_txt_records(domain):
 
 
 def get_authoritative_nameserver(domain):
+    """Get the authoritative nameserver for the given domain"""
     n = dns.name.from_text(domain)
 
     depth = 2

From 5a401b2d87526f3ef88405a21c0978a15c6d1895 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 31 Jul 2019 12:04:42 -0700
Subject: [PATCH 14/40] Added the Zone class and Record class to ultradns.py
 and removed the respective files

---
 lemur/plugins/lemur_acme/ultradns.py        | 66 ++++++++++++++++++++-
 lemur/plugins/lemur_acme/ultradns_record.py | 26 --------
 lemur/plugins/lemur_acme/ultradns_zone.py   | 33 -----------
 3 files changed, 64 insertions(+), 61 deletions(-)
 delete mode 100644 lemur/plugins/lemur_acme/ultradns_record.py
 delete mode 100644 lemur/plugins/lemur_acme/ultradns_zone.py

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index 1c3aa961..40661740 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -2,8 +2,6 @@ import time
 import requests
 import json
 import sys
-from .ultradns_zone import Zone
-from .ultradns_record import Record
 
 import dns
 import dns.exception
@@ -15,6 +13,70 @@ from flask import current_app
 from lemur.extensions import metrics, sentry
 
 
+class Record:
+    """
+    This class implements an Ultra DNS record.
+
+    Accepts the response from the API call as the argument.
+    """
+
+    def __init__(self, _data):
+        # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response.
+        # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response.
+        self._data = _data["rrSets"][0]
+
+    @property
+    def name(self):
+        return self._data["ownerName"]
+
+    @property
+    def rrtype(self):
+        return self._data["rrtype"]
+
+    @property
+    def rdata(self):
+        return self._data["rdata"]
+
+    @property
+    def ttl(self):
+        return self._data["ttl"]
+
+
+class Zone:
+    """
+    This class implements an Ultra DNS zone.
+    """
+
+    def __init__(self, _data, _client="Client"):
+        self._data = _data
+        self._client = _client
+
+    @property
+    def name(self):
+        """
+        Zone name, has a trailing "." at the end, which we manually remove.
+        """
+        return self._data["properties"]["name"][:-1]
+
+    @property
+    def authoritative_type(self):
+        """
+        Indicates whether the zone is setup as a PRIMARY or SECONDARY
+        """
+        return self._data["properties"]["type"]
+
+    @property
+    def record_count(self):
+        return self._data["properties"]["resourceRecordCount"]
+
+    @property
+    def status(self):
+        """
+        Returns the status of the zone - ACTIVE, SUSPENDED, etc
+        """
+        return self._data["properties"]["status"]
+
+
 def get_ultradns_token():
     """
     Function to call the UltraDNS Authorization API.
diff --git a/lemur/plugins/lemur_acme/ultradns_record.py b/lemur/plugins/lemur_acme/ultradns_record.py
deleted file mode 100644
index 9ec8d4d8..00000000
--- a/lemur/plugins/lemur_acme/ultradns_record.py
+++ /dev/null
@@ -1,26 +0,0 @@
-class Record:
-    """
-    This class implements an Ultra DNS record.
-    Accepts the response from the API call as the argument.
-    """
-
-    def __init__(self, _data):
-        # Since we are dealing with only TXT records for Lemur, we expect only 1 RRSet in the response.
-        # Thus we default to picking up the first entry (_data["rrsets"][0]) from the response.
-        self._data = _data["rrSets"][0]
-
-    @property
-    def name(self):
-        return self._data["ownerName"]
-
-    @property
-    def rrtype(self):
-        return self._data["rrtype"]
-
-    @property
-    def rdata(self):
-        return self._data["rdata"]
-
-    @property
-    def ttl(self):
-        return self._data["ttl"]
diff --git a/lemur/plugins/lemur_acme/ultradns_zone.py b/lemur/plugins/lemur_acme/ultradns_zone.py
deleted file mode 100644
index c6d90422..00000000
--- a/lemur/plugins/lemur_acme/ultradns_zone.py
+++ /dev/null
@@ -1,33 +0,0 @@
-class Zone:
-    """
-    This class implements an Ultra DNS zone.
-    """
-
-    def __init__(self, _data, _client="Client"):
-        self._data = _data
-        self._client = _client
-
-    @property
-    def name(self):
-        """
-        Zone name, has a trailing "." at the end, which we manually remove.
-        """
-        return self._data["properties"]["name"][:-1]
-
-    @property
-    def authoritative_type(self):
-        """
-        Indicates whether the zone is setup as a PRIMARY or SECONDARY
-        """
-        return self._data["properties"]["type"]
-
-    @property
-    def record_count(self):
-        return self._data["properties"]["resourceRecordCount"]
-
-    @property
-    def status(self):
-        """
-        Returns the status of the zone - ACTIVE, SUSPENDED, etc
-        """
-        return self._data["properties"]["status"]

From 2903799b85e3c2db11d9a9b46a2d05cd850b266b Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 31 Jul 2019 14:19:49 -0700
Subject: [PATCH 15/40] Changed string formatting from "{}".format() to f"{}"
 for consistency

---
 lemur/plugins/lemur_acme/ultradns.py | 38 ++++++++++++++--------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/lemur/plugins/lemur_acme/ultradns.py b/lemur/plugins/lemur_acme/ultradns.py
index 40661740..dcf3e3c6 100644
--- a/lemur/plugins/lemur_acme/ultradns.py
+++ b/lemur/plugins/lemur_acme/ultradns.py
@@ -91,7 +91,7 @@ def get_ultradns_token():
         "password": current_app.config.get("ACME_ULTRADNS_PASSWORD", ""),
     }
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
-    resp = requests.post("{0}{1}".format(base_uri, path), data=data, verify=True)
+    resp = requests.post(f"{base_uri}{path}", data=data, verify=True)
     return resp.json()["access_token"]
 
 
@@ -102,7 +102,7 @@ def _generate_header():
     Contains the Authorization access_key obtained from the get_ultradns_token() function.
     """
     access_token = get_ultradns_token()
-    return {"Authorization": "Bearer {}".format(access_token), "Content-Type": "application/json"}
+    return {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"}
 
 
 def _paginate(path, key):
@@ -120,7 +120,7 @@ def _get(path, params=None):
     """Function to execute a GET request on the given URL (base_uri + path) with given params"""
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.get(
-        "{0}{1}".format(base_uri, path),
+        f"{base_uri}{path}",
         headers=_generate_header(),
         params=params,
         verify=True,
@@ -133,7 +133,7 @@ def _delete(path):
     """Function to execute a DELETE request on the given URL"""
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.delete(
-        "{0}{1}".format(base_uri, path),
+        f"{base_uri}{path}",
         headers=_generate_header(),
         verify=True,
     )
@@ -144,7 +144,7 @@ def _post(path, params):
     """Executes a POST request on given URL. Body is sent in JSON format"""
     base_uri = current_app.config.get("ACME_ULTRADNS_DOMAIN", "")
     resp = requests.post(
-        "{0}{1}".format(base_uri, path),
+        f"{base_uri}{path}",
         headers=_generate_header(),
         data=json.dumps(params),
         verify=True,
@@ -168,13 +168,13 @@ def _has_dns_propagated(name, token, domain):
                 txt_records.append(txt_record.decode("utf-8"))
     except dns.exception.DNSException:
         function = sys._getframe().f_code.co_name
-        metrics.send("{}.fail".format(function), "counter", 1)
+        metrics.send(f"{function}.fail", "counter", 1)
         return False
 
     for txt_record in txt_records:
         if txt_record == token:
             function = sys._getframe().f_code.co_name
-            metrics.send("{}.success".format(function), "counter", 1)
+            metrics.send(f"{function}.success", "counter", 1)
             return True
 
     return False
@@ -216,11 +216,11 @@ def wait_for_dns_change(change_id, account_number=None):
             }
             current_app.logger.debug(log_data)
             if status:
-                metrics.send("{}.success".format(function), "counter", 1)
+                metrics.send(f"{function}.success", "counter", 1)
                 break
             time.sleep(10)
     if not status:
-        metrics.send("{}.fail".format, "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token})
+        metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": fqdn, "txt_record": token})
         sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)})
     return
 
@@ -253,8 +253,8 @@ def get_zone_name(domain, account_number):
                 zone_name = z
     if not zone_name:
         function = sys._getframe().f_code.co_name
-        metrics.send("{}.fail".format(function), "counter", 1)
-        raise Exception("No UltraDNS zone found for domain: {}".format(domain))
+        metrics.send(f"{function}.fail", "counter", 1)
+        raise Exception(f"No UltraDNS zone found for domain: {domain}")
     return zone_name
 
 
@@ -274,12 +274,12 @@ def create_txt_record(domain, token, account_number):
     zone_name = get_zone_name(domain, account_number)
     zone_parts = len(zone_name.split("."))
     node_name = ".".join(domain.split(".")[:-zone_parts])
-    fqdn = "{0}.{1}".format(node_name, zone_name)
-    path = "/v2/zones/{0}/rrsets/TXT/{1}".format(zone_name, node_name)
+    fqdn = f"{node_name}.{zone_name}"
+    path = f"/v2/zones/{zone_name}/rrsets/TXT/{node_name}"
     params = {
         "ttl": 5,
         "rdata": [
-            "{}".format(token)
+            f"{token}"
         ],
     }
 
@@ -334,19 +334,19 @@ def delete_txt_record(change_id, account_number, domain, token):
     zone_name = get_zone_name(domain, account_number)
     zone_parts = len(zone_name.split("."))
     node_name = ".".join(domain.split(".")[:-zone_parts])
-    path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name)
+    path = f"/v2/zones/{zone_name}/rrsets/16/{node_name}"
 
     try:
         rrsets = _get(path)
         record = Record(rrsets)
     except Exception as e:
         function = sys._getframe().f_code.co_name
-        metrics.send("{}.geterror".format(function), "counter", 1)
+        metrics.send(f"{function}.geterror", "counter", 1)
         # No Text Records remain or host is not in the zone anymore because all records have been deleted.
         return
     try:
         # Remove the record from the RRSet locally
-        record.rdata.remove("{}".format(token))
+        record.rdata.remove(f"{token}")
     except ValueError:
         function = sys._getframe().f_code.co_name
         log_data = {
@@ -394,7 +394,7 @@ def delete_acme_txt_records(domain):
     zone_name = get_zone_name(domain)
     zone_parts = len(zone_name.split("."))
     node_name = ".".join(domain.split(".")[:-zone_parts])
-    path = "/v2/zones/{}/rrsets/16/{}".format(zone_name, node_name)
+    path = f"/v2/zones/{zone_name}/rrsets/16/{node_name}"
 
     _delete(path)
 
@@ -420,7 +420,7 @@ def get_authoritative_nameserver(domain):
         rcode = response.rcode()
         if rcode != dns.rcode.NOERROR:
             function = sys._getframe().f_code.co_name
-            metrics.send("{}.error".format(function), "counter", 1)
+            metrics.send(f"{function}.error", "counter", 1)
             if rcode == dns.rcode.NXDOMAIN:
                 raise Exception("%s does not exist." % sub)
             else:

From a7c2b970b0b04fc3a1b15b1d7f0acc1ca1bc8a6b Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Mon, 5 Aug 2019 13:59:59 -0700
Subject: [PATCH 16/40] Unit testing Part 1

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 63 ++++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 3bf1d05c..29c9534e 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -1,8 +1,10 @@
+import json
 import unittest
+from requests.models import Response
 
 from mock import MagicMock, Mock, patch
 
-from lemur.plugins.lemur_acme import plugin
+from lemur.plugins.lemur_acme import plugin, ultradns
 
 
 class TestAcme(unittest.TestCase):
@@ -360,3 +362,62 @@ class TestAcme(unittest.TestCase):
         mock_request_certificate.return_value = ("pem_certificate", "chain")
         result = provider.create_certificate(csr, issuer_options)
         assert result
+
+    @patch("lemur.plugins.lemur_acme.ultradns.requests")
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    def test_get_ultradns_token(self, mock_current_app, mock_requests):
+        # ret_val = json.dumps({"access_token": "access"})
+        the_response = Response()
+        the_response._content = b'{"access_token": "access"}'
+        mock_requests.post = Mock(return_value=the_response)
+        mock_current_app.config.get = Mock(return_value="Test")
+        result = ultradns.get_ultradns_token()
+        self.assertTrue(len(result) > 0)
+
+    @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name")
+    @patch("lemur.plugins.lemur_acme.ultradns._post")
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    def test_create_txt_record(self, mock_current_app, mock__post, mock_get_zone_name):
+        domain = "test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        change_id = (domain, token)
+        mock_current_app.logger.debug = Mock()
+        mock_get_zone_name = Mock(domain, account_number, return_value="example.com")
+        path = "a/b/c"
+        params = {
+            "test": "Test"
+        }
+        mock__post = Mock(path, params)
+        result = ultradns.create_txt_record(domain, token, account_number)
+        self.assertEqual(type(change_id), type(result))
+
+    # @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name")
+    # @patch("lemur.plugins.lemur_acme.ultradns._get")
+    # @patch("lemur.plugins.lemur_acme.ultradns._delete")
+    # @patch("lemur.plugins.lemur_acme.ultradns._post")
+    # @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    # def test_delete_txt_record(self, mock_get_zone_name):
+    #     domain = "test.example.com"
+    #     token = "ABCDEFGHIJ"
+    #     account_number = "1234567890"
+    #     change_id = (domain, token)
+    #     mock_get_zone_name = Mock(domain, account_number, return_value="example.com")
+
+    # @patch("lemur.plugins.lemur_acme.ultradns.get_authoritative_nameserver")
+    # @patch("lemur.plugins.lemur_acme.ultradns._has_dns_propagated")
+    # @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    # def test_wait_for_dns_change(self, mock_current_app, mock_has_dns_propagated, mock_get_authoritative_nameserver):
+    #     domain = "test.example.com"
+    #     token = "ABCDEFGHIJ"
+    #     account_number = "1234567890"
+    #     change_id = (domain, token)
+    #     mock_current_app.logger.debug = Mock()
+    #     result = ultradns.wait_for_dns_change(change_id, token)
+    #     self.assertEqual(result, true)
+
+    # def test_has_dns_propagated(self):
+
+
+
+

From b885cdf9d0cf9515a1351774a1eb4929297c2604 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Wed, 7 Aug 2019 10:24:38 -0700
Subject: [PATCH 17/40] adding multi profile name support with DigiCert plug.
 This requires that the configs are a dict, with multiple entries, where the
 key is the name of the Authority used to issue certs with.

DIGICERT_CIS_PROFILE_NAMES = {"sha2-rsa-ecc-root": "ssl_plus"}
DIGICERT_CIS_ROOTS = {"root": "ROOT"}
DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"}

Hence, in DB one need to add
1) the corresponding authority table, with digicert-cis-issuer. Note the names here are used to mapping in the above config
2) the corresponding intermediary in the certificate table , with root_aurhority_id set to the id of the new authority_id
---
 lemur/plugins/lemur_digicert/plugin.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py
index c5b01cc4..5e104094 100644
--- a/lemur/plugins/lemur_digicert/plugin.py
+++ b/lemur/plugins/lemur_digicert/plugin.py
@@ -158,7 +158,7 @@ def map_cis_fields(options, csr):
         )
 
     data = {
-        "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAME"),
+        "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES")[options['authority'].name],
         "common_name": options["common_name"],
         "additional_dns_names": get_additional_names(options),
         "csr": csr,
@@ -423,9 +423,9 @@ class DigiCertCISSourcePlugin(SourcePlugin):
         required_vars = [
             "DIGICERT_CIS_API_KEY",
             "DIGICERT_CIS_URL",
-            "DIGICERT_CIS_ROOT",
-            "DIGICERT_CIS_INTERMEDIATE",
-            "DIGICERT_CIS_PROFILE_NAME",
+            "DIGICERT_CIS_ROOTS",
+            "DIGICERT_CIS_INTERMEDIATES",
+            "DIGICERT_CIS_PROFILE_NAMES",
         ]
         validate_conf(current_app, required_vars)
 
@@ -498,9 +498,9 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
         required_vars = [
             "DIGICERT_CIS_API_KEY",
             "DIGICERT_CIS_URL",
-            "DIGICERT_CIS_ROOT",
-            "DIGICERT_CIS_INTERMEDIATE",
-            "DIGICERT_CIS_PROFILE_NAME",
+            "DIGICERT_CIS_ROOTS",
+            "DIGICERT_CIS_INTERMEDIATES",
+            "DIGICERT_CIS_PROFILE_NAMES",
         ]
 
         validate_conf(current_app, required_vars)
@@ -537,14 +537,14 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
         if "ECC" in issuer_options["key_type"]:
             return (
                 "\n".join(str(end_entity).splitlines()),
-                current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATE"),
+                current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES")[issuer_options['authority'].name],
                 data["id"],
             )
 
         # By default return RSA
         return (
             "\n".join(str(end_entity).splitlines()),
-            current_app.config.get("DIGICERT_CIS_INTERMEDIATE"),
+            current_app.config.get("DIGICERT_CIS_INTERMEDIATES")[issuer_options['authority'].name],
             data["id"],
         )
 
@@ -577,4 +577,4 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
         :return:
         """
         role = {"username": "", "password": "", "name": "digicert"}
-        return current_app.config.get("DIGICERT_CIS_ROOT"), "", [role]
+        return current_app.config.get("DIGICERT_CIS_ROOTS")[options['authority'].name], "", [role]

From e2ea2ca4d1663820894caac1bc86c962bffac010 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Wed, 7 Aug 2019 11:05:07 -0700
Subject: [PATCH 18/40] providing sample config

---
 lemur/tests/conf.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py
index 6d0d6967..af0c09ce 100644
--- a/lemur/tests/conf.py
+++ b/lemur/tests/conf.py
@@ -80,6 +80,13 @@ DIGICERT_API_KEY = "api-key"
 DIGICERT_ORG_ID = 111111
 DIGICERT_ROOT = "ROOT"
 
+DIGICERT_CIS_URL = "mock://www.digicert.com"
+DIGICERT_CIS_PROFILE_NAMES = {"sha2-rsa-ecc-root": "ssl_plus"}
+DIGICERT_CIS_API_KEY = "api-key"
+DIGICERT_CIS_ROOTS = {"root": "ROOT"}
+DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"}
+
+
 VERISIGN_URL = "http://example.com"
 VERISIGN_PEM_PATH = "~/"
 VERISIGN_FIRST_NAME = "Jim"

From bbda9b1d6f4bec461e1653ea8f9825f2d22d0fcc Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Wed, 7 Aug 2019 12:05:13 -0700
Subject: [PATCH 19/40] making sure to handle when no config file provided,
 though we do a check for that

---
 lemur/plugins/lemur_digicert/plugin.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py
index 5e104094..6f137281 100644
--- a/lemur/plugins/lemur_digicert/plugin.py
+++ b/lemur/plugins/lemur_digicert/plugin.py
@@ -158,7 +158,7 @@ def map_cis_fields(options, csr):
         )
 
     data = {
-        "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES")[options['authority'].name],
+        "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES", {}).get(options['authority'].name),
         "common_name": options["common_name"],
         "additional_dns_names": get_additional_names(options),
         "csr": csr,
@@ -537,14 +537,14 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
         if "ECC" in issuer_options["key_type"]:
             return (
                 "\n".join(str(end_entity).splitlines()),
-                current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES")[issuer_options['authority'].name],
+                current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name),
                 data["id"],
             )
 
         # By default return RSA
         return (
             "\n".join(str(end_entity).splitlines()),
-            current_app.config.get("DIGICERT_CIS_INTERMEDIATES")[issuer_options['authority'].name],
+            current_app.config.get("DIGICERT_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name),
             data["id"],
         )
 
@@ -577,4 +577,4 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
         :return:
         """
         role = {"username": "", "password": "", "name": "digicert"}
-        return current_app.config.get("DIGICERT_CIS_ROOTS")[options['authority'].name], "", [role]
+        return current_app.config.get("DIGICERT_CIS_ROOTS", {}).get(options['authority'].name), "", [role]

From ff1f73f985df4258f46aba9f7076059c8d2a2ed0 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Wed, 7 Aug 2019 12:05:36 -0700
Subject: [PATCH 20/40] fixing the plugin test to include authority

---
 lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py
index 71efbad4..77b0a1fa 100644
--- a/lemur/plugins/lemur_digicert/tests/test_digicert.py
+++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py
@@ -66,7 +66,7 @@ def test_map_fields_with_validity_years(app):
     }
 
 
-def test_map_cis_fields(app):
+def test_map_cis_fields(app, authority):
     from lemur.plugins.lemur_digicert.plugin import map_cis_fields
 
     names = [u"one.example.com", u"two.example.com", u"three.example.com"]
@@ -80,6 +80,7 @@ def test_map_cis_fields(app):
         "organizational_unit": "Example Org",
         "validity_end": arrow.get(2017, 5, 7),
         "validity_start": arrow.get(2016, 10, 30),
+        "authority": authority,
     }
 
     data = map_cis_fields(options, CSR_STR)
@@ -104,6 +105,7 @@ def test_map_cis_fields(app):
         "organization": "Example, Inc.",
         "organizational_unit": "Example Org",
         "validity_years": 2,
+        "authority": authority,
     }
 
     with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):

From 6e84e1fd59f682ec0efa6603835809cadfb741ca Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:04:38 -0700
Subject: [PATCH 21/40] Unit Tests for create_txt_record, delete_txt_record,
 wait_for_dns_change

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 117 +++++++++++++-------
 1 file changed, 76 insertions(+), 41 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 29c9534e..d0535718 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -374,50 +374,85 @@ class TestAcme(unittest.TestCase):
         result = ultradns.get_ultradns_token()
         self.assertTrue(len(result) > 0)
 
-    @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name")
-    @patch("lemur.plugins.lemur_acme.ultradns._post")
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    def test_create_txt_record(self, mock_current_app, mock__post, mock_get_zone_name):
-        domain = "test.example.com"
+    def test_create_txt_record(self, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        path = "a/b/c"
+        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+            'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.',
+                                        'pdns154.ultradns.org.']}}, 'inherit': 'ALL'},
+                             {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'},
+                             {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'}]
+        ultradns._paginate = Mock(path, "zones")
+        ultradns._paginate.side_effect = [[paginate_response]]
+        mock_current_app.logger.debug = Mock()
+        ultradns._post = Mock()
+        log_data = {
+            "function": "create_txt_record",
+            "fqdn": domain,
+            "token": token,
+            "message": "TXT record created"
+        }
+        result = ultradns.create_txt_record(domain, token, account_number)
+        mock_current_app.logger.debug.assert_called_with(log_data)
+
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    @patch("lemur.extensions.metrics")
+    def test_delete_txt_record(self, mock_metrics, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
         token = "ABCDEFGHIJ"
         account_number = "1234567890"
         change_id = (domain, token)
-        mock_current_app.logger.debug = Mock()
-        mock_get_zone_name = Mock(domain, account_number, return_value="example.com")
         path = "a/b/c"
-        params = {
-            "test": "Test"
-        }
-        mock__post = Mock(path, params)
-        result = ultradns.create_txt_record(domain, token, account_number)
-        self.assertEqual(type(change_id), type(result))
-
-    # @patch("lemur.plugins.lemur_acme.ultradns.get_zone_name")
-    # @patch("lemur.plugins.lemur_acme.ultradns._get")
-    # @patch("lemur.plugins.lemur_acme.ultradns._delete")
-    # @patch("lemur.plugins.lemur_acme.ultradns._post")
-    # @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    # def test_delete_txt_record(self, mock_get_zone_name):
-    #     domain = "test.example.com"
-    #     token = "ABCDEFGHIJ"
-    #     account_number = "1234567890"
-    #     change_id = (domain, token)
-    #     mock_get_zone_name = Mock(domain, account_number, return_value="example.com")
-
-    # @patch("lemur.plugins.lemur_acme.ultradns.get_authoritative_nameserver")
-    # @patch("lemur.plugins.lemur_acme.ultradns._has_dns_propagated")
-    # @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    # def test_wait_for_dns_change(self, mock_current_app, mock_has_dns_propagated, mock_get_authoritative_nameserver):
-    #     domain = "test.example.com"
-    #     token = "ABCDEFGHIJ"
-    #     account_number = "1234567890"
-    #     change_id = (domain, token)
-    #     mock_current_app.logger.debug = Mock()
-    #     result = ultradns.wait_for_dns_change(change_id, token)
-    #     self.assertEqual(result, true)
-
-    # def test_has_dns_propagated(self):
-
-
-
+        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+            'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.',
+                                        'pdns154.ultradns.org.']}}, 'inherit': 'ALL'},
+                             {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'},
+                             {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'}]
+        ultradns._paginate = Mock(path, "zones")
+        ultradns._paginate.side_effect = [[paginate_response]]
+        mock_current_app.logger.debug = Mock()
+        ultradns._post = Mock()
+        ultradns._get = Mock()
+        ultradns._get.return_value = {'zoneName': 'test.example.com.com',
+                'rrSets': [{'ownerName': '_acme-challenge.test.example.com.',
+                            'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}],
+                'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100},
+                'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}}
+        ultradns._delete = Mock()
+        mock_metrics.send = Mock()
+        mock_current_app.logger.debug.assert_not_called()
+        mock_metrics.send.assert_not_called()
 
+    @patch("lemur.extensions.metrics")
+    def test_wait_for_dns_change(self, mock_metrics):
+        ultradns._has_dns_propagated = Mock(return_value=True)
+        ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0")
+        mock_metrics.send = Mock()
+        mock_metrics.send.assert_not_called()

From f2cbddf9e21e3360f010bb6f6a8f473c54d51b0b Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:17:16 -0700
Subject: [PATCH 22/40] Unit tests for get_zone_name, get_zones

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 40 +++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index d0535718..a5e7c3e7 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -383,8 +383,9 @@ class TestAcme(unittest.TestCase):
         paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-            'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.',
-                                        'pdns154.ultradns.org.']}}, 'inherit': 'ALL'},
+                                'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                            'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'},
                              {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
@@ -456,3 +457,38 @@ class TestAcme(unittest.TestCase):
         ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0")
         mock_metrics.send = Mock()
         mock_metrics.send.assert_not_called()
+
+    def test_get_zone_name(self):
+        zones = ['example.com', 'test.example.com']
+        zone = "test.example.com"
+        domain = "_acme-challenge.test.example.com"
+        account_number = "1234567890"
+        ultradns.get_zones = Mock(return_value=zones)
+        result = ultradns.get_zone_name(domain, account_number)
+        self.assertEqual(result, zone)
+
+    def test_get_zones(self):
+        account_number = "1234567890"
+        path = "a/b/c"
+        zones = ['example.com', 'test.example.com']
+        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+            'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.',
+                                        'pdns154.ultradns.org.']}}, 'inherit': 'ALL'},
+                             {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'},
+                             {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY',
+                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
+                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
+                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                              'inherit': 'ALL'}]
+        ultradns._paginate = Mock(path, "zones")
+        ultradns._paginate.side_effect = [[paginate_response]]
+        result = ultradns.get_zones(account_number)
+        self.assertEqual(result, zones)

From 785c1ca73ec18ed7c4bdbd806513d063c296f5c4 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:20:24 -0700
Subject: [PATCH 23/40] test_create_txt_record modified - get_zone_name mocked
 to return the zone name directly, instead of actually running the function.

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 23 ++-------------------
 1 file changed, 2 insertions(+), 21 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index a5e7c3e7..da935a46 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -377,29 +377,10 @@ class TestAcme(unittest.TestCase):
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
     def test_create_txt_record(self, mock_current_app):
         domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
         token = "ABCDEFGHIJ"
         account_number = "1234567890"
-        path = "a/b/c"
-        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                            'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
-                              'inherit': 'ALL'},
-                             {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
-                              'inherit': 'ALL'},
-                             {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
-                              'inherit': 'ALL'}]
-        ultradns._paginate = Mock(path, "zones")
-        ultradns._paginate.side_effect = [[paginate_response]]
+        ultradns.get_zone_name = Mock(return_value=zone)
         mock_current_app.logger.debug = Mock()
         ultradns._post = Mock()
         log_data = {

From 31c2d207a2bafd83676cba7b94e68d9f23fe61b7 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:23:05 -0700
Subject: [PATCH 24/40] test_delete_txt_record fixed. Function call was missing
 earlier

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index da935a46..ae78f911 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -429,6 +429,7 @@ class TestAcme(unittest.TestCase):
                 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}}
         ultradns._delete = Mock()
         mock_metrics.send = Mock()
+        ultradns.delete_txt_record(change_id, account_number, domain, token)
         mock_current_app.logger.debug.assert_not_called()
         mock_metrics.send.assert_not_called()
 

From 37a1b55b0832b85bf2ccfbbd22f2fa92851bb2f9 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:27:21 -0700
Subject: [PATCH 25/40] test_delete_txt_record changed to mock get_zone_name
 and return the value directly instead of executing the function.

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 22 ++-------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index ae78f911..f6fe5b2f 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -396,30 +396,12 @@ class TestAcme(unittest.TestCase):
     @patch("lemur.extensions.metrics")
     def test_delete_txt_record(self, mock_metrics, mock_current_app):
         domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
         token = "ABCDEFGHIJ"
         account_number = "1234567890"
         change_id = (domain, token)
-        path = "a/b/c"
-        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-            'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.',
-                                        'pdns154.ultradns.org.']}}, 'inherit': 'ALL'},
-                             {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
-                              'inherit': 'ALL'},
-                             {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
-                              'inherit': 'ALL'}]
-        ultradns._paginate = Mock(path, "zones")
-        ultradns._paginate.side_effect = [[paginate_response]]
         mock_current_app.logger.debug = Mock()
+        ultradns.get_zone_name = Mock(return_value=zone)
         ultradns._post = Mock()
         ultradns._get = Mock()
         ultradns._get.return_value = {'zoneName': 'test.example.com.com',

From 894502644c03aa0dc2fcbac12ed34360f8d8d9e0 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:39:20 -0700
Subject: [PATCH 26/40] test_wait_for_dns_change fixed!

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index f6fe5b2f..199b4a05 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -415,12 +415,26 @@ class TestAcme(unittest.TestCase):
         mock_current_app.logger.debug.assert_not_called()
         mock_metrics.send.assert_not_called()
 
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
     @patch("lemur.extensions.metrics")
-    def test_wait_for_dns_change(self, mock_metrics):
+    def test_wait_for_dns_change(self, mock_metrics, mock_current_app):
         ultradns._has_dns_propagated = Mock(return_value=True)
         ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0")
         mock_metrics.send = Mock()
-        mock_metrics.send.assert_not_called()
+        domain = "_acme-challenge.test.example.com"
+        token = "ABCDEFGHIJ"
+        change_id = (domain, token)
+        mock_current_app.logger.debug = Mock()
+        ultradns.wait_for_dns_change(change_id)
+        # mock_metrics.send.assert_not_called()
+        log_data = {
+            "function": "wait_for_dns_change",
+            "fqdn": domain,
+            "status": True,
+            "message": "Record status on Public DNS"
+        }
+        mock_current_app.logger.debug.assert_called_with(log_data)
+
 
     def test_get_zone_name(self):
         zones = ['example.com', 'test.example.com']

From 3ff56fc5950ffd7ecb236a450f15b8218e5eedb5 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:42:11 -0700
Subject: [PATCH 27/40] Blank line removed

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 199b4a05..61a738bc 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -435,7 +435,6 @@ class TestAcme(unittest.TestCase):
         }
         mock_current_app.logger.debug.assert_called_with(log_data)
 
-
     def test_get_zone_name(self):
         zones = ['example.com', 'test.example.com']
         zone = "test.example.com"

From fa7f71d8599800fa5b54c2a2696f9793865a7481 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:53:10 -0700
Subject: [PATCH 28/40] Modified paginate response to dummy values

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 61a738bc..2d2055d8 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -390,6 +390,7 @@ class TestAcme(unittest.TestCase):
             "message": "TXT record created"
         }
         result = ultradns.create_txt_record(domain, token, account_number)
+        # TODO: check change_id
         mock_current_app.logger.debug.assert_called_with(log_data)
 
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
@@ -448,22 +449,22 @@ class TestAcme(unittest.TestCase):
         account_number = "1234567890"
         path = "a/b/c"
         zones = ['example.com', 'test.example.com']
-        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-            'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.', 'pdns154.ultradns.biz.',
-                                        'pdns154.ultradns.org.']}}, 'inherit': 'ALL'},
-                             {'properties': {'name': 'test.example.com.', 'accountName': 'netflix', 'type': 'PRIMARY',
+            'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', 'example.ultradns.biz.',
+                                        'example.ultradns.org.']}}, 'inherit': 'ALL'},
+                             {'properties': {'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                                 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                                             'example.ultradns.biz.', 'example.ultradns.org.']}},
                               'inherit': 'ALL'},
-                             {'properties': {'name': 'example2.com.', 'accountName': 'netflix', 'type': 'SECONDARY',
+                             {'properties': {'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['pdns154.ultradns.com.', 'pdns154.ultradns.net.',
-                                                             'pdns154.ultradns.biz.', 'pdns154.ultradns.org.']}},
+                                 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                                             'example.ultradns.biz.', 'example.ultradns.org.']}},
                               'inherit': 'ALL'}]
         ultradns._paginate = Mock(path, "zones")
         ultradns._paginate.side_effect = [[paginate_response]]

From b4f4e4dc241bb182b87fa97584dc2f399e3aa71c Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 13:55:02 -0700
Subject: [PATCH 29/40] Added extra check for return value to
 test_create_txt_record

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 2d2055d8..58857b75 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -380,6 +380,7 @@ class TestAcme(unittest.TestCase):
         zone = "test.example.com"
         token = "ABCDEFGHIJ"
         account_number = "1234567890"
+        change_id = (domain, token)
         ultradns.get_zone_name = Mock(return_value=zone)
         mock_current_app.logger.debug = Mock()
         ultradns._post = Mock()
@@ -390,8 +391,8 @@ class TestAcme(unittest.TestCase):
             "message": "TXT record created"
         }
         result = ultradns.create_txt_record(domain, token, account_number)
-        # TODO: check change_id
         mock_current_app.logger.debug.assert_called_with(log_data)
+        self.assertEqual(result, change_id)
 
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
     @patch("lemur.extensions.metrics")

From cadf372f7b5ca909a142ad732fdd92aaa56d0399 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 14:02:10 -0700
Subject: [PATCH 30/40] Removed hardcoded value from function call

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 58857b75..b66e6d58 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -421,7 +421,8 @@ class TestAcme(unittest.TestCase):
     @patch("lemur.extensions.metrics")
     def test_wait_for_dns_change(self, mock_metrics, mock_current_app):
         ultradns._has_dns_propagated = Mock(return_value=True)
-        ultradns.get_authoritative_nameserver = Mock(return_value="0.0.0.0")
+        nameserver = "0.0.0.0"
+        ultradns.get_authoritative_nameserver = Mock(return_value=nameserver)
         mock_metrics.send = Mock()
         domain = "_acme-challenge.test.example.com"
         token = "ABCDEFGHIJ"

From 43f5c8b34e74648bab47c63bebd783c1bd5a1410 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 14:08:06 -0700
Subject: [PATCH 31/40] Fixed indentation

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index b66e6d58..c4d10039 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -454,8 +454,9 @@ class TestAcme(unittest.TestCase):
         paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-            'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.', 'example.ultradns.biz.',
-                                        'example.ultradns.org.']}}, 'inherit': 'ALL'},
+                                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+                              'inherit': 'ALL'},
                              {'properties': {'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY',
                                              'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
                                              'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {

From a6bf081bec357fde6a17153fa57c1f51f0a621a6 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 14:08:27 -0700
Subject: [PATCH 32/40] Remove unused import

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index c4d10039..31a9e370 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -1,4 +1,3 @@
-import json
 import unittest
 from requests.models import Response
 

From a97283f0a43bd3203dc8a00220e6d65a524bc13b Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 14:23:09 -0700
Subject: [PATCH 33/40] Fixed indentation

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 43 ++++++++++++---------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index 31a9e370..f49141a8 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -450,24 +450,31 @@ class TestAcme(unittest.TestCase):
         account_number = "1234567890"
         path = "a/b/c"
         zones = ['example.com', 'test.example.com']
-        paginate_response = [{'properties': {'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
-                                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
-                              'inherit': 'ALL'},
-                             {'properties': {'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
-                                                             'example.ultradns.biz.', 'example.ultradns.org.']}},
-                              'inherit': 'ALL'},
-                             {'properties': {'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY',
-                                             'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                                             'lastModifiedDateTime': '2017-06-14T06:45Z'}, 'registrarInfo': {
-                                 'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
-                                                             'example.ultradns.biz.', 'example.ultradns.org.']}},
-                              'inherit': 'ALL'}]
+        paginate_response = [{
+            'properties': {
+                'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY',
+                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                'lastModifiedDateTime': '2017-06-14T06:45Z'},
+            'registrarInfo': {
+                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+            'inherit': 'ALL'}, {
+            'properties': {
+                'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY',
+                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                'lastModifiedDateTime': '2017-06-14T06:45Z'},
+            'registrarInfo': {
+                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+            'inherit': 'ALL'}, {
+            'properties': {
+                'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY',
+                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                'lastModifiedDateTime': '2017-06-14T06:45Z'},
+            'registrarInfo': {
+                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+            'inherit': 'ALL'}]
         ultradns._paginate = Mock(path, "zones")
         ultradns._paginate.side_effect = [[paginate_response]]
         result = ultradns.get_zones(account_number)

From d9aef2da3e51cfe1c20383301b457207b183d3d0 Mon Sep 17 00:00:00 2001
From: Kush Bavishi <kbavishi@netflix.com>
Date: Wed, 7 Aug 2019 14:38:18 -0700
Subject: [PATCH 34/40] Changed dummy nameserver value

---
 lemur/plugins/lemur_acme/tests/test_acme.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py
index f49141a8..2f9dd719 100644
--- a/lemur/plugins/lemur_acme/tests/test_acme.py
+++ b/lemur/plugins/lemur_acme/tests/test_acme.py
@@ -420,7 +420,7 @@ class TestAcme(unittest.TestCase):
     @patch("lemur.extensions.metrics")
     def test_wait_for_dns_change(self, mock_metrics, mock_current_app):
         ultradns._has_dns_propagated = Mock(return_value=True)
-        nameserver = "0.0.0.0"
+        nameserver = "1.1.1.1"
         ultradns.get_authoritative_nameserver = Mock(return_value=nameserver)
         mock_metrics.send = Mock()
         domain = "_acme-challenge.test.example.com"

From da9c91afb4f5b6bc6cb4016d5e3e049e4db41f13 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Thu, 8 Aug 2019 17:56:22 -0700
Subject: [PATCH 35/40] fixing metric bug

---
 lemur/common/celery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index f5edb9ab..b19a9607 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -348,7 +348,7 @@ def sync_source(source):
         return
     try:
         sync([source])
-        metrics.send(f"{function}.success", 'counter', '1', metric_tags={"source": source})
+        metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
     except SoftTimeLimitExceeded:
         log_data["message"] = "Error syncing source: Time limit exceeded."
         current_app.logger.error(log_data)

From bf47f87c215f9c6042374ddf3a43f5f4bbc24d43 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Mon, 12 Aug 2019 13:52:01 -0700
Subject: [PATCH 36/40] preventing celery duplicate tasks

---
 lemur/common/celery.py | 91 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index b19a9607..a79ec838 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -248,6 +248,15 @@ def remove_old_acme_certs():
     }
     pending_certs = pending_certificate_service.get_pending_certs("all")
 
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+    
     # Delete pending certs more than a week old
     for cert in pending_certs:
         if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
@@ -311,6 +320,17 @@ def sync_all_sources():
         "function": function,
         "message": "creating celery task to sync source",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+
     sources = validate_sources("all")
     for source in sources:
         log_data["source"] = source.label
@@ -340,6 +360,17 @@ def sync_source(source):
         "source": source,
         "task_id": task_id,
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+
     current_app.logger.debug(log_data)
 
     if task_id and is_task_active(function, task_id, (source,)):
@@ -378,6 +409,16 @@ def sync_source_destination():
         "function": function,
         "message": "syncing AWS destinations and sources",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     for dst in destinations_service.get_all():
         if add_aws_destination_to_sources(dst):
@@ -402,6 +443,16 @@ def certificate_reissue():
         "function": function,
         "message": "reissuing certificates",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     cli_certificate.reissue(None, True)
     log_data["message"] = "reissuance completed"
@@ -421,6 +472,16 @@ def certificate_rotate():
         "function": function,
         "message": "rotating certificates",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     cli_certificate.rotate(None, None, None, None, True)
     log_data["message"] = "rotation completed"
@@ -440,6 +501,16 @@ def endpoints_expire():
         "function": function,
         "message": "endpoints expire",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     cli_endpoints.expire(2)  # Time in hours
     red.set(f'{function}.last_success', int(time.time()))
@@ -457,6 +528,16 @@ def get_all_zones():
         "function": function,
         "message": "refresh all zones from available DNS providers",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     cli_dns_providers.get_all_zones()
     red.set(f'{function}.last_success', int(time.time()))
@@ -491,6 +572,16 @@ def notify_expirations():
         "function": function,
         "message": "notify for cert expiration",
     }
+
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    if task_id and is_task_active(function, task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
     red.set(f'{function}.last_success', int(time.time()))

From 07a9c56fb86b97c07a04b67fe576540d961cfb7b Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Tue, 13 Aug 2019 09:35:57 -0700
Subject: [PATCH 37/40] making lint happy

---
 lemur/common/celery.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index a79ec838..bcd7b580 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -31,7 +31,6 @@ from lemur.dns_providers import cli as cli_dns_providers
 from lemur.notifications import cli as cli_notification
 from lemur.endpoints import cli as cli_endpoints
 
-
 if current_app:
     flask_app = current_app
 else:
@@ -256,7 +255,7 @@ def remove_old_acme_certs():
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
-    
+
     # Delete pending certs more than a week old
     for cert in pending_certs:
         if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
@@ -330,7 +329,6 @@ def sync_all_sources():
         current_app.logger.debug(log_data)
         return
 
-
     sources = validate_sources("all")
     for source in sources:
         log_data["source"] = source.label
@@ -370,7 +368,6 @@ def sync_source(source):
         current_app.logger.debug(log_data)
         return
 
-
     current_app.logger.debug(log_data)
 
     if task_id and is_task_active(function, task_id, (source,)):

From 4d728738eeb2bcbc7afa4a30d15f6d7656674873 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Tue, 13 Aug 2019 11:42:43 -0700
Subject: [PATCH 38/40] handling celery tasks without any arguments

---
 lemur/common/celery.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index bcd7b580..fa739029 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -65,6 +65,9 @@ celery = make_celery(flask_app)
 def is_task_active(fun, task_id, args):
     from celery.task.control import inspect
 
+    if not args:
+        args = '()'  # empty args
+
     i = inspect()
     active_tasks = i.active()
     for _, tasks in active_tasks.items():

From c29f2825607df669ff67d3713fcfc4feb3fa96bc Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Tue, 13 Aug 2019 11:52:56 -0700
Subject: [PATCH 39/40] improved the flow for checking if the task is active

---
 lemur/common/celery.py | 169 ++++++++++++++++++++++++++---------------
 1 file changed, 109 insertions(+), 60 deletions(-)

diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index fa739029..a37f96e5 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -91,6 +91,21 @@ def report_celery_last_success_metrics():
 
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "recurrent task",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_time = int(time.time())
     schedule = current_app.config.get('CELERYBEAT_SCHEDULE')
     for _, t in schedule.items():
@@ -215,15 +230,25 @@ def fetch_acme_cert(id):
 @celery.task()
 def fetch_all_pending_acme_certs():
     """Instantiate celery workers to resolve all pending Acme certificates"""
-    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
 
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
     log_data = {
         "function": function,
         "message": "Starting job.",
+        "task_id": task_id,
     }
 
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
 
     # We only care about certs using the acme-issuer plugin
     for cert in pending_certs:
@@ -244,21 +269,23 @@ def fetch_all_pending_acme_certs():
 def remove_old_acme_certs():
     """Prune old pending acme certificates from the database"""
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "Starting job.",
-    }
-    pending_certs = pending_certificate_service.get_pending_certs("all")
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "Starting job.",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
 
+    pending_certs = pending_certificate_service.get_pending_certs("all")
+
     # Delete pending certs more than a week old
     for cert in pending_certs:
         if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
@@ -279,10 +306,21 @@ def clean_all_sources():
     be ran periodically. This function triggers one celery task per source.
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
     log_data = {
         "function": function,
         "message": "Creating celery task to clean source",
+        "task_id": task_id,
     }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     sources = validate_sources("all")
     for source in sources:
         log_data["source"] = source.label
@@ -303,11 +341,22 @@ def clean_source(source):
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
     log_data = {
         "function": function,
         "message": "Cleaning source",
         "source": source,
+        "task_id": task_id,
     }
+
+    if task_id and is_task_active(function, task_id, (source,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     current_app.logger.debug(log_data)
     clean([source], True)
 
@@ -318,16 +367,17 @@ def sync_all_sources():
     This function will sync certificates from all sources. This function triggers one celery task per source.
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "creating celery task to sync source",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "creating celery task to sync source",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
@@ -355,6 +405,7 @@ def sync_source(source):
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
+
     log_data = {
         "function": function,
         "message": "Syncing source",
@@ -362,21 +413,12 @@ def sync_source(source):
         "task_id": task_id,
     }
 
-    task_id = None
-    if celery.current_task:
-        task_id = celery.current_task.request.id
-
-    if task_id and is_task_active(function, task_id, (id,)):
+    if task_id and is_task_active(function, task_id, (source,)):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
 
     current_app.logger.debug(log_data)
-
-    if task_id and is_task_active(function, task_id, (source,)):
-        log_data["message"] = "Skipping task: Task is already active"
-        current_app.logger.debug(log_data)
-        return
     try:
         sync([source])
         metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
@@ -405,16 +447,17 @@ def sync_source_destination():
     We rely on account numbers to avoid duplicates.
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "syncing AWS destinations and sources",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "syncing AWS destinations and sources",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
@@ -439,16 +482,17 @@ def certificate_reissue():
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "reissuing certificates",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "reissuing certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
@@ -468,16 +512,18 @@ def certificate_rotate():
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "rotating certificates",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "rotating certificates",
+        "task_id": task_id,
+
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
@@ -497,16 +543,17 @@ def endpoints_expire():
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "endpoints expire",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "endpoints expire",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
@@ -524,16 +571,17 @@ def get_all_zones():
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "refresh all zones from available DNS providers",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "refresh all zones from available DNS providers",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return
@@ -568,16 +616,17 @@ def notify_expirations():
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": "notify for cert expiration",
-    }
-
     task_id = None
     if celery.current_task:
         task_id = celery.current_task.request.id
 
-    if task_id and is_task_active(function, task_id, (id,)):
+    log_data = {
+        "function": function,
+        "message": "notify for cert expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
         log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
         return

From 22c60fedad9047edf22d4126442e5ed48a338a70 Mon Sep 17 00:00:00 2001
From: Hossein Shafagh <hshafagh@netflix.com>
Date: Tue, 13 Aug 2019 12:11:04 -0700
Subject: [PATCH 40/40] cosmetics

---
 lemur/common/celery.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index a37f96e5..06b93a31 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -31,6 +31,7 @@ from lemur.dns_providers import cli as cli_dns_providers
 from lemur.notifications import cli as cli_notification
 from lemur.endpoints import cli as cli_endpoints
 
+
 if current_app:
     flask_app = current_app
 else: