Merge branch 'master' into vault-k8s-auth

docker/.dockerignore

@@ -0,0 +1,3 @@
+*-env
+docker-compose.yml
+Dockerfile

docker/Dockerfile

@@ -8,12 +8,6 @@ ENV gid 1337
 ENV user lemur
 ENV group lemur
 
-COPY entrypoint /
-COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
-COPY supervisor.conf /
-COPY nginx/default.conf /etc/nginx/conf.d/
-COPY nginx/default-ssl.conf /etc/nginx/conf.d/
-
 RUN addgroup -S ${group} -g ${gid} && \
     adduser -D -S ${user} -G ${group} -u ${uid} && \
     apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
@@ -40,7 +34,6 @@ RUN addgroup -S ${group} -g ${gid} && \
     curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
     pip3 install --upgrade pip && \
     pip3 install --upgrade setuptools && \
-    chmod +x /entrypoint && \
     mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
     chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
     
@@ -52,6 +45,13 @@ RUN npm install --unsafe-perm && \
     node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
     apk del build-dependencies
 
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
 WORKDIR /
 
 HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  

docker/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  postgres:
+    image: "postgres:10"
+    restart: always
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    env_file:
+      - pgsql-env
+
+  lemur:
+    # image: "netlix-lemur:latest"
+    build: .
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - lemur-env
+      - pgsql-env
+    ports:
+      - 80:80
+      - 443:443
+
+  redis:
+    image: "redis:alpine"
+
+volumes:
+    pg_data: {}

docker/entrypoint

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+set -eo pipefail
 
 if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
   echo "Database vars not set"
@@ -7,22 +9,23 @@ fi
 
 export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
 
-echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
+export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
+
+export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
 
-source /etc/profile
 
 PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
 
 echo " # Create Postgres trgm extension"
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 echo " # Done"
 
 if [ -z "${SKIP_SSL}" ]; then
   if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
     openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
   fi
-  mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
-  mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+  [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
 fi
 
 # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
@@ -33,7 +36,7 @@ fi
 # fi
 
 echo " # Running init"
-su lemur -c "python3 /opt/lemur/lemur/manage.py init"
+su lemur -s /bin/bash -c "cd /opt/lemur/lemur; python3 /opt/lemur/lemur/manage.py init -p ${LEMUR_ADMIN_PASSWORD}"
 echo " # Done"
 
 # echo "Creating user"

docker/lemur-env

@@ -0,0 +1,25 @@
+# SKIP_SSL=1
+# LEMUR_TOKEN_SECRET=
+# LEMUR_DEFAULT_COUNTRY=
+# LEMUR_DEFAULT_STATE=
+# LEMUR_DEFAULT_LOCATION=
+# LEMUR_DEFAULT_ORGANIZATION=
+# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT=
+# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer
+# LEMUR_DEFAULT_AUTHORITY=cryptography
+# MAIL_SERVER=mail.example.com
+# MAIL_PORT=25
+# LEMUR_EMAIL=lemur@example.com
+# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com']
+# LEMUR_TOKEN_SECRET=
+# LEMUR_ENCRYPTION_KEYS=['']
+# DEBUG=True
+# LDAP_DEBUG=True
+# LDAP_AUTH=True
+# LDAP_BIND_URI=ldap://example.com
+# LDAP_BASE_DN=DC=example,DC=com
+# LDAP_EMAIL_DOMAIN=example.com
+# LDAP_USE_TLS=False
+# LDAP_REQUIRED_GROUP=certificate-management-admins
+# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'}
+# LDAP_IS_ACTIVE_DIRECTORY=False

docker/nginx/default-ssl.conf

@@ -9,7 +9,7 @@ server {
 }
 
 server {
-   listen 443;
+   listen 443 ssl;
    server_name  _;
    access_log /dev/stdout;
    error_log /dev/stderr;

docker/pgsql-env

@@ -0,0 +1,4 @@
+POSTGRES_USER=lemur
+POSTGRES_PASSWORD=12345
+POSTGRES_DB=lemur
+POSTGRES_HOST=postgres

docker/src/lemur.conf.py

@@ -1,4 +1,6 @@
 import os
+from ast import literal_eval
+
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
 CORS = os.environ.get("CORS") == "True"
@@ -29,3 +31,13 @@ LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
 LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
 
 SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')
+
+LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True"
+LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True"
+LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True"
+LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI',''))
+LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN',''))
+LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN',''))
+LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS',''))
+LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP',''))
+LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}")

lemur/acme_providers/cli.py

@@ -0,0 +1,86 @@
+import time
+import json
+
+from flask_script import Manager
+from flask import current_app
+
+from lemur.extensions import sentry
+from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins.lemur_acme.plugin import AcmeHandler
+
+manager = Manager(
+    usage="Handles all ACME related tasks"
+)
+
+
+@manager.option(
+    "-d",
+    "--domain",
+    dest="domain",
+    required=True,
+    help="Name of the Domain to store to (ex. \"_acme-chall.test.com\".",
+)
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    required=True,
+    help="Value of the Token to store in DNS as content.",
+)
+def dnstest(domain, token):
+    """
+    Create, verify, and delete DNS TXT records using an autodetected provider.
+    """
+    print("[+] Starting ACME Tests.")
+    change_id = (domain, token)
+
+    acme_handler = AcmeHandler()
+    acme_handler.autodetect_dns_providers(domain)
+    if not acme_handler.dns_providers_for_domain[domain]:
+        raise Exception(f"No DNS providers found for domain: {format(domain)}.")
+
+    # Create TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        print(f"[+] Creating TXT Record in `{dns_provider.name}` provider")
+        change_id = dns_provider_plugin.create_txt_record(domain, token, account_number)
+
+    print("[+] Verifying TXT Record has propagated to DNS.")
+    print("[+] This step could take a while...")
+    time.sleep(10)
+
+    # Verify TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        try:
+            dns_provider_plugin.wait_for_dns_change(change_id, account_number)
+            print(f"[+] Verified TXT Record in `{dns_provider.name}` provider")
+        except Exception:
+            sentry.captureException()
+            current_app.logger.debug(
+                f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
+                f"{account_number}",
+                exc_info=True,
+            )
+            print(f"[+] Unable to Verify TXT Record in `{dns_provider.name}` provider")
+
+    time.sleep(10)
+
+    # Delete TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        # TODO(csine@: Add Exception Handling
+        dns_provider_plugin.delete_txt_record(change_id, account_number, domain, token)
+        print(f"[+] Deleted TXT Record in `{dns_provider.name}` provider")
+
+    status = SUCCESS_METRIC_STATUS
+    print("[+] Done with ACME Tests.")

lemur/auth/ldap.py

@@ -105,7 +105,7 @@ class LdapPrincipal:
             role = role_service.get_by_name(self.ldap_default_role)
             if role:
                 if not role.third_party:
-                    role = role.set_third_party(role.id, third_party_status=True)
+                    role = role_service.set_third_party(role.id, third_party_status=True)
                 roles.add(role)
 
         # update their 'roles'

lemur/certificates/schemas.py

@@ -119,6 +119,9 @@ class CertificateInputSchema(CertificateCreationSchema):
 
     @validates_schema
     def validate_authority(self, data):
+        if 'authority' not in data:
+            raise ValidationError("Missing Authority.")
+
         if isinstance(data["authority"], str):
             raise ValidationError("Authority not found.")
 

lemur/dns_providers/cli.py

@@ -1,8 +1,10 @@
 from flask_script import Manager
 
+import sys
+
 from lemur.constants import SUCCESS_METRIC_STATUS
 from lemur.dns_providers.service import get_all_dns_providers, set_domains
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.plugins.base import plugins
 
 manager = Manager(
@@ -19,13 +21,20 @@ def get_all_zones():
     dns_providers = get_all_dns_providers()
     acme_plugin = plugins.get("acme-issuer")
 
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    log_data = {
+        "function": function,
+        "message": "",
+    }
+
     for dns_provider in dns_providers:
         try:
             zones = acme_plugin.get_all_zones(dns_provider)
             set_domains(dns_provider, zones)
         except Exception as e:
             print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e))
-            set_domains(dns_provider, [])
+            log_data["message"] = f"get all zones failed for {dns_provider} {e}."
+            sentry.captureException(extra=log_data)
 
     status = SUCCESS_METRIC_STATUS
 

lemur/dns_providers/service.py

@@ -99,6 +99,7 @@ def get_types():
                 },
                 {"name": "dyn"},
                 {"name": "ultradns"},
+                {"name": "powerdns"},
             ]
         },
     )

lemur/dns_providers/util.py

@@ -0,0 +1,101 @@
+import sys
+import dns
+import dns.exception
+import dns.name
+import dns.query
+import dns.resolver
+import re
+
+from lemur.extensions import sentry
+from lemur.extensions import metrics
+
+
+class DNSError(Exception):
+    """Base class for DNS Exceptions."""
+    pass
+
+
+class BadDomainError(DNSError):
+    """Error for when a Bad Domain Name is given."""
+
+    def __init__(self, message):
+        self.message = message
+
+
+class DNSResolveError(DNSError):
+    """Error for DNS Resolution Errors."""
+
+    def __init__(self, message):
+        self.message = message
+
+
+def is_valid_domain(domain):
+    """Checks if a domain is syntactically valid and returns a bool"""
+    if len(domain) > 253:
+        return False
+    if domain[-1] == ".":
+        domain = domain[:-1]
+    fqdn_re = re.compile("(?=^.{1,254}$)(^(?:(?!\d+\.|-)[a-zA-Z0-9_\-]{1,63}(?<!-)\.?)+(?:[a-zA-Z]{2,})$)", re.IGNORECASE)
+    return all(fqdn_re.match(d) for d in domain.split("."))
+
+
+def get_authoritative_nameserver(domain):
+    """Get the authoritative nameservers for the given domain"""
+    if not is_valid_domain(domain):
+        raise BadDomainError(f"{domain} is not a valid FQDN")
+
+    n = dns.name.from_text(domain)
+
+    depth = 2
+    default = dns.resolver.get_default_resolver()
+    nameserver = default.nameservers[0]
+
+    last = False
+    while not last:
+        s = n.split(depth)
+
+        last = s[0].to_unicode() == u"@"
+        sub = s[1]
+
+        query = dns.message.make_query(sub, dns.rdatatype.NS)
+        response = dns.query.udp(query, nameserver)
+
+        rcode = response.rcode()
+        if rcode != dns.rcode.NOERROR:
+            function = sys._getframe().f_code.co_name
+            metrics.send(f"{function}.error", "counter", 1)
+            if rcode == dns.rcode.NXDOMAIN:
+                raise DNSResolveError(f"{sub} does not exist.")
+            else:
+                raise DNSResolveError(f"Error: {dns.rcode.to_text(rcode)}")
+
+        if len(response.authority) > 0:
+            rrset = response.authority[0]
+        else:
+            rrset = response.answer[0]
+
+        rr = rrset[0]
+        if rr.rdtype != dns.rdatatype.SOA:
+            authority = rr.target
+            nameserver = default.query(authority).rrset[0].to_text()
+
+        depth += 1
+
+    return nameserver
+
+
+def get_dns_records(domain, rdtype, nameserver):
+    """Retrieves the DNS records matching the name and type and returns a list of records"""
+    records = []
+    try:
+        dns_resolver = dns.resolver.Resolver()
+        dns_resolver.nameservers = [nameserver]
+        dns_response = dns_resolver.query(domain, rdtype)
+        for rdata in dns_response:
+            for record in rdata.strings:
+                records.append(record.decode("utf-8"))
+    except dns.exception.DNSException:
+        sentry.captureException()
+        function = sys._getframe().f_code.co_name
+        metrics.send(f"{function}.fail", "counter", 1)
+    return records

lemur/manage.py

@@ -17,6 +17,7 @@ from flask_migrate import Migrate, MigrateCommand, stamp
 from flask_script.commands import ShowUrls, Clean, Server
 
 from lemur.dns_providers.cli import manager as dns_provider_manager
+from lemur.acme_providers.cli import manager as acme_manager
 from lemur.sources.cli import manager as source_manager
 from lemur.policies.cli import manager as policy_manager
 from lemur.reporting.cli import manager as report_manager
@@ -584,6 +585,7 @@ def main():
     manager.add_command("policy", policy_manager)
     manager.add_command("pending_certs", pending_certificate_manager)
     manager.add_command("dns_providers", dns_provider_manager)
+    manager.add_command("acme", acme_manager)
     manager.run()
 
 

lemur/plugins/lemur_acme/plugin.py

@@ -31,7 +31,7 @@ from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProv
 from lemur.extensions import metrics, sentry
 from lemur.plugins import lemur_acme as acme
 from lemur.plugins.bases import IssuerPlugin
-from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns
+from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns
 from retrying import retry
 
 
@@ -377,6 +377,7 @@ class AcmeHandler(object):
             "dyn": dyn,
             "route53": route53,
             "ultradns": ultradns,
+            "powerdns": powerdns
         }
         provider = provider_types.get(type)
         if not provider:
@@ -436,6 +437,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
             "dyn": dyn,
             "route53": route53,
             "ultradns": ultradns,
+            "powerdns": powerdns
         }
         provider = provider_types.get(type)
         if not provider:

lemur/plugins/lemur_acme/powerdns.py

@@ -0,0 +1,267 @@
+import time
+import requests
+import json
+import sys
+
+import lemur.common.utils as utils
+import lemur.dns_providers.util as dnsutil
+
+from flask import current_app
+from lemur.extensions import metrics, sentry
+
+REQUIRED_VARIABLES = [
+    "ACME_POWERDNS_APIKEYNAME",
+    "ACME_POWERDNS_APIKEY",
+    "ACME_POWERDNS_DOMAIN",
+]
+
+
+class Zone:
+    """ This class implements a PowerDNS zone in JSON. """
+
+    def __init__(self, _data):
+        self._data = _data
+
+    @property
+    def id(self):
+        """ Zone id, has a trailing "." at the end, which we manually remove. """
+        return self._data["id"][:-1]
+
+    @property
+    def name(self):
+        """ Zone name, has a trailing "." at the end, which we manually remove. """
+        return self._data["name"][:-1]
+
+    @property
+    def kind(self):
+        """ Indicates whether the zone is setup as a PRIMARY or SECONDARY """
+        return self._data["kind"]
+
+
+class Record:
+    """ This class implements a PowerDNS record. """
+
+    def __init__(self, _data):
+        self._data = _data
+
+    @property
+    def name(self):
+        return self._data["name"]
+
+    @property
+    def disabled(self):
+        return self._data["disabled"]
+
+    @property
+    def content(self):
+        return self._data["content"]
+
+    @property
+    def ttl(self):
+        return self._data["ttl"]
+
+
+def get_zones(account_number):
+    """Retrieve authoritative zones from the PowerDNS API and return a list"""
+    _check_conf()
+    server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
+    path = f"/api/v1/servers/{server_id}/zones"
+    zones = []
+    function = sys._getframe().f_code.co_name
+    log_data = {
+        "function": function
+    }
+    try:
+        records = _get(path)
+        log_data["message"] = "Retrieved Zones Successfully"
+        current_app.logger.debug(log_data)
+
+    except Exception as e:
+        sentry.captureException()
+        log_data["message"] = "Failed to Retrieve Zone Data"
+        current_app.logger.debug(log_data)
+        raise
+
+    for record in records:
+        zone = Zone(record)
+        if zone.kind == 'Master':
+            zones.append(zone.name)
+    return zones
+
+
+def create_txt_record(domain, token, account_number):
+    """ Create a TXT record for the given domain and token and return a change_id tuple """
+    _check_conf()
+    zone_name = _get_zone_name(domain, account_number)
+    server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
+    zone_id = zone_name + "."
+    domain_id = domain + "."
+    path = f"/api/v1/servers/{server_id}/zones/{zone_id}"
+    payload = {
+        "rrsets": [
+            {
+                "name": domain_id,
+                "type": "TXT",
+                "ttl": 300,
+                "changetype": "REPLACE",
+                "records": [
+                    {
+                        "content": f"\"{token}\"",
+                        "disabled": False
+                    }
+                ],
+                "comments": []
+            }
+        ]
+    }
+    function = sys._getframe().f_code.co_name
+    log_data = {
+        "function": function,
+        "fqdn": domain,
+        "token": token,
+    }
+    try:
+        _patch(path, payload)
+        log_data["message"] = "TXT record successfully created"
+        current_app.logger.debug(log_data)
+    except Exception as e:
+        sentry.captureException()
+        log_data["Exception"] = e
+        log_data["message"] = "Unable to create TXT record"
+        current_app.logger.debug(log_data)
+
+    change_id = (domain, token)
+    return change_id
+
+
+def wait_for_dns_change(change_id, account_number=None):
+    """
+    Checks the authoritative DNS Server to see if changes have propagated to DNS
+    Retries and waits until successful.
+    """
+    _check_conf()
+    domain, token = change_id
+    number_of_attempts = current_app.config.get("ACME_POWERDNS_RETRIES", 3)
+    zone_name = _get_zone_name(domain, account_number)
+    nameserver = dnsutil.get_authoritative_nameserver(zone_name)
+    record_found = False
+    for attempts in range(0, number_of_attempts):
+        txt_records = dnsutil.get_dns_records(domain, "TXT", nameserver)
+        for txt_record in txt_records:
+            if txt_record == token:
+                record_found = True
+                break
+        if record_found:
+            break
+        time.sleep(10)
+
+    function = sys._getframe().f_code.co_name
+    log_data = {
+        "function": function,
+        "fqdn": domain,
+        "status": record_found,
+        "message": "Record status on PowerDNS authoritative server"
+    }
+    current_app.logger.debug(log_data)
+
+    if record_found:
+        metrics.send(f"{function}.success", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token})
+    else:
+        metrics.send(f"{function}.fail", "counter", 1, metric_tags={"fqdn": domain, "txt_record": token})
+
+
+def delete_txt_record(change_id, account_number, domain, token):
+    """ Delete the TXT record for the given domain and token """
+    _check_conf()
+    zone_name = _get_zone_name(domain, account_number)
+    server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
+    zone_id = zone_name + "."
+    domain_id = domain + "."
+    path = f"/api/v1/servers/{server_id}/zones/{zone_id}"
+    payload = {
+        "rrsets": [
+            {
+                "name": domain_id,
+                "type": "TXT",
+                "ttl": 300,
+                "changetype": "DELETE",
+                "records": [
+                    {
+                        "content": f"\"{token}\"",
+                        "disabled": False
+                    }
+                ],
+                "comments": []
+            }
+        ]
+    }
+    function = sys._getframe().f_code.co_name
+    log_data = {
+        "function": function,
+        "fqdn": domain,
+        "token": token
+    }
+    try:
+        _patch(path, payload)
+        log_data["message"] = "TXT record successfully deleted"
+        current_app.logger.debug(log_data)
+    except Exception as e:
+        sentry.captureException()
+        log_data["Exception"] = e
+        log_data["message"] = "Unable to delete TXT record"
+        current_app.logger.debug(log_data)
+
+
+def _check_conf():
+    utils.validate_conf(current_app, REQUIRED_VARIABLES)
+
+
+def _generate_header():
+    """Generate a PowerDNS API header and return it as a dictionary"""
+    api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME")
+    api_key = current_app.config.get("ACME_POWERDNS_APIKEY")
+    headers = {api_key_name: api_key}
+    return headers
+
+
+def _get_zone_name(domain, account_number):
+    """Get most specific matching zone for the given domain and return as a String"""
+    zones = get_zones(account_number)
+    zone_name = ""
+    for z in zones:
+        if domain.endswith(z):
+            if z.count(".") > zone_name.count("."):
+                zone_name = z
+    if not zone_name:
+        function = sys._getframe().f_code.co_name
+        log_data = {
+            "function": function,
+            "fqdn": domain,
+            "message": "No PowerDNS zone name found.",
+        }
+        metrics.send(f"{function}.fail", "counter", 1)
+    return zone_name
+
+
+def _get(path, params=None):
+    """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """
+    base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN")
+    resp = requests.get(
+        f"{base_uri}{path}",
+        headers=_generate_header(),
+        params=params,
+        verify=True,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def _patch(path, payload):
+    """ Execute a Patch request on the given URL (base_uri + path) with given payload """
+    base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN")
+    resp = requests.patch(
+        f"{base_uri}{path}",
+        data=json.dumps(payload),
+        headers=_generate_header()
+    )
+    resp.raise_for_status()

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -364,7 +364,7 @@ class TestAcme(unittest.TestCase):
 
     @patch("lemur.plugins.lemur_acme.ultradns.requests")
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    def test_get_ultradns_token(self, mock_current_app, mock_requests):
+    def test_ultradns_get_token(self, mock_current_app, mock_requests):
         # ret_val = json.dumps({"access_token": "access"})
         the_response = Response()
         the_response._content = b'{"access_token": "access"}'
@@ -374,7 +374,7 @@ class TestAcme(unittest.TestCase):
         self.assertTrue(len(result) > 0)
 
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    def test_create_txt_record(self, mock_current_app):
+    def test_ultradns_create_txt_record(self, mock_current_app):
         domain = "_acme_challenge.test.example.com"
         zone = "test.example.com"
         token = "ABCDEFGHIJ"
@@ -395,7 +395,7 @@ class TestAcme(unittest.TestCase):
 
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
     @patch("lemur.extensions.metrics")
-    def test_delete_txt_record(self, mock_metrics, mock_current_app):
+    def test_ultradns_delete_txt_record(self, mock_metrics, mock_current_app):
         domain = "_acme_challenge.test.example.com"
         zone = "test.example.com"
         token = "ABCDEFGHIJ"
@@ -418,7 +418,7 @@ class TestAcme(unittest.TestCase):
 
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
     @patch("lemur.extensions.metrics")
-    def test_wait_for_dns_change(self, mock_metrics, mock_current_app):
+    def test_ultradns_wait_for_dns_change(self, mock_metrics, mock_current_app):
         ultradns._has_dns_propagated = Mock(return_value=True)
         nameserver = "1.1.1.1"
         ultradns.get_authoritative_nameserver = Mock(return_value=nameserver)
@@ -437,7 +437,7 @@ class TestAcme(unittest.TestCase):
         }
         mock_current_app.logger.debug.assert_called_with(log_data)
 
-    def test_get_zone_name(self):
+    def test_ultradns_get_zone_name(self):
         zones = ['example.com', 'test.example.com']
         zone = "test.example.com"
         domain = "_acme-challenge.test.example.com"
@@ -446,7 +446,7 @@ class TestAcme(unittest.TestCase):
         result = ultradns.get_zone_name(domain, account_number)
         self.assertEqual(result, zone)
 
-    def test_get_zones(self):
+    def test_ultradns_get_zones(self):
         account_number = "1234567890"
         path = "a/b/c"
         zones = ['example.com', 'test.example.com']

lemur/plugins/lemur_acme/tests/test_powerdns.py

@@ -0,0 +1,120 @@
+import unittest
+from mock import Mock, patch
+from lemur.plugins.lemur_acme import plugin, powerdns
+
+
+class TestPowerdns(unittest.TestCase):
+    @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
+    def setUp(self, mock_dns_provider_service):
+        self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin()
+        self.acme = plugin.AcmeHandler()
+        mock_dns_provider = Mock()
+        mock_dns_provider.name = "powerdns"
+        mock_dns_provider.credentials = "{}"
+        mock_dns_provider.provider_type = "powerdns"
+        self.acme.dns_providers_for_domain = {
+            "www.test.com": [mock_dns_provider],
+            "test.fakedomain.net": [mock_dns_provider],
+        }
+
+    @patch("lemur.plugins.lemur_acme.powerdns.current_app")
+    def test_get_zones(self, mock_current_app):
+        account_number = "1234567890"
+        path = "a/b/c"
+        zones = ['example.com', 'test.example.com']
+        get_response = [{'account': '', 'dnssec': 'False', 'id': 'example.com.', 'kind': 'Master', 'last_check': 0, 'masters': [],
+          'name': 'example.com.', 'notified_serial': '2019111907', 'serial': '2019111907',
+          'url': '/api/v1/servers/localhost/zones/example.com.'},
+         {'account': '', 'dnssec': 'False', 'id': 'bad.example.com.', 'kind': 'Secondary', 'last_check': 0, 'masters': [],
+          'name': 'bad.example.com.', 'notified_serial': '2018053104', 'serial': '2018053104',
+          'url': '/api/v1/servers/localhost/zones/bad.example.com.'},
+         {'account': '', 'dnssec': 'False', 'id': 'test.example.com.', 'kind': 'Master', 'last_check': 0,
+          'masters': [], 'name': 'test.example.com.', 'notified_serial': '2019112501', 'serial': '2019112501',
+          'url': '/api/v1/servers/localhost/zones/test.example.com.'}]
+        powerdns._check_conf = Mock()
+        powerdns._get = Mock(path)
+        powerdns._get.side_effect = [get_response]
+        mock_current_app.config.get = Mock(return_value="localhost")
+        result = powerdns.get_zones(account_number)
+        self.assertEqual(result, zones)
+
+    def test_get_zone_name(self):
+        zones = ['example.com', 'test.example.com']
+        zone = "test.example.com"
+        domain = "_acme-challenge.test.example.com"
+        account_number = "1234567890"
+        powerdns.get_zones = Mock(return_value=zones)
+        result = powerdns._get_zone_name(domain, account_number)
+        self.assertEqual(result, zone)
+
+    @patch("lemur.plugins.lemur_acme.powerdns.current_app")
+    def test_create_txt_record(self, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        change_id = (domain, token)
+        powerdns._check_conf = Mock()
+        powerdns._get_zone_name = Mock(return_value=zone)
+        mock_current_app.logger.debug = Mock()
+        mock_current_app.config.get = Mock(return_value="localhost")
+        powerdns._patch = Mock()
+        log_data = {
+            "function": "create_txt_record",
+            "fqdn": domain,
+            "token": token,
+            "message": "TXT record successfully created"
+        }
+        result = powerdns.create_txt_record(domain, token, account_number)
+        mock_current_app.logger.debug.assert_called_with(log_data)
+        self.assertEqual(result, change_id)
+
+    @patch("lemur.plugins.lemur_acme.powerdns.dnsutil")
+    @patch("lemur.plugins.lemur_acme.powerdns.current_app")
+    @patch("lemur.extensions.metrics")
+    @patch("time.sleep")
+    def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil):
+        domain = "_acme-challenge.test.example.com"
+        token = "ABCDEFG"
+        zone_name = "test.example.com"
+        nameserver = "1.1.1.1"
+        change_id = (domain, token)
+        powerdns._check_conf = Mock()
+        mock_records = (token,)
+        mock_current_app.config.get = Mock(return_value=1)
+        powerdns._get_zone_name = Mock(return_value=zone_name)
+        mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver)
+        mock_dnsutil.get_dns_records = Mock(return_value=mock_records)
+        mock_sleep.return_value = False
+        mock_metrics.send = Mock()
+        mock_current_app.logger.debug = Mock()
+        powerdns.wait_for_dns_change(change_id)
+
+        log_data = {
+            "function": "wait_for_dns_change",
+            "fqdn": domain,
+            "status": True,
+            "message": "Record status on PowerDNS authoritative server"
+        }
+        mock_current_app.logger.debug.assert_called_with(log_data)
+
+    @patch("lemur.plugins.lemur_acme.powerdns.current_app")
+    def test_delete_txt_record(self, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        change_id = (domain, token)
+        powerdns._check_conf = Mock()
+        powerdns._get_zone_name = Mock(return_value=zone)
+        mock_current_app.logger.debug = Mock()
+        mock_current_app.config.get = Mock(return_value="localhost")
+        powerdns._patch = Mock()
+        log_data = {
+            "function": "delete_txt_record",
+            "fqdn": domain,
+            "token": token,
+            "message": "TXT record successfully deleted"
+        }
+        powerdns.delete_txt_record(change_id, account_number, domain, token)
+        mock_current_app.logger.debug.assert_called_with(log_data)

lemur/plugins/lemur_adcs/plugin.py

@@ -46,7 +46,7 @@ class ADCSIssuerPlugin(IssuerPlugin):
         )
         current_app.logger.info("Requesting CSR: {0}".format(csr))
         current_app.logger.info("Issuer options: {0}".format(issuer_options))
-        cert, req_id = (
+        cert = (
             ca_server.get_cert(csr, adcs_template, encoding="b64")
             .decode("utf-8")
             .replace("\r\n", "\n")
@@ -54,7 +54,7 @@ class ADCSIssuerPlugin(IssuerPlugin):
         chain = (
             ca_server.get_ca_cert(encoding="b64").decode("utf-8").replace("\r\n", "\n")
         )
-        return cert, chain, req_id
+        return cert, chain, None
 
     def revoke_certificate(self, certificate, comments):
         raise NotImplementedError("Not implemented\n", self, certificate, comments)

lemur/tests/test_dns_providers.py

@@ -0,0 +1,12 @@
+import unittest
+from lemur.dns_providers import util as dnsutil
+
+
+class TestDNSProvider(unittest.TestCase):
+    def test_is_valid_domain(self):
+        self.assertTrue(dnsutil.is_valid_domain("example.com"))
+        self.assertTrue(dnsutil.is_valid_domain("foo.bar.org"))
+        self.assertTrue(dnsutil.is_valid_domain("_acme-chall.example.com"))
+        self.assertFalse(dnsutil.is_valid_domain("e/xample.com"))
+        self.assertFalse(dnsutil.is_valid_domain("exam\ple.com"))
+        self.assertFalse(dnsutil.is_valid_domain("*.example.com"))

setup.py

@@ -23,7 +23,11 @@ from setuptools import setup, find_packages
 from subprocess import check_output
 
 import pip
-if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
+if tuple(map(int, pip.__version__.split('.'))) >= (19, 3, 0):
+    from pip._internal.network.session import PipSession
+    from pip._internal.req import parse_requirements
+
+elif tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
     from pip._internal.download import PipSession
     from pip._internal.req import parse_requirements
 else: