Merge branch 'master' into dependabot/pip/boto3-1.16.24

docs/production/index.rst

@@ -415,8 +415,8 @@ And the worker can be started with desired options such as the following::
 
 supervisor or systemd configurations should be created for these in production environments as appropriate.
 
-Add support for LetsEncrypt
+Add support for LetsEncrypt/ACME
-===========================
+================================
 
 LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
 for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
@@ -424,7 +424,10 @@ LetsEncrypt requires that we prove ownership of a domain before we're able to is
 time we want a certificate.
 
 The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
-through the creation of DNS TXT records.
+through the creation of DNS TXT records as well as HTTP validation, reusing the destination concept.
+
+ACME DNS Challenge
+------------------
 
 In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
 token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
@@ -462,6 +465,24 @@ possible. To enable this functionality, periodically (or through Cron/Celery) ru
 This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
 Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.
 
+ACME HTTP Challenge
+-------------------
+
+The flow for requesting a certificate using the HTTP challenge is not that different from the one described for the DNS
+challenge. The only difference is, that instead of creating a DNS TXT record, a file is uploaded to a Webserver which
+serves the file at `http://<domain>/.well-known/acme-challenge/<token>`
+
+Currently the HTTP challenge also works without Celery, since it's done while creating the certificate, and doesn't
+rely on celery to create the DNS record. This will change when we implement mix & match of acme challenge types.
+
+To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the
+challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to
+be the exact path that is called when the ACME providers calls ``http://<domain>/.well-known/acme-challenge/`. The
+token part will be added dynamically by the acme_upload.
+Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge.
+
+Afterwards you can create a new certificate authority as described in the DNS challenge, but need to choose
+`Acme HTTP-01` as the plugin type, and then the destination you created beforehand.
 
 LetsEncrypt: pinning to cross-signed ICA
 ----------------------------------------

lemur/common/utils.py

@@ -10,6 +10,7 @@ import random
 import re
 import string
 import pem
+import base64
 
 import sqlalchemy
 from cryptography import x509
@@ -34,6 +35,12 @@ paginated_parser.add_argument("filter", type=str, location="args")
 paginated_parser.add_argument("owner", type=str, location="args")
 
 
+def base64encode(string):
+    # Performs Base64 encoding of string to string using the base64.b64encode() function
+    # which encodes bytes to bytes.
+    return base64.b64encode(string.encode()).decode()
+
+
 def get_psuedo_random_string():
     """
     Create a random and strongish challenge.

lemur/plugins/lemur_azure_dest/__init__.py

@@ -0,0 +1,4 @@
+try:
+    VERSION = __import__("pkg_resources").get_distribution(__name__).version
+except Exception as e:
+    VERSION = "unknown"

lemur/plugins/lemur_azure_dest/plugin.py

@@ -0,0 +1,184 @@
+"""
+.. module: lemur.plugins.lemur_azure_dest.plugin
+    :platform: Unix
+    :copyright: (c) 2019
+    :license: Apache, see LICENCE for more details.
+
+    Plugin for uploading certificates and private key as secret to azure key-vault
+     that can be pulled down by end point nodes.
+
+.. moduleauthor:: sirferl
+"""
+from flask import current_app
+
+from lemur.common.defaults import common_name, bitstrength
+from lemur.common.utils import parse_certificate, parse_private_key
+from lemur.plugins.bases import DestinationPlugin
+
+from cryptography.hazmat.primitives import serialization
+import requests
+import json
+import sys
+
+
+def handle_response(my_response):
+    """
+    Helper function for parsing responses from the Entrust API.
+    :param my_response:
+    :return: :raise Exception:
+    """
+    msg = {
+        200: "The request was successful.",
+        400: "Keyvault Error"
+    }
+
+    try:
+        data = json.loads(my_response.content)
+    except ValueError:
+        # catch an empty jason object here
+        data = {'response': 'No detailed message'}
+    status_code = my_response.status_code
+    if status_code > 399:
+        raise Exception(f"AZURE error: {msg.get(status_code, status_code)}\n{data}")
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Response",
+        "status": status_code,
+        "response": data
+    }
+    current_app.logger.info(log_data)
+    if data == {'response': 'No detailed message'}:
+        # status if no data
+        return status_code
+    else:
+        #  return data from the response
+        return data
+
+
+def get_access_token(tenant, appID, password, self):
+    """
+    Gets the access token with the appid and the password and returns it
+
+    Improvment option: we can try to save it and renew it only when necessary
+
+    :param tenant: Tenant used
+    :param appID: Application ID from Azure
+    :param password: password for Application ID
+    :return: Access token to post to the keyvault
+    """
+    # prepare the call for the access_token
+    auth_url = f"https://login.microsoftonline.com/{tenant}/oauth2/token"
+    post_data = {
+        'grant_type': 'client_credentials',
+        'client_id': appID,
+        'client_secret': password,
+        'resource': 'https://vault.azure.net'
+    }
+    try:
+        response = self.session.post(auth_url, data=post_data)
+    except requests.exceptions.RequestException as e:
+        current_app.logger.exception(f"AZURE: Error for POST {e}")
+
+    access_token = json.loads(response.content)["access_token"]
+    return access_token
+
+
+class AzureDestinationPlugin(DestinationPlugin):
+    """Azure Keyvault Destination plugin for Lemur"""
+
+    title = "Azure"
+    slug = "azure-keyvault-destination"
+    description = "Allow the uploading of certificates to Azure key vault"
+
+    author = "Sirferl"
+    author_url = "https://github.com/sirferl/lemur"
+
+    options = [
+        {
+            "name": "vaultUrl",
+            "type": "str",
+            "required": True,
+            "validation": "^https?://[a-zA-Z0-9.:-]+$",
+            "helpMessage": "Valid URL to Azure key vault instance",
+        },
+        {
+            "name": "azureTenant",
+            "type": "str",
+            "required": True,
+            "validation": "^([a-zA-Z0-9/-/?)+$",
+            "helpMessage": "Tenant for the Azure Key Vault",
+        },
+        {
+            "name": "appID",
+            "type": "str",
+            "required": True,
+            "validation": "^([a-zA-Z0-9/-/?)+$",
+            "helpMessage": "AppID for the Azure Key Vault",
+        },
+        {
+            "name": "azurePassword",
+            "type": "str",
+            "required": True,
+            "validation": "[0-9a-zA-Z.:_-~]+",
+            "helpMessage": "Tenant password for the Azure Key Vault",
+        }
+    ]
+
+    def __init__(self, *args, **kwargs):
+        self.session = requests.Session()
+        super(AzureDestinationPlugin, self).__init__(*args, **kwargs)
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        """
+        Upload certificate and private key
+
+        :param private_key:
+        :param cert_chain:
+        :return:
+        """
+
+        # we use the common name to identify the certificate
+        # Azure does not allow "." in the certificate name we replace them with "-"
+        cert = parse_certificate(body)
+        certificate_name = common_name(cert).replace(".", "-")
+
+        vault_URI = self.get_option("vaultUrl", options)
+        tenant = self.get_option("azureTenant", options)
+        app_id = self.get_option("appID", options)
+        password = self.get_option("azurePassword", options)
+
+        access_token = get_access_token(tenant, app_id, password, self)
+
+        cert_url = f"{vault_URI}/certificates/{certificate_name}/import?api-version=7.1"
+        post_header = {
+            "Authorization": f"Bearer {access_token}"
+        }
+        key_pkcs8 = parse_private_key(private_key).private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        key_pkcs8 = key_pkcs8.decode("utf-8").replace('\\n', '\n')
+        cert_package = f"{body}\n{key_pkcs8}"
+
+        post_body = {
+            "value": cert_package,
+            "policy": {
+                "key_props": {
+                    "exportable": True,
+                    "kty": "RSA",
+                    "key_size": bitstrength(cert),
+                    "reuse_key": True
+                },
+                "secret_props": {
+                    "contentType": "application/x-pem-file"
+                }
+            }
+        }
+
+        try:
+            response = self.session.post(cert_url, headers=post_header, json=post_body)
+        except requests.exceptions.RequestException as e:
+            current_app.logger.exception(f"AZURE: Error for POST {e}")
+        return_value = handle_response(response)

lemur/plugins/lemur_azure_dest/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_kubernetes/plugin.py

@@ -10,7 +10,6 @@
 
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 """
-import base64
 import itertools
 import os
 
@@ -18,7 +17,7 @@ import requests
 from flask import current_app
 
 from lemur.common.defaults import common_name
-from lemur.common.utils import parse_certificate
+from lemur.common.utils import parse_certificate, base64encode
 from lemur.plugins.bases import DestinationPlugin
 
 DEFAULT_API_VERSION = "v1"
@@ -73,12 +72,6 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V
     )
 
 
-# Performs Base64 encoding of string to string using the base64.b64encode() function
-# which encodes bytes to bytes.
-def base64encode(string):
-    return base64.b64encode(string.encode()).decode()
-
-
 def build_secret(secret_format, secret_name, body, private_key, cert_chain):
     secret = {
         "apiVersion": "v1",

requirements-dev.txt

@@ -32,7 +32,7 @@ pygments==2.6.1           # via readme-renderer
 pyyaml==5.3.1             # via -r requirements-dev.in, pre-commit
 readme-renderer==25.0     # via twine
 requests-toolbelt==0.9.1  # via twine
-requests==2.24.0          # via requests-toolbelt, twine
+requests==2.25.0          # via requests-toolbelt, twine
 rfc3986==1.4.0            # via twine
 secretstorage==3.1.2      # via keyring
 six==1.15.0               # via bleach, cryptography, readme-renderer, virtualenv

requirements-docs.txt

@@ -85,7 +85,7 @@ pyyaml==5.3.1             # via -r requirements.txt, cloudflare
 raven[flask]==6.10.0      # via -r requirements.txt
 redis==3.5.3              # via -r requirements.txt, celery
 requests-toolbelt==0.9.1  # via -r requirements.txt, acme
-requests[security]==2.24.0  # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx
+requests[security]==2.25.0  # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx
 retrying==1.3.3           # via -r requirements.txt
 s3transfer==0.3.3         # via -r requirements.txt, boto3
 six==1.15.0               # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils

requirements-tests.txt

@@ -69,7 +69,7 @@ pyyaml==5.3.1             # via -r requirements-tests.in, bandit, cfn-lint, moto
 redis==3.5.3              # via fakeredis
 regex==2020.4.4           # via black
 requests-mock==1.8.0      # via -r requirements-tests.in
-requests==2.24.0          # via docker, moto, requests-mock, responses
+requests==2.25.0          # via docker, moto, requests-mock, responses
 responses==0.10.12        # via moto
 rsa==4.0                  # via python-jose
 s3transfer==0.3.3         # via boto3

requirements.txt

@@ -78,7 +78,7 @@ pyyaml==5.3.1             # via -r requirements.in, cloudflare
 raven[flask]==6.10.0      # via -r requirements.in
 redis==3.5.3              # via -r requirements.in, celery
 requests-toolbelt==0.9.1  # via acme
-requests[security]==2.24.0  # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt
+requests[security]==2.25.0  # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt
 retrying==1.3.3           # via -r requirements.in
 s3transfer==0.3.3         # via boto3
 six==1.15.0               # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils

setup.py

@@ -157,7 +157,8 @@ setup(
             'adcs_issuer = lemur.plugins.lemur_adcs.plugin:ADCSIssuerPlugin',
             'adcs_source = lemur.plugins.lemur_adcs.plugin:ADCSSourcePlugin',
             'entrust_issuer = lemur.plugins.lemur_entrust.plugin:EntrustIssuerPlugin',
-            'entrust_source = lemur.plugins.lemur_entrust.plugin:EntrustSourcePlugin'
+            'entrust_source = lemur.plugins.lemur_entrust.plugin:EntrustSourcePlugin',
+            'azure_destination = lemur.plugins.lemur_azure_dest.plugin:AzureDestinationPlugin'
         ],
     },
     classifiers=[