Merge branch 'master' into dependabot/pip/boto3-1.16.24

This commit is contained in:
Hossein Shafagh 2020-11-24 15:02:52 -08:00 committed by GitHub
commit 2710bcc263
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 227 additions and 16 deletions

View File

@ -415,8 +415,8 @@ And the worker can be started with desired options such as the following::
supervisor or systemd configurations should be created for these in production environments as appropriate. supervisor or systemd configurations should be created for these in production environments as appropriate.
Add support for LetsEncrypt Add support for LetsEncrypt/ACME
=========================== ================================
LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV). for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
@ -424,7 +424,10 @@ LetsEncrypt requires that we prove ownership of a domain before we're able to is
time we want a certificate. time we want a certificate.
The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
through the creation of DNS TXT records. through the creation of DNS TXT records as well as HTTP validation, reusing the destination concept.
ACME DNS Challenge
------------------
In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
@ -462,6 +465,24 @@ possible. To enable this functionality, periodically (or through Cron/Celery) ru
This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
Lemur's database (in the dns_providers table). Alternatively, you can manually input this data. Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.
ACME HTTP Challenge
-------------------
The flow for requesting a certificate using the HTTP challenge is not that different from the one described for the DNS
challenge. The only difference is, that instead of creating a DNS TXT record, a file is uploaded to a Webserver which
serves the file at `http://<domain>/.well-known/acme-challenge/<token>`
Currently the HTTP challenge also works without Celery, since it's done while creating the certificate, and doesn't
rely on celery to create the DNS record. This will change when we implement mix & match of acme challenge types.
To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the
challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to
be the exact path that is called when the ACME providers calls ``http://<domain>/.well-known/acme-challenge/`. The
token part will be added dynamically by the acme_upload.
Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge.
Afterwards you can create a new certificate authority as described in the DNS challenge, but need to choose
`Acme HTTP-01` as the plugin type, and then the destination you created beforehand.
LetsEncrypt: pinning to cross-signed ICA LetsEncrypt: pinning to cross-signed ICA
---------------------------------------- ----------------------------------------

View File

@ -10,6 +10,7 @@ import random
import re import re
import string import string
import pem import pem
import base64
import sqlalchemy import sqlalchemy
from cryptography import x509 from cryptography import x509
@ -34,6 +35,12 @@ paginated_parser.add_argument("filter", type=str, location="args")
paginated_parser.add_argument("owner", type=str, location="args") paginated_parser.add_argument("owner", type=str, location="args")
def base64encode(string):
# Performs Base64 encoding of string to string using the base64.b64encode() function
# which encodes bytes to bytes.
return base64.b64encode(string.encode()).decode()
def get_psuedo_random_string(): def get_psuedo_random_string():
""" """
Create a random and strongish challenge. Create a random and strongish challenge.

View File

@ -0,0 +1,4 @@
try:
VERSION = __import__("pkg_resources").get_distribution(__name__).version
except Exception as e:
VERSION = "unknown"

View File

@ -0,0 +1,184 @@
"""
.. module: lemur.plugins.lemur_azure_dest.plugin
:platform: Unix
:copyright: (c) 2019
:license: Apache, see LICENCE for more details.
Plugin for uploading certificates and private key as secret to azure key-vault
that can be pulled down by end point nodes.
.. moduleauthor:: sirferl
"""
from flask import current_app
from lemur.common.defaults import common_name, bitstrength
from lemur.common.utils import parse_certificate, parse_private_key
from lemur.plugins.bases import DestinationPlugin
from cryptography.hazmat.primitives import serialization
import requests
import json
import sys
def handle_response(my_response):
"""
Helper function for parsing responses from the Entrust API.
:param my_response:
:return: :raise Exception:
"""
msg = {
200: "The request was successful.",
400: "Keyvault Error"
}
try:
data = json.loads(my_response.content)
except ValueError:
# catch an empty jason object here
data = {'response': 'No detailed message'}
status_code = my_response.status_code
if status_code > 399:
raise Exception(f"AZURE error: {msg.get(status_code, status_code)}\n{data}")
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Response",
"status": status_code,
"response": data
}
current_app.logger.info(log_data)
if data == {'response': 'No detailed message'}:
# status if no data
return status_code
else:
# return data from the response
return data
def get_access_token(tenant, appID, password, self):
"""
Gets the access token with the appid and the password and returns it
Improvment option: we can try to save it and renew it only when necessary
:param tenant: Tenant used
:param appID: Application ID from Azure
:param password: password for Application ID
:return: Access token to post to the keyvault
"""
# prepare the call for the access_token
auth_url = f"https://login.microsoftonline.com/{tenant}/oauth2/token"
post_data = {
'grant_type': 'client_credentials',
'client_id': appID,
'client_secret': password,
'resource': 'https://vault.azure.net'
}
try:
response = self.session.post(auth_url, data=post_data)
except requests.exceptions.RequestException as e:
current_app.logger.exception(f"AZURE: Error for POST {e}")
access_token = json.loads(response.content)["access_token"]
return access_token
class AzureDestinationPlugin(DestinationPlugin):
"""Azure Keyvault Destination plugin for Lemur"""
title = "Azure"
slug = "azure-keyvault-destination"
description = "Allow the uploading of certificates to Azure key vault"
author = "Sirferl"
author_url = "https://github.com/sirferl/lemur"
options = [
{
"name": "vaultUrl",
"type": "str",
"required": True,
"validation": "^https?://[a-zA-Z0-9.:-]+$",
"helpMessage": "Valid URL to Azure key vault instance",
},
{
"name": "azureTenant",
"type": "str",
"required": True,
"validation": "^([a-zA-Z0-9/-/?)+$",
"helpMessage": "Tenant for the Azure Key Vault",
},
{
"name": "appID",
"type": "str",
"required": True,
"validation": "^([a-zA-Z0-9/-/?)+$",
"helpMessage": "AppID for the Azure Key Vault",
},
{
"name": "azurePassword",
"type": "str",
"required": True,
"validation": "[0-9a-zA-Z.:_-~]+",
"helpMessage": "Tenant password for the Azure Key Vault",
}
]
def __init__(self, *args, **kwargs):
self.session = requests.Session()
super(AzureDestinationPlugin, self).__init__(*args, **kwargs)
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
"""
Upload certificate and private key
:param private_key:
:param cert_chain:
:return:
"""
# we use the common name to identify the certificate
# Azure does not allow "." in the certificate name we replace them with "-"
cert = parse_certificate(body)
certificate_name = common_name(cert).replace(".", "-")
vault_URI = self.get_option("vaultUrl", options)
tenant = self.get_option("azureTenant", options)
app_id = self.get_option("appID", options)
password = self.get_option("azurePassword", options)
access_token = get_access_token(tenant, app_id, password, self)
cert_url = f"{vault_URI}/certificates/{certificate_name}/import?api-version=7.1"
post_header = {
"Authorization": f"Bearer {access_token}"
}
key_pkcs8 = parse_private_key(private_key).private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
)
key_pkcs8 = key_pkcs8.decode("utf-8").replace('\\n', '\n')
cert_package = f"{body}\n{key_pkcs8}"
post_body = {
"value": cert_package,
"policy": {
"key_props": {
"exportable": True,
"kty": "RSA",
"key_size": bitstrength(cert),
"reuse_key": True
},
"secret_props": {
"contentType": "application/x-pem-file"
}
}
}
try:
response = self.session.post(cert_url, headers=post_header, json=post_body)
except requests.exceptions.RequestException as e:
current_app.logger.exception(f"AZURE: Error for POST {e}")
return_value = handle_response(response)

View File

@ -0,0 +1 @@
from lemur.tests.conftest import * # noqa

View File

@ -10,7 +10,6 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com> .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
""" """
import base64
import itertools import itertools
import os import os
@ -18,7 +17,7 @@ import requests
from flask import current_app from flask import current_app
from lemur.common.defaults import common_name from lemur.common.defaults import common_name
from lemur.common.utils import parse_certificate from lemur.common.utils import parse_certificate, base64encode
from lemur.plugins.bases import DestinationPlugin from lemur.plugins.bases import DestinationPlugin
DEFAULT_API_VERSION = "v1" DEFAULT_API_VERSION = "v1"
@ -73,12 +72,6 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V
) )
# Performs Base64 encoding of string to string using the base64.b64encode() function
# which encodes bytes to bytes.
def base64encode(string):
return base64.b64encode(string.encode()).decode()
def build_secret(secret_format, secret_name, body, private_key, cert_chain): def build_secret(secret_format, secret_name, body, private_key, cert_chain):
secret = { secret = {
"apiVersion": "v1", "apiVersion": "v1",

View File

@ -32,7 +32,7 @@ pygments==2.6.1 # via readme-renderer
pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit
readme-renderer==25.0 # via twine readme-renderer==25.0 # via twine
requests-toolbelt==0.9.1 # via twine requests-toolbelt==0.9.1 # via twine
requests==2.24.0 # via requests-toolbelt, twine requests==2.25.0 # via requests-toolbelt, twine
rfc3986==1.4.0 # via twine rfc3986==1.4.0 # via twine
secretstorage==3.1.2 # via keyring secretstorage==3.1.2 # via keyring
six==1.15.0 # via bleach, cryptography, readme-renderer, virtualenv six==1.15.0 # via bleach, cryptography, readme-renderer, virtualenv

View File

@ -85,7 +85,7 @@ pyyaml==5.3.1 # via -r requirements.txt, cloudflare
raven[flask]==6.10.0 # via -r requirements.txt raven[flask]==6.10.0 # via -r requirements.txt
redis==3.5.3 # via -r requirements.txt, celery redis==3.5.3 # via -r requirements.txt, celery
requests-toolbelt==0.9.1 # via -r requirements.txt, acme requests-toolbelt==0.9.1 # via -r requirements.txt, acme
requests[security]==2.24.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx requests[security]==2.25.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx
retrying==1.3.3 # via -r requirements.txt retrying==1.3.3 # via -r requirements.txt
s3transfer==0.3.3 # via -r requirements.txt, boto3 s3transfer==0.3.3 # via -r requirements.txt, boto3
six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils

View File

@ -69,7 +69,7 @@ pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto
redis==3.5.3 # via fakeredis redis==3.5.3 # via fakeredis
regex==2020.4.4 # via black regex==2020.4.4 # via black
requests-mock==1.8.0 # via -r requirements-tests.in requests-mock==1.8.0 # via -r requirements-tests.in
requests==2.24.0 # via docker, moto, requests-mock, responses requests==2.25.0 # via docker, moto, requests-mock, responses
responses==0.10.12 # via moto responses==0.10.12 # via moto
rsa==4.0 # via python-jose rsa==4.0 # via python-jose
s3transfer==0.3.3 # via boto3 s3transfer==0.3.3 # via boto3

View File

@ -78,7 +78,7 @@ pyyaml==5.3.1 # via -r requirements.in, cloudflare
raven[flask]==6.10.0 # via -r requirements.in raven[flask]==6.10.0 # via -r requirements.in
redis==3.5.3 # via -r requirements.in, celery redis==3.5.3 # via -r requirements.in, celery
requests-toolbelt==0.9.1 # via acme requests-toolbelt==0.9.1 # via acme
requests[security]==2.24.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt requests[security]==2.25.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt
retrying==1.3.3 # via -r requirements.in retrying==1.3.3 # via -r requirements.in
s3transfer==0.3.3 # via boto3 s3transfer==0.3.3 # via boto3
six==1.15.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils six==1.15.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils

View File

@ -157,7 +157,8 @@ setup(
'adcs_issuer = lemur.plugins.lemur_adcs.plugin:ADCSIssuerPlugin', 'adcs_issuer = lemur.plugins.lemur_adcs.plugin:ADCSIssuerPlugin',
'adcs_source = lemur.plugins.lemur_adcs.plugin:ADCSSourcePlugin', 'adcs_source = lemur.plugins.lemur_adcs.plugin:ADCSSourcePlugin',
'entrust_issuer = lemur.plugins.lemur_entrust.plugin:EntrustIssuerPlugin', 'entrust_issuer = lemur.plugins.lemur_entrust.plugin:EntrustIssuerPlugin',
'entrust_source = lemur.plugins.lemur_entrust.plugin:EntrustSourcePlugin' 'entrust_source = lemur.plugins.lemur_entrust.plugin:EntrustSourcePlugin',
'azure_destination = lemur.plugins.lemur_azure_dest.plugin:AzureDestinationPlugin'
], ],
}, },
classifiers=[ classifiers=[