Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge

Browse Source
This commit is contained in:
Jasmine Schladen
2021-02-18 14:07:55 -08:00
parent dac6838c3b 2fb595046f
commit 46864f038c
107 changed files with 5421 additions and 1613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lemur/__about__.py
View File

@ -15,7 +15,7 @@ __title__ = "lemur"
__summary__ = "Certificate management and orchestration service"
__uri__ = "https://github.com/Netflix/lemur"
__version__ = "0.7.0"
__version__ = "0.8.0"
__author__ = "The Lemur developers"
__email__ = "security@netflix.com"

9
lemur/api_keys/service.py
View File

@ -7,6 +7,7 @@
"""
from lemur import database
from lemur.api_keys.models import ApiKey
from lemur.logs import service as log_service
def get(aid):
@ -24,6 +25,7 @@ def delete(access_key):
:param access_key:
:return:
"""
log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key")
database.delete(access_key)
@ -34,8 +36,9 @@ def revoke(aid):
:return:
"""
api_key = get(aid)
setattr(api_key, "revoked", False)
setattr(api_key, "revoked", True)
log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key")
return database.update(api_key)
@ -55,6 +58,9 @@ def create(**kwargs):
:return:
"""
api_key = ApiKey(**kwargs)
# this logs only metadata about the api key
log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}")
database.create(api_key)
return api_key
@ -69,6 +75,7 @@ def update(api_key, **kwargs):
for key, value in kwargs.items():
setattr(api_key, key, value)
log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}")
return database.update(api_key)

4
lemur/api_keys/views.py
View File

@ -105,6 +105,7 @@ class ApiKeyList(AuthenticatedResource):
POST /keys HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "my custom name",
@ -225,6 +226,7 @@ class ApiKeyUserList(AuthenticatedResource):
POST /users/1/keys HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "my custom name"
@ -332,6 +334,7 @@ class ApiKeys(AuthenticatedResource):
PUT /keys/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "new_name",
@ -474,6 +477,7 @@ class UserApiKeys(AuthenticatedResource):
PUT /users/1/keys/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "new_name",

2
lemur/auth/ldap.py
View File

@ -211,7 +211,7 @@ class LdapPrincipal:
for group in lgroups:
(dn, values) = group
if type(values) == dict:
self.ldap_groups.append(values["cn"][0].decode("ascii"))
self.ldap_groups.append(values["cn"][0].decode("utf-8"))
else:
lgroups = self.ldap_client.search_s(
self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs

9
lemur/auth/service.py
View File

@ -75,9 +75,9 @@ def create_token(user, aid=None, ttl=None):
if ttl == -1:
del payload["exp"]
else:
payload["exp"] = ttl
payload["exp"] = datetime.utcnow() + timedelta(days=ttl)
token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"])
return token.decode("unicode_escape")
return token
def login_required(f):
@ -116,9 +116,8 @@ def login_required(f):
return dict(message="Token has been revoked"), 403
if access_key.ttl != -1:
current_time = datetime.utcnow()
expired_time = datetime.fromtimestamp(
access_key.issued_at + access_key.ttl
)
# API key uses days
expired_time = datetime.fromtimestamp(access_key.issued_at) + timedelta(days=access_key.ttl)
if current_time >= expired_time:
return dict(message="Token has expired"), 403

64
lemur/auth/views.py
View File

@ -5,6 +5,8 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import json
import jwt
import base64
import requests
@ -20,9 +22,10 @@ from lemur.common.utils import get_psuedo_random_string
from lemur.users import service as user_service
from lemur.roles import service as role_service
from lemur.logs import service as log_service
from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
from lemur.auth import ldap
from lemur.plugins.base import plugins
mod = Blueprint("auth", __name__)
api = Api(mod)
@ -137,6 +140,47 @@ def retrieve_user(user_api_url, access_token):
return user, profile
def retrieve_user_memberships(user_api_url, user_membership_api_url, access_token):
user, profile = retrieve_user(user_api_url, access_token)
if user_membership_api_url is None:
return user, profile
"""
Potentially, below code can be made more generic i.e., plugin driven. Unaware of the usage of this
code across the community, current implementation is config driven. Without user_membership_api_url
configured, it is backward compatible.
"""
tls_provider = plugins.get(current_app.config.get("PING_USER_MEMBERSHIP_TLS_PROVIDER"))
# put user id in url
user_membership_api_url = user_membership_api_url.replace("%user_id%", profile["userId"])
session = tls_provider.session(current_app.config.get("PING_USER_MEMBERSHIP_SERVICE"))
headers = {"Content-Type": "application/json"}
data = {"relation": "DIRECT_ONLY", "groupFilter": {"type": "GOOGLE"}, "size": 500}
user_membership = {"email": profile["email"],
"thumbnailPhotoUrl": profile["thumbnailPhotoUrl"],
"googleGroups": []}
while True:
# retrieve information about the current user memberships
r = session.post(user_membership_api_url, data=json.dumps(data), headers=headers)
if r.status_code == 200:
response = r.json()
membership_details = response["data"]
for membership in membership_details:
user_membership["googleGroups"].append(membership["membership"]["name"])
if "nextPageToken" in response and response["nextPageToken"]:
data["nextPageToken"] = response["nextPageToken"]
else:
break
else:
current_app.logger.error(f"Response Code:{r.status_code} {r.text}")
break
return user, user_membership
def create_user_roles(profile):
"""Creates new roles based on profile information.
@ -155,7 +199,7 @@ def create_user_roles(profile):
description="This is a google group based role created by Lemur",
third_party=True,
)
if not role.third_party:
if (group != 'admin') and (not role.third_party):
role = role_service.set_third_party(role.id, third_party_status=True)
roles.append(role)
else:
@ -198,7 +242,6 @@ def update_user(user, profile, roles):
:param profile:
:param roles:
"""
# if we get an sso user create them an account
if not user:
user = user_service.create(
@ -212,10 +255,16 @@ def update_user(user, profile, roles):
else:
# we add 'lemur' specific roles, so they do not get marked as removed
removed_roles = []
for ur in user.roles:
if not ur.third_party:
roles.append(ur)
elif ur not in roles:
# This is a role assigned in lemur, but not returned by sso during current login
removed_roles.append(ur.name)
if removed_roles:
log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
# update any changes to the user
user_service.update(
user.id,
@ -262,6 +311,7 @@ class Login(Resource):
POST /auth/login HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"username": "test",
@ -368,7 +418,6 @@ class Ping(Resource):
# you can either discover these dynamically or simply configure them
access_token_url = current_app.config.get("PING_ACCESS_TOKEN_URL")
user_api_url = current_app.config.get("PING_USER_API_URL")
secret = current_app.config.get("PING_SECRET")
@ -384,7 +433,12 @@ class Ping(Resource):
error_code = validate_id_token(id_token, args["clientId"], jwks_url)
if error_code:
return error_code
user, profile = retrieve_user(user_api_url, access_token)
user, profile = retrieve_user_memberships(
current_app.config.get("PING_USER_API_URL"),
current_app.config.get("PING_USER_MEMBERSHIP_URL"),
access_token
)
roles = create_user_roles(profile)
update_user(user, profile, roles)

114
lemur/authorities/views.py
View File

@ -130,32 +130,33 @@ class AuthoritiesList(AuthenticatedResource):
POST /authorities HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"country": "US",
"state": "California",
"location": "Los Gatos",
"organization": "Netflix",
"organizationalUnit": "Operations",
"type": "root",
"signingAlgorithm": "sha256WithRSA",
"sensitivity": "medium",
"keyType": "RSA2048",
"plugin": {
"slug": "cloudca-issuer"
},
"name": "TimeTestAuthority5",
"owner": "secure@example.com",
"description": "test",
"commonName": "AcommonName",
"validityYears": "20",
"extensions": {
"subAltNames": {
"names": []
},
"custom": []
}
}
{
"country": "US",
"state": "California",
"location": "Los Gatos",
"organization": "Netflix",
"organizationalUnit": "Operations",
"type": "root",
"signingAlgorithm": "sha256WithRSA",
"sensitivity": "medium",
"keyType": "RSA2048",
"plugin": {
"slug": "cloudca-issuer"
},
"name": "TimeTestAuthority5",
"owner": "secure@example.com",
"description": "test",
"commonName": "AcommonName",
"validityYears": "20",
"extensions": {
"subAltNames": {
"names": []
},
"custom": []
}
}
**Example response**:
@ -217,8 +218,7 @@ class AuthoritiesList(AuthenticatedResource):
:arg parent: the parent authority if this is to be a subca
:arg signingAlgorithm: algorithm used to sign the authority
:arg keyType: key type
:arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
in an HSM
:arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored in an HSM
:arg keyName: name of the key to store in the HSM (CloudCA)
:arg serialNumber: serial number of the authority
:arg firstSerial: specifies the starting serial number for certificates issued off of this authority
@ -301,6 +301,7 @@ class Authorities(AuthenticatedResource):
PUT /authorities/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "TestAuthority5",
@ -492,23 +493,48 @@ class CertificateAuthority(AuthenticatedResource):
class AuthorityVisualizations(AuthenticatedResource):
def get(self, authority_id):
"""
{"name": "flare",
"children": [
{
"name": "analytics",
"children": [
{
"name": "cluster",
"children": [
{"name": "AgglomerativeCluster", "size": 3938},
{"name": "CommunityStructure", "size": 3812},
{"name": "HierarchicalCluster", "size": 6714},
{"name": "MergeEdge", "size": 743}
]
}
]
}
]}
.. http:get:: /authorities/1/visualize
Authority visualization
**Example request**:
.. sourcecode:: http
GET /certificates/1/visualize HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{"name": "flare",
"children": [
{
"name": "analytics",
"children": [
{
"name": "cluster",
"children": [
{"name": "AgglomerativeCluster", "size": 3938},
{"name": "CommunityStructure", "size": 3812},
{"name": "HierarchicalCluster", "size": 6714},
{"name": "MergeEdge", "size": 743}
]
}
]
}
]
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
authority = service.get(authority_id)
return dict(

128
lemur/certificates/cli.py
View File

@ -5,13 +5,13 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import multiprocessing
import sys
from flask import current_app
from flask_principal import Identity, identity_changed
from flask_script import Manager
from sqlalchemy import or_
from tabulate import tabulate
from time import sleep
from lemur import database
from lemur.authorities.models import Authority
@ -26,9 +26,10 @@ from lemur.certificates.service import (
get_all_valid_certs,
get,
get_all_certs_attached_to_endpoint_without_autorotate,
revoke as revoke_certificate,
)
from lemur.certificates.verify import verify_string
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS, CRLReason
from lemur.deployment import service as deployment_service
from lemur.domains.models import Domain
from lemur.endpoints import service as endpoint_service
@ -118,13 +119,20 @@ def request_rotation(endpoint, certificate, message, commit):
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException(extra={"certificate_name": str(certificate.name),
"endpoint": str(endpoint.dnsname)})
current_app.logger.exception(
f"Error rotating certificate: {certificate.name}", exc_info=True
)
print(
"[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
endpoint.name, certificate.name, e
)
)
metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status})
metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status,
"certificate_name": str(certificate.name),
"endpoint": str(endpoint.dnsname)})
def request_reissue(certificate, commit):
@ -223,7 +231,7 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
log_data["message"] = "Rotating endpoint"
log_data["message"] = "Rotating one endpoint"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = new_cert.name
request_rotation(endpoint, new_cert, message, commit)
@ -231,8 +239,6 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
elif old_cert and new_cert:
print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
log_data["message"] = "Rotating all endpoints"
log_data["certificate"] = new_cert.name
log_data["certificate_old"] = old_cert.name
log_data["message"] = "Rotating endpoint from old to new cert"
@ -243,41 +249,23 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
current_app.logger.info(log_data)
else:
# No certificate name or endpoint is provided. We will now fetch all endpoints,
# which are associated with a certificate that has been replaced
print("[+] Rotating all endpoints that have new certificates available")
log_data["message"] = "Rotating all endpoints that have new certificates available"
for endpoint in endpoint_service.get_all_pending_rotation():
log_data["endpoint"] = endpoint.dnsname
if len(endpoint.certificate.replaced) == 1:
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
log_data["certificate"] = endpoint.certificate.replaced[0].name
request_rotation(
endpoint, endpoint.certificate.replaced[0], message, commit
)
current_app.logger.info(log_data)
else:
log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
print(log_data)
metrics.send(
"endpoint_rotation",
"counter",
1,
metric_tags={
"status": FAILURE_METRIC_STATUS,
"old_certificate_name": str(old_cert),
"new_certificate_name": str(
endpoint.certificate.replaced[0].name
),
"endpoint_name": str(endpoint.name),
"message": str(message),
},
)
print(
f"[!] Failed to rotate endpoint {endpoint.name} reason: "
"Multiple replacement certificates found."
)
log_data["message"] = "Rotating endpoint from old to new cert"
if len(endpoint.certificate.replaced) > 1:
log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
f"{len(endpoint.certificate.replaced)}"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = endpoint.certificate.replaced[0].name
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
current_app.logger.info(log_data)
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
@ -368,6 +356,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
:param message: Send a rotation notification to the certificates owner.
:param commit: Persist changes.
:param region: Region in which to rotate the endpoint.
#todo: merge this method with rotate()
"""
if commit:
print("[!] Running in COMMIT mode.")
@ -417,24 +406,20 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
1,
metric_tags={
"region": region,
"old_certificate_name": str(old_cert),
"new_certificate_name": str(endpoint.certificate.replaced[0].name),
"endpoint_name": str(endpoint.dnsname),
},
)
continue
if len(endpoint.certificate.replaced) == 1:
log_data["certificate"] = endpoint.certificate.replaced[0].name
log_data["message"] = "Rotating all endpoints in region"
print(log_data)
current_app.logger.info(log_data)
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
status = SUCCESS_METRIC_STATUS
else:
status = FAILURE_METRIC_STATUS
log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
print(log_data)
current_app.logger.info(log_data)
log_data["certificate"] = endpoint.certificate.replaced[0].name
log_data["message"] = "Rotating all endpoints in region"
if len(endpoint.certificate.replaced) > 1:
log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
f"{len(endpoint.certificate.replaced)}"
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
current_app.logger.info(log_data)
metrics.send(
"endpoint_rotation_region",
@ -442,8 +427,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
1,
metric_tags={
"status": FAILURE_METRIC_STATUS,
"old_certificate_name": str(old_cert),
"new_certificate_name": str(endpoint.certificate.replaced[0].name),
"new_certificate_name": str(log_data["certificate"]),
"endpoint_name": str(endpoint.dnsname),
"message": str(message),
"region": str(region),
@ -586,11 +570,10 @@ def worker(data, commit, reason):
parts = [x for x in data.split(" ") if x]
try:
cert = get(int(parts[0].strip()))
plugin = plugins.get(cert.authority.plugin_name)
print("[+] Revoking certificate. Id: {0} Name: {1}".format(cert.id, cert.name))
if commit:
plugin.revoke_certificate(cert, reason)
revoke_certificate(cert, reason)
metrics.send(
"certificate_revoke",
@ -620,10 +603,10 @@ def clear_pending():
v.clear_pending_certificates()
@manager.option(
"-p", "--path", dest="path", help="Absolute file path to a Lemur query csv."
)
@manager.option("-r", "--reason", dest="reason", help="Reason to revoke certificate.")
@manager.option("-p", "--path", dest="path", help="Absolute file path to a Lemur query csv.")
@manager.option("-id", "--certid", dest="cert_id", help="ID of the certificate to be revoked")
@manager.option("-r", "--reason", dest="reason", default="unspecified", help="CRL Reason as per RFC 5280 section 5.3.1")
@manager.option("-m", "--message", dest="message", help="Message explaining reason for revocation")
@manager.option(
"-c",
"--commit",
@ -632,20 +615,32 @@ def clear_pending():
default=False,
help="Persist changes.",
)
def revoke(path, reason, commit):
def revoke(path, cert_id, reason, message, commit):
"""
Revokes given certificate.
"""
if not path and not cert_id:
print("[!] No input certificates mentioned to revoke")
return
if path and cert_id:
print("[!] Please mention single certificate id (-id) or input file (-p)")
return
if commit:
print("[!] Running in COMMIT mode.")
print("[+] Starting certificate revocation.")
with open(path, "r") as f:
args = [[x, commit, reason] for x in f.readlines()[2:]]
if reason not in CRLReason.__members__:
reason = CRLReason.unspecified.name
comments = {"comments": message, "crl_reason": reason}
with multiprocessing.Pool(processes=3) as pool:
pool.starmap(worker, args)
if cert_id:
worker(cert_id, commit, comments)
else:
with open(path, "r") as f:
for x in f.readlines()[2:]:
worker(x, commit, comments)
@manager.command
@ -750,7 +745,10 @@ def deactivate_entrust_certificates():
certificates = get_all_valid_certs(['entrust-issuer'])
entrust_plugin = plugins.get('entrust-issuer')
for cert in certificates:
for index, cert in enumerate(certificates):
if (index % 10) == 0:
# Entrust enforces a 10 request per 30s rate limit
sleep(30)
try:
response = entrust_plugin.deactivate_certificate(cert)
if response == 200:

8
lemur/certificates/schemas.py
View File

@ -16,7 +16,7 @@ from lemur.certificates import utils as cert_utils
from lemur.common import missing, utils, validators
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.constants import CERTIFICATE_KEY_TYPES
from lemur.constants import CERTIFICATE_KEY_TYPES, CRLReason
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.dns_providers.schemas import DnsProvidersNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
@ -89,7 +89,7 @@ class CertificateInputSchema(CertificateCreationSchema):
csr = fields.String(allow_none=True, validate=validators.csr)
key_type = fields.String(
validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="RSA2048"
validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="ECCPRIME256V1"
)
notify = fields.Boolean(default=True)
@ -160,7 +160,7 @@ class CertificateInputSchema(CertificateCreationSchema):
if data.get("body"):
data["key_type"] = utils.get_key_type_from_certificate(data["body"])
else:
data["key_type"] = "RSA2048" # default value
data["key_type"] = "ECCPRIME256V1" # default value
return missing.convert_validity_years(data)
@ -441,6 +441,7 @@ class CertificateExportInputSchema(LemurInputSchema):
class CertificateNotificationOutputSchema(LemurOutputSchema):
id = fields.Integer()
description = fields.String()
issuer = fields.String()
name = fields.String()
@ -455,6 +456,7 @@ class CertificateNotificationOutputSchema(LemurOutputSchema):
class CertificateRevokeSchema(LemurInputSchema):
comments = fields.String()
crl_reason = fields.String(validate=validate.OneOf(CRLReason.__members__), missing="unspecified")
certificates_list_request_parser = RequestParser()

98
lemur/certificates/service.py
View File

@ -21,6 +21,7 @@ from lemur.certificates.schemas import CertificateOutputSchema, CertificateInput
from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.endpoints import service as endpoint_service
from lemur.extensions import metrics, sentry, signals
from lemur.models import certificate_associations
from lemur.notifications.models import Notification
@ -387,6 +388,7 @@ def create(**kwargs):
cert = Certificate(**kwargs)
kwargs["creator"].certificates.append(cert)
else:
# ACME path
cert = PendingCertificate(**kwargs)
kwargs["creator"].pending_certificates.append(cert)
@ -562,10 +564,15 @@ def query_common_name(common_name, args):
:return:
"""
owner = args.pop("owner")
page = args.pop("page")
count = args.pop("count")
paginate = page and count
query = database.session_query(Certificate) if paginate else Certificate.query
# only not expired certificates
current_time = arrow.utcnow()
query = Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
query = query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
.filter(not_(Certificate.revoked))\
.filter(not_(Certificate.replaced.any())) # ignore rotated certificates to avoid duplicates
@ -576,6 +583,9 @@ def query_common_name(common_name, args):
# if common_name is a wildcard ('%'), no need to include it in the query
query = query.filter(Certificate.cn.ilike(common_name))
if paginate:
return database.paginate(query, page, count)
return query.all()
@ -794,6 +804,90 @@ def reissue_certificate(certificate, replace=None, user=None):
else:
primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
# Rotate the certificate to ECCPRIME256V1 if cert owner is present in the configured list
# This is a temporary change intending to rotate certificates to ECC, if opted in by certificate owners
# Unless identified a use case, this will be removed in mid-Q2 2021
ecc_reissue_owner_list = current_app.config.get("ROTATE_TO_ECC_OWNER_LIST", [])
ecc_reissue_exclude_cn_list = current_app.config.get("ECC_NON_COMPATIBLE_COMMON_NAMES", [])
if (certificate.owner in ecc_reissue_owner_list) and (certificate.cn not in ecc_reissue_exclude_cn_list):
primitives["key_type"] = "ECCPRIME256V1"
new_cert = create(**primitives)
return new_cert
def is_attached_to_endpoint(certificate_name, endpoint_name):
"""
Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name.
This method talks to elb and finds the real time information.
:param certificate_name:
:param endpoint_name:
:return: True if certificate is attached to the given endpoint, False otherwise
"""
endpoint = endpoint_service.get_by_name(endpoint_name)
attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint)
return certificate_name in attached_certificates
def remove_from_destination(certificate, destination):
"""
Remove the certificate from given destination if clean() is implemented
:param certificate:
:param destination:
:return:
"""
plugin = plugins.get(destination.plugin_name)
if not hasattr(plugin, "clean"):
info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'"
current_app.logger.warning(info_text)
else:
plugin.clean(certificate=certificate, options=destination.options)
def revoke(certificate, reason):
plugin = plugins.get(certificate.authority.plugin_name)
plugin.revoke_certificate(certificate, reason)
# Perform cleanup after revoke
return cleanup_after_revoke(certificate)
def cleanup_after_revoke(certificate):
"""
Perform the needed cleanup for a revoked certificate. This includes -
1. Disabling notification
2. Disabling auto-rotation
3. Update certificate status to 'revoked'
4. Remove from AWS
:param certificate: Certificate object to modify and update in DB
:return: None
"""
certificate.notify = False
certificate.rotation = False
certificate.status = 'revoked'
error_message = ""
for destination in list(certificate.destinations):
try:
remove_from_destination(certificate, destination)
certificate.destinations.remove(destination)
except Exception as e:
# This cleanup is the best-effort since certificate is already revoked at this point.
# We will capture the exception and move on to the next destination
sentry.captureException()
error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
database.update(certificate)
return error_message
def get_issued_cert_count_for_authority(authority):
"""
Returns the count of certs issued by the specified authority.
:return:
"""
return database.db.session.query(Certificate).filter(Certificate.authority_id == authority.id).count()

88
lemur/certificates/views.py
View File

@ -19,7 +19,7 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission
from lemur.certificates import service
from lemur.certificates.models import Certificate
from lemur.plugins.base import plugins
from lemur.extensions import sentry
from lemur.certificates.schemas import (
certificate_input_schema,
certificate_output_schema,
@ -28,6 +28,7 @@ from lemur.certificates.schemas import (
certificate_export_input_schema,
certificate_edit_input_schema,
certificates_list_output_schema_factory,
certificate_revoke_schema,
)
from lemur.roles import service as role_service
@ -50,17 +51,21 @@ class CertificatesListValid(AuthenticatedResource):
"""
.. http:get:: /certificates/valid/<query>
The current list of not-expired certificates for a given common name, and owner
The current list of not-expired certificates for a given common name, and owner. The API offers
optional pagination. One can send page number(>=1) and desired count per page. The returned data
contains total number of certificates which can help in determining the last page. Pagination
will not be offered if page or count info is not sent or if it is zero.
**Example request**:
.. sourcecode:: http
GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com
GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com&page=1&count=20
HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
**Example response (with single cert to be concise)**:
.. sourcecode:: http
@ -127,10 +132,15 @@ class CertificatesListValid(AuthenticatedResource):
:statuscode 403: unauthenticated
"""
parser = paginated_parser.copy()
args = parser.parse_args()
# using non-paginated parser to ensure backward compatibility
self.reqparse.add_argument("filter", type=str, location="args")
self.reqparse.add_argument("owner", type=str, location="args")
self.reqparse.add_argument("count", type=int, location="args")
self.reqparse.add_argument("page", type=int, location="args")
args = self.reqparse.parse_args()
args["user"] = g.user
common_name = args["filter"].split(";")[1]
common_name = args.pop("filter").split(";")[1]
return service.query_common_name(common_name, args)
@ -368,6 +378,7 @@ class CertificatesList(AuthenticatedResource):
POST /certificates HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"owner": "secure@example.net",
@ -517,6 +528,7 @@ class CertificatesUpload(AuthenticatedResource):
POST /certificates/upload HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"owner": "joe@example.com",
@ -783,6 +795,7 @@ class Certificates(AuthenticatedResource):
PUT /certificates/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"owner": "jimbob@example.com",
@ -888,8 +901,24 @@ class Certificates(AuthenticatedResource):
if cert.owner != data["owner"]:
service.cleanup_owner_roles_notification(cert.owner, data)
error_message = ""
# if destination is removed, cleanup the certificate from AWS
for destination in cert.destinations:
if destination not in data["destinations"]:
try:
service.remove_from_destination(cert, destination)
except Exception as e:
sentry.captureException()
# Add the removed destination back
data["destinations"].append(destination)
error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
# go ahead with DB update
cert = service.update(certificate_id, **data)
log_service.create(g.current_user, "update_cert", certificate=cert)
if error_message:
return dict(message=f"Edit Successful except -\n\n {error_message}"), 400
return cert
@validate_schema(certificate_edit_input_schema, certificate_output_schema)
@ -906,6 +935,7 @@ class Certificates(AuthenticatedResource):
POST /certificates/1/update/notify HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"notify": false
@ -1274,6 +1304,7 @@ class CertificateExport(AuthenticatedResource):
PUT /certificates/1/export HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"export": {
@ -1381,7 +1412,7 @@ class CertificateRevoke(AuthenticatedResource):
self.reqparse = reqparse.RequestParser()
super(CertificateRevoke, self).__init__()
@validate_schema(None, None)
@validate_schema(certificate_revoke_schema, None)
def put(self, certificate_id, data=None):
"""
.. http:put:: /certificates/1/revoke
@ -1395,6 +1426,12 @@ class CertificateRevoke(AuthenticatedResource):
POST /certificates/1/revoke HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"crlReason": "affiliationChanged",
"comments": "Additional details if any"
}
**Example response**:
@ -1405,12 +1442,13 @@ class CertificateRevoke(AuthenticatedResource):
Content-Type: text/javascript
{
'id': 1
"id": 1
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
:statuscode 403: unauthenticated or cert attached to LB
:statuscode 400: encountered error, more details in error message
"""
cert = service.get(certificate_id)
@ -1433,17 +1471,27 @@ class CertificateRevoke(AuthenticatedResource):
return dict(message="Cannot revoke certificate. No external id found."), 400
if cert.endpoints:
return (
dict(
message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
),
403,
)
for endpoint in cert.endpoints:
if service.is_attached_to_endpoint(cert.name, endpoint.name):
return (
dict(
message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
),
403,
)
plugin = plugins.get(cert.authority.plugin_name)
plugin.revoke_certificate(cert, data)
log_service.create(g.current_user, "revoke_cert", certificate=cert)
return dict(id=cert.id)
try:
error_message = service.revoke(cert, data)
log_service.create(g.current_user, "revoke_cert", certificate=cert)
if error_message:
return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400
return dict(id=cert.id)
except NotImplementedError as ne:
return dict(message="Revoke is not implemented for issuer of this certificate"), 400
except Exception as e:
sentry.captureException()
return dict(message=f"Failed to revoke: {str(e)}"), 400
api.add_resource(

83
lemur/common/celery.py
View File

@ -20,6 +20,7 @@ from flask import current_app
from lemur.authorities.service import get as get_authority
from lemur.certificates import cli as cli_certificate
from lemur.common.redis import RedisHandler
from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
from lemur.destinations import service as destinations_service
from lemur.dns_providers import cli as cli_dns_providers
from lemur.endpoints import cli as cli_endpoints
@ -273,7 +274,8 @@ def fetch_acme_cert(id):
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if not pending_cert:
if not pending_cert or pending_cert.resolved:
# pending_cert is cleared or it was resolved by another process
log_data[
"message"
] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
@ -301,7 +303,7 @@ def fetch_acme_cert(id):
error_log["last_error"] = cert.get("last_error")
error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > 4:
if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(
pending_cert, notify_owner=pending_cert.notify
@ -656,11 +658,12 @@ def certificate_rotate(**kwargs):
current_app.logger.debug(log_data)
try:
notify = current_app.config.get("ENABLE_ROTATION_NOTIFICATION", None)
if region:
log_data["region"] = region
cli_certificate.rotate_region(None, None, None, None, True, region)
cli_certificate.rotate_region(None, None, None, notify, True, region)
else:
cli_certificate.rotate(None, None, None, None, True)
cli_certificate.rotate(None, None, None, notify, True)
except SoftTimeLimitExceeded:
log_data["message"] = "Certificate rotate: Time limit exceeded."
current_app.logger.error(log_data)
@ -820,6 +823,78 @@ def notify_expirations():
return log_data
@celery.task(soft_time_limit=3600)
def notify_authority_expirations():
"""
This celery task notifies about expiring certificate authority certs
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "notify for certificate authority cert expiration",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_notification.authority_expirations()
except SoftTimeLimitExceeded:
log_data["message"] = "Notify expiring CA Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
metrics.send(f"{function}.success", "counter", 1)
return log_data
@celery.task(soft_time_limit=3600)
def send_security_expiration_summary():
"""
This celery task sends a summary about expiring certificates to the security team.
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "send summary for certificate expiration",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_notification.security_expiration_summary(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
except SoftTimeLimitExceeded:
log_data["message"] = "Send summary for expiring certs Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
metrics.send(f"{function}.success", "counter", 1)
return log_data
@celery.task(soft_time_limit=3600)
def enable_autorotate_for_certs_attached_to_endpoint():
"""

7
lemur/common/utils.py
View File

@ -10,6 +10,7 @@ import random
import re
import string
import pem
import base64
import sqlalchemy
from cryptography import x509
@ -34,6 +35,12 @@ paginated_parser.add_argument("filter", type=str, location="args")
paginated_parser.add_argument("owner", type=str, location="args")
def base64encode(string):
# Performs Base64 encoding of string to string using the base64.b64encode() function
# which encodes bytes to bytes.
return base64.b64encode(string.encode()).decode()
def get_psuedo_random_string():
"""
Create a random and strongish challenge.

19
lemur/constants.py
View File

@ -3,6 +3,8 @@
:copyright: (c) 2018 by Netflix Inc.
:license: Apache, see LICENSE for more details.
"""
from enum import IntEnum
SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"
DEFAULT_NAMING_TEMPLATE = "{subject}-{issuer}-{not_before}-{not_after}"
NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
@ -10,6 +12,9 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
SUCCESS_METRIC_STATUS = "success"
FAILURE_METRIC_STATUS = "failure"
# when ACME attempts to resolve a certificate try in total 3 times
ACME_ADDITIONAL_ATTEMPTS = 2
CERTIFICATE_KEY_TYPES = [
"RSA2048",
"RSA4096",
@ -32,3 +37,17 @@ CERTIFICATE_KEY_TYPES = [
"ECCSECT409R1",
"ECCSECT571R2",
]
# As per RFC 5280 section 5.3.1 (https://tools.ietf.org/html/rfc5280#section-5.3.1)
class CRLReason(IntEnum):
unspecified = 0,
keyCompromise = 1,
cACompromise = 2,
affiliationChanged = 3,
superseded = 4,
cessationOfOperation = 5,
certificateHold = 6,
removeFromCRL = 8,
privilegeWithdrawn = 9,
aACompromise = 10

18
lemur/database.py
View File

@ -9,6 +9,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import math
from inflection import underscore
from sqlalchemy import exc, func, distinct
from sqlalchemy.orm import make_transient, lazyload
@ -219,13 +220,20 @@ def sort(query, model, field, direction):
def paginate(query, page, count):
"""
Returns the items given the count and page specified
Returns the items given the count and page specified. The items would be an empty list
if page number exceeds max page number based on count per page and total number of records.
:param query:
:param page:
:param count:
:param query: search query
:param page: current page number
:param count: results per page
"""
return query.paginate(page, count)
total = get_count(query)
# Check if input page is higher than total number of pages based on count per page and total
# In such a case Flask-SQLAlchemy pagination call results in 404
if math.ceil(total / count) < page:
return dict(items=[], total=total)
items = query.paginate(page, count).items
return dict(items=items, total=total)
def update_list(model, model_attr, item_model, items):

6
lemur/destinations/service.py
View File

@ -21,7 +21,7 @@ def create(label, plugin_name, options, description=None):
:param label: Destination common name
:param description:
:rtype : Destination
:rtype: Destination
:return: New destination
"""
# remove any sub-plugin objects before try to save the json options
@ -50,7 +50,7 @@ def update(destination_id, label, plugin_name, options, description):
:param plugin_name:
:param options:
:param description:
:rtype : Destination
:rtype: Destination
:return:
"""
destination = get(destination_id)
@ -81,7 +81,7 @@ def get(destination_id):
Retrieves an destination by its lemur assigned ID.
:param destination_id: Lemur assigned ID
:rtype : Destination
:rtype: Destination
:return:
"""
return database.get(Destination, destination_id)

2
lemur/destinations/views.py
View File

@ -113,6 +113,7 @@ class DestinationsList(AuthenticatedResource):
POST /destinations HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"description": "test33",
@ -264,6 +265,7 @@ class Destinations(AuthenticatedResource):
POST /destinations/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{

6
lemur/dns_providers/cli.py
View File

@ -3,9 +3,9 @@ from flask_script import Manager
import sys
from lemur.constants import SUCCESS_METRIC_STATUS
from lemur.plugins.lemur_acme.acme_handlers import AcmeDnsHandler
from lemur.dns_providers.service import get_all_dns_providers, set_domains
from lemur.extensions import metrics, sentry
from lemur.plugins.base import plugins
manager = Manager(
usage="Iterates through all DNS providers and sets DNS zones in the database."
@ -19,7 +19,7 @@ def get_all_zones():
"""
print("[+] Starting dns provider zone lookup and configuration.")
dns_providers = get_all_dns_providers()
acme_plugin = plugins.get("acme-issuer")
acme_dns_handler = AcmeDnsHandler()
function = f"{__name__}.{sys._getframe().f_code.co_name}"
log_data = {
@ -29,7 +29,7 @@ def get_all_zones():
for dns_provider in dns_providers:
try:
zones = acme_plugin.get_all_zones(dns_provider)
zones = acme_dns_handler.get_all_zones(dns_provider)
set_domains(dns_provider, zones)
except Exception as e:
print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e))

16
lemur/logs/service.py
View File

@ -7,7 +7,7 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
from flask import current_app, g
from lemur import database
from lemur.logs.models import Log
@ -34,6 +34,20 @@ def create(user, type, certificate=None):
database.commit()
def audit_log(action, entity, message):
"""
Logs given action
:param action: The action being logged e.g. assign_role, create_role etc
:param entity: The entity undergoing the action e.g. name of the role
:param message: Additional info e.g. Role being assigned to user X
:return:
"""
user = g.current_user.email if hasattr(g, 'current_user') else "LEMUR"
current_app.logger.info(
f"[lemur-audit] action: {action}, user: {user}, entity: {entity}, details: {message}"
)
def get_all():
"""
Retrieve all logs from the database.

55
lemur/notifications/cli.py
View File

@ -10,6 +10,8 @@ from flask_script import Manager
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.extensions import sentry, metrics
from lemur.notifications.messaging import send_expiration_notifications
from lemur.notifications.messaging import send_authority_expiration_notifications
from lemur.notifications.messaging import send_security_expiration_summary
manager = Manager(usage="Handles notification related tasks.")
@ -24,7 +26,7 @@ manager = Manager(usage="Handles notification related tasks.")
)
def expirations(exclude):
"""
Runs Lemur's notification engine, that looks for expired certificates and sends
Runs Lemur's notification engine, that looks for expiring certificates and sends
notifications out to those that have subscribed to them.
Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur
@ -39,9 +41,7 @@ def expirations(exclude):
print("Starting to notify subscribers about expiring certificates!")
success, failed = send_expiration_notifications(exclude)
print(
"Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}".format(
success=success, failed=failed
)
f"Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}"
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
@ -50,3 +50,50 @@ def expirations(exclude):
metrics.send(
"expiration_notification_job", "counter", 1, metric_tags={"status": status}
)
def authority_expirations():
"""
Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends
notifications out to the security team and owner.
:return:
"""
status = FAILURE_METRIC_STATUS
try:
print("Starting to notify subscribers about expiring certificate authority certificates!")
success, failed = send_authority_expiration_notifications()
print(
"Finished notifying subscribers about expiring certificate authority certificates! "
f"Sent: {success} Failed: {failed}"
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
metrics.send(
"authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
)
def security_expiration_summary(exclude):
"""
Sends a summary email with info on all expiring certs (that match the configured expiry intervals).
:return:
"""
status = FAILURE_METRIC_STATUS
try:
print("Starting to notify security team about expiring certificates!")
success = send_security_expiration_summary(exclude)
print(
f"Finished notifying security team about expiring certificates! Success: {success}"
)
if success:
status = SUCCESS_METRIC_STATUS
except Exception:
sentry.captureException()
metrics.send(
"security_expiration_notification_job", "counter", 1, metric_tags={"status": status}
)

246
lemur/notifications/messaging.py
View File

@ -19,9 +19,10 @@ from sqlalchemy import and_
from sqlalchemy.sql.expression import false, true
from lemur import database
from lemur.certificates import service as certificates_service
from lemur.certificates.models import Certificate
from lemur.certificates.schemas import certificate_notification_output_schema
from lemur.common.utils import windowed_query
from lemur.common.utils import windowed_query, is_selfsigned
from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS
from lemur.extensions import metrics, sentry
from lemur.pending_certificates.schemas import pending_certificate_output_schema
@ -62,6 +63,67 @@ def get_certificates(exclude=None):
return certs
def get_certificates_for_security_summary_email(exclude=None):
"""
Finds all certificates that are eligible for expiration notifications for the security expiration summary.
:param exclude:
:return:
"""
now = arrow.utcnow()
threshold_days = current_app.config.get("LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS", 14)
max_not_after = now + timedelta(days=threshold_days + 1)
q = (
database.db.session.query(Certificate)
.filter(Certificate.not_after <= max_not_after)
.filter(Certificate.notify == true())
.filter(Certificate.expired == false())
.filter(Certificate.revoked == false())
)
exclude_conditions = []
if exclude:
for e in exclude:
exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e)))
q = q.filter(and_(*exclude_conditions))
certs = []
for c in windowed_query(q, Certificate.id, 10000):
days_remaining = (c.not_after - now).days
if days_remaining <= threshold_days:
certs.append(c)
return certs
def get_expiring_authority_certificates():
"""
Finds all certificate authority certificates that are eligible for expiration notifications.
:return:
"""
now = arrow.utcnow()
authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS",
[365, 180])
max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1)
q = (
database.db.session.query(Certificate)
.filter(Certificate.not_after < max_not_after)
.filter(Certificate.notify == true())
.filter(Certificate.expired == false())
.filter(Certificate.revoked == false())
.filter(Certificate.root_authority_id.isnot(None))
.filter(Certificate.authority_id.is_(None))
)
certs = []
for c in windowed_query(q, Certificate.id, 10000):
days_remaining = (c.not_after - now).days
if days_remaining in authority_expiration_intervals:
certs.append(c)
return certs
def get_eligible_certificates(exclude=None):
"""
Finds all certificates that are eligible for certificate expiration notification.
@ -90,6 +152,37 @@ def get_eligible_certificates(exclude=None):
return certificates
def get_eligible_security_summary_certs(exclude=None):
certificates = defaultdict(list)
all_certs = get_certificates_for_security_summary_email(exclude=exclude)
now = arrow.utcnow()
# group by expiration interval
for interval, interval_certs in groupby(all_certs, lambda x: (x.not_after - now).days):
certificates[interval] = list(interval_certs)
return certificates
def get_eligible_authority_certificates():
"""
Finds all certificate authority certificates that are eligible for certificate expiration notification.
Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs.
:return:
"""
certificates = defaultdict(dict)
all_certs = get_expiring_authority_certificates()
now = arrow.utcnow()
# group by owner
for owner, owner_certs in groupby(all_certs, lambda x: x.owner):
# group by expiration interval
for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days):
certificates[owner][interval] = list(interval_certs)
return certificates
def send_plugin_notification(event_type, data, recipients, notification):
"""
Executes the plugin and handles failure.
@ -103,8 +196,9 @@ def send_plugin_notification(event_type, data, recipients, notification):
function = f"{__name__}.{sys._getframe().f_code.co_name}"
log_data = {
"function": function,
"message": f"Sending expiration notification for to recipients {recipients}",
"notification_type": "expiration",
"message": f"Sending {event_type} notification for to recipients {recipients}",
"notification_type": event_type,
"notification_plugin": notification.plugin.slug,
"certificate_targets": recipients,
"plugin": notification.plugin.slug,
}
@ -143,7 +237,6 @@ def send_expiration_notifications(exclude):
for notification_label, certificates in notification_group.items():
notification_data = []
security_data = []
notification = certificates[0][0]
@ -153,33 +246,60 @@ def send_expiration_notifications(exclude):
certificate
).data
notification_data.append(cert_data)
security_data.append(cert_data)
if send_default_notification(
"expiration", notification_data, [owner], notification.options
):
success += 1
else:
failure += 1
recipients = notification.plugin.filter_recipients(notification.options, security_email + [owner])
email_recipients = notification.plugin.get_recipients(notification.options, security_email + [owner])
# Plugin will ONLY use the provided recipients if it's email; any other notification plugin ignores them
if send_plugin_notification(
"expiration",
notification_data,
recipients,
notification,
"expiration", notification_data, email_recipients, notification
):
success += 1
success += len(email_recipients)
else:
failure += 1
failure += len(email_recipients)
# If we're using an email plugin, we're done,
# since "security_email + [owner]" were added as email_recipients.
# If we're not using an email plugin, we also need to send an email to the security team and owner,
# since the plugin notification didn't send anything to them.
if notification.plugin.slug != "email-notification":
if send_default_notification(
"expiration", notification_data, email_recipients, notification.options
):
success = 1 + len(email_recipients)
else:
failure = 1 + len(email_recipients)
return success, failure
def send_authority_expiration_notifications():
"""
This function will check for upcoming certificate authority certificate expiration,
and send out notification emails at configured intervals.
"""
success = failure = 0
# security team gets all
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
for owner, owner_cert_groups in get_eligible_authority_certificates().items():
for interval, certificates in owner_cert_groups.items():
notification_data = []
for certificate in certificates:
cert_data = certificate_notification_output_schema.dump(
certificate
).data
cert_data['self_signed'] = is_selfsigned(certificate.parsed_cert)
cert_data['issued_cert_count'] = certificates_service.get_issued_cert_count_for_authority(certificate.root_authority)
notification_data.append(cert_data)
email_recipients = security_email + [owner]
if send_default_notification(
"expiration", security_data, security_email, notification.options
"authority_expiration", notification_data, email_recipients,
notification_options=[{'name': 'interval', 'value': interval}]
):
success += 1
success = len(email_recipients)
else:
failure += 1
failure = len(email_recipients)
return success, failure
@ -196,15 +316,16 @@ def send_default_notification(notification_type, data, targets, notification_opt
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
log_data = {
"function": function,
"message": f"Sending notification for certificate data {data}",
"notification_type": notification_type,
}
status = FAILURE_METRIC_STATUS
notification_plugin = plugins.get(
current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
)
log_data = {
"function": function,
"message": f"Sending {notification_type} notification for certificate data {data} to targets {targets}",
"notification_type": notification_type,
"notification_plugin": notification_plugin.slug,
}
try:
current_app.logger.debug(log_data)
@ -213,7 +334,7 @@ def send_default_notification(notification_type, data, targets, notification_opt
status = SUCCESS_METRIC_STATUS
except Exception as e:
log_data["message"] = f"Unable to send {notification_type} notification for certificate data {data} " \
f"to target {targets}"
f"to targets {targets}"
current_app.logger.error(log_data, exc_info=True)
sentry.captureException()
@ -221,7 +342,7 @@ def send_default_notification(notification_type, data, targets, notification_opt
"notification",
"counter",
1,
metric_tags={"status": status, "event_type": notification_type},
metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug},
)
if status == SUCCESS_METRIC_STATUS:
@ -248,15 +369,14 @@ def send_pending_failure_notification(
data = pending_certificate_output_schema.dump(pending_cert).data
data["security_email"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
notify_owner_success = False
email_recipients = []
if notify_owner:
notify_owner_success = send_default_notification("failed", data, [data["owner"]], pending_cert)
email_recipients = email_recipients + [data["owner"]]
notify_security_success = False
if notify_security:
notify_security_success = send_default_notification("failed", data, data["security_email"], pending_cert)
email_recipients = email_recipients + data["security_email"]
return notify_owner_success or notify_security_success
return send_default_notification("failed", data, email_recipients, pending_cert)
def needs_notification(certificate):
@ -296,3 +416,59 @@ def needs_notification(certificate):
if days == interval:
notifications.append(notification)
return notifications
def send_security_expiration_summary(exclude=None):
"""
Sends a report to the security team with a summary of all expiring certificates.
All expiring certificates are included here, regardless of notification configuration.
Certificates with notifications disabled are omitted.
:param exclude:
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
status = FAILURE_METRIC_STATUS
notification_plugin = plugins.get(
current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
)
notification_type = "expiration_summary"
log_data = {
"function": function,
"message": "Sending expiration summary notification for to security team",
"notification_type": notification_type,
"notification_plugin": notification_plugin.slug,
}
intervals_and_certs = get_eligible_security_summary_certs(exclude)
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
try:
current_app.logger.debug(log_data)
message_data = []
for interval, certs in intervals_and_certs.items():
cert_data = []
for certificate in certs:
cert_data.append(certificate_notification_output_schema.dump(certificate).data)
interval_data = {"interval": interval, "certificates": cert_data}
message_data.append(interval_data)
notification_plugin.send(notification_type, message_data, security_email, None)
status = SUCCESS_METRIC_STATUS
except Exception:
log_data["message"] = f"Unable to send {notification_type} notification for certificates " \
f"{intervals_and_certs} to targets {security_email}"
current_app.logger.error(log_data, exc_info=True)
sentry.captureException()
metrics.send(
"notification",
"counter",
1,
metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug},
)
if status == SUCCESS_METRIC_STATUS:
return True

6
lemur/notifications/service.py
View File

@ -94,7 +94,7 @@ def create(label, plugin_name, options, description, certificates):
:param options:
:param description:
:param certificates:
:rtype : Notification
:rtype: Notification
:return:
"""
notification = Notification(
@ -117,7 +117,7 @@ def update(notification_id, label, plugin_name, options, description, active, ce
:param certificates:
:param added_certificates:
:param removed_certificates:
:rtype : Notification
:rtype: Notification
:return:
"""
notification = get(notification_id)
@ -150,7 +150,7 @@ def get(notification_id):
Retrieves an notification by its lemur assigned ID.
:param notification_id: Lemur assigned ID
:rtype : Notification
:rtype: Notification
:return:
"""
return database.get(Notification, notification_id)

2
lemur/notifications/views.py
View File

@ -126,6 +126,7 @@ class NotificationsList(AuthenticatedResource):
POST /notifications HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"description": "a test",
@ -314,6 +315,7 @@ class Notifications(AuthenticatedResource):
POST /notifications/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
**Example response**:

4
lemur/pending_certificates/cli.py
View File

@ -12,10 +12,12 @@ from flask import current_app
from flask_script import Manager
from lemur.authorities.service import get as get_authority
from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
manager = Manager(usage="Handles pending certificate related tasks.")
@ -107,7 +109,7 @@ def fetch_all_acme():
error_log["last_error"] = cert.get("last_error")
error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > 4:
if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
error_log["message"] = "Marking pending certificate as resolved"
send_pending_failure_notification(
pending_cert, notify_owner=pending_cert.notify

2
lemur/pending_certificates/views.py
View File

@ -224,6 +224,7 @@ class PendingCertificates(AuthenticatedResource):
PUT /pending certificates/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"owner": "jimbob@example.com",
@ -465,6 +466,7 @@ class PendingCertificatesUpload(AuthenticatedResource):
POST /certificates/1/upload HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"body": "-----BEGIN CERTIFICATE-----...",

1
lemur/plugins/bases/__init__.py
View File

@ -3,3 +3,4 @@ from .issuer import IssuerPlugin # noqa
from .source import SourcePlugin # noqa
from .notification import NotificationPlugin, ExpirationNotificationPlugin # noqa
from .export import ExportPlugin # noqa
from .tls import TLSPlugin # noqa

2
lemur/plugins/bases/issuer.py
View File

@ -23,7 +23,7 @@ class IssuerPlugin(Plugin):
def create_authority(self, options):
raise NotImplementedError
def revoke_certificate(self, certificate, comments):
def revoke_certificate(self, certificate, reason):
raise NotImplementedError
def get_ordered_certificate(self, certificate):

10
lemur/plugins/bases/notification.py
View File

@ -20,14 +20,14 @@ class NotificationPlugin(Plugin):
def send(self, notification_type, message, targets, options, **kwargs):
raise NotImplementedError
def filter_recipients(self, options, excluded_recipients):
def get_recipients(self, options, additional_recipients):
"""
Given a set of options (which should include configured recipient info), filters out recipients that
we do NOT want to notify.
Given a set of options (which should include configured recipient info), returns the parsed list of recipients
from those options plus the additional recipients specified. The returned value has no duplicates.
For any notification types where recipients can't be dynamically modified, this returns an empty list.
For any notification types where recipients can't be dynamically modified, this returns only the additional recipients.
"""
return []
return additional_recipients
class ExpirationNotificationPlugin(NotificationPlugin):

20
lemur/plugins/bases/tls.py Normal file
View File

@ -0,0 +1,20 @@
"""
.. module: lemur.plugins.bases.tls
:platform: Unix
:copyright: (c) 2021 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Sayali Charhate <scharhate@netflix.com>
"""
from lemur.plugins.base import Plugin
class TLSPlugin(Plugin):
"""
This is the base class from which all supported
tls session providers will inherit from.
"""
type = "tls"
def session(self, server_application):
raise NotImplementedError

524
lemur/plugins/lemur_acme/acme_handlers.py Normal file
View File

@ -0,0 +1,524 @@
"""
.. module: lemur.plugins.lemur_acme.plugin
:platform: Unix
:synopsis: This module contains handlers for certain acme related tasks. It needed to be refactored to avoid circular imports
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
.. moduleauthor:: Mathias Petermann <mathias.petermann@projektfokus.ch>
"""
import datetime
import json
import time
import OpenSSL.crypto
import josepy as jose
import dns.resolver
from acme import challenges, errors, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork
from acme.errors import TimeoutError
from acme.messages import Error as AcmeError
from flask import current_app
from lemur.common.utils import generate_private_key
from lemur.dns_providers import service as dns_provider_service
from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration
from lemur.extensions import metrics, sentry
from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns
from lemur.authorities import service as authorities_service
from retrying import retry
class AuthorizationRecord(object):
def __init__(self, domain, target_domain, authz, dns_challenge, change_id, cname_delegation):
self.domain = domain
self.target_domain = target_domain
self.authz = authz
self.dns_challenge = dns_challenge
self.change_id = change_id
self.cname_delegation = cname_delegation
class AcmeHandler(object):
def reuse_account(self, authority):
if not authority.options:
raise InvalidAuthority("Invalid authority. Options not set")
existing_key = False
existing_regr = False
for option in json.loads(authority.options):
if option["name"] == "acme_private_key" and option["value"]:
existing_key = True
if option["name"] == "acme_regr" and option["value"]:
existing_regr = True
if not existing_key and current_app.config.get("ACME_PRIVATE_KEY"):
existing_key = True
if not existing_regr and current_app.config.get("ACME_REGR"):
existing_regr = True
if existing_key and existing_regr:
return True
else:
return False
def strip_wildcard(self, host):
"""Removes the leading *. and returns Host and whether it was removed or not (True/False)"""
prefix = "*."
if host.startswith(prefix):
return host[len(prefix):], True
return host, False
def maybe_add_extension(self, host, dns_provider_options):
if dns_provider_options and dns_provider_options.get(
"acme_challenge_extension"
):
host = host + dns_provider_options.get("acme_challenge_extension")
return host
def request_certificate(self, acme_client, authorizations, order):
for authorization in authorizations:
for authz in authorization.authz:
authorization_resource, _ = acme_client.poll(authz)
deadline = datetime.datetime.now() + datetime.timedelta(seconds=360)
try:
orderr = acme_client.poll_and_finalize(order, deadline)
except (AcmeError, TimeoutError):
sentry.captureException(extra={"order_url": str(order.uri)})
metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri})
current_app.logger.error(
f"Unable to resolve Acme order: {order.uri}", exc_info=True
)
raise
except errors.ValidationError:
if order.fullchain_pem:
orderr = order
else:
raise
metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri})
current_app.logger.info(
f"Successfully resolved Acme order: {order.uri}", exc_info=True
)
pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem)
current_app.logger.debug(
"{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))
)
return pem_certificate, pem_certificate_chain
def extract_cert_and_chain(self, fullchain_pem):
pem_certificate = OpenSSL.crypto.dump_certificate(
OpenSSL.crypto.FILETYPE_PEM,
OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM, fullchain_pem
),
).decode()
if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \
and datetime.datetime.now() < datetime.datetime.strptime(
current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'):
pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA")
else:
pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip()
return pem_certificate, pem_certificate_chain
@retry(stop_max_attempt_number=5, wait_fixed=5000)
def setup_acme_client(self, authority):
if not authority.options:
raise InvalidAuthority("Invalid authority. Options not set")
options = {}
for option in json.loads(authority.options):
options[option["name"]] = option.get("value")
email = options.get("email", current_app.config.get("ACME_EMAIL"))
tel = options.get("telephone", current_app.config.get("ACME_TEL"))
directory_url = options.get(
"acme_url", current_app.config.get("ACME_DIRECTORY_URL")
)
existing_key = options.get(
"acme_private_key", current_app.config.get("ACME_PRIVATE_KEY")
)
existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR"))
if existing_key and existing_regr:
current_app.logger.debug("Reusing existing ACME account")
# Reuse the same account for each certificate issuance
key = jose.JWK.json_loads(existing_key)
regr = messages.RegistrationResource.json_loads(existing_regr)
current_app.logger.debug(
"Connecting with directory at {0}".format(directory_url)
)
net = ClientNetwork(key, account=regr)
client = BackwardsCompatibleClientV2(net, key, directory_url)
return client, {}
else:
# Create an account for each certificate issuance
key = jose.JWKRSA(key=generate_private_key("RSA2048"))
current_app.logger.debug("Creating a new ACME account")
current_app.logger.debug(
"Connecting with directory at {0}".format(directory_url)
)
net = ClientNetwork(key, account=None, timeout=3600)
client = BackwardsCompatibleClientV2(net, key, directory_url)
registration = client.new_account_and_tos(
messages.NewRegistration.from_data(email=email)
)
# if store_account is checked, add the private_key and registration resources to the options
if options['store_account']:
new_options = json.loads(authority.options)
# the key returned by fields_to_partial_json is missing the key type, so we add it manually
key_dict = key.fields_to_partial_json()
key_dict["kty"] = "RSA"
acme_private_key = {
"name": "acme_private_key",
"value": json.dumps(key_dict)
}
new_options.append(acme_private_key)
acme_regr = {
"name": "acme_regr",
"value": json.dumps({"body": {}, "uri": registration.uri})
}
new_options.append(acme_regr)
authorities_service.update_options(authority.id, options=json.dumps(new_options))
current_app.logger.debug("Connected: {0}".format(registration.uri))
return client, registration
def get_domains(self, options):
"""
Fetches all domains currently requested
:param options:
:return:
"""
current_app.logger.debug("Fetching domains")
domains = [options["common_name"]]
if options.get("extensions"):
for dns_name in options["extensions"]["sub_alt_names"]["names"]:
if dns_name.value not in domains:
domains.append(dns_name.value)
current_app.logger.debug("Got these domains: {0}".format(domains))
return domains
def revoke_certificate(self, certificate, crl_reason=0):
if not self.reuse_account(certificate.authority):
raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.")
acme_client, _ = self.setup_acme_client(certificate.authority)
fullchain_com = jose.ComparableX509(
OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM, certificate.body))
try:
acme_client.revoke(fullchain_com, crl_reason) # revocation reason as int (per RFC 5280 section 5.3.1)
except (errors.ConflictError, errors.ClientError, errors.Error) as e:
# Certificate already revoked.
current_app.logger.error("Certificate revocation failed with message: " + e.detail)
metrics.send("acme_revoke_certificate_failure", "counter", 1)
return False
current_app.logger.warning("Certificate succesfully revoked: " + certificate.name)
metrics.send("acme_revoke_certificate_success", "counter", 1)
return True
class AcmeDnsHandler(AcmeHandler):
def __init__(self):
self.dns_providers_for_domain = {}
try:
self.all_dns_providers = dns_provider_service.get_all_dns_providers()
except Exception as e:
metrics.send("AcmeHandler_init_error", "counter", 1)
sentry.captureException()
current_app.logger.error(f"Unable to fetch DNS Providers: {e}")
self.all_dns_providers = []
def get_all_zones(self, dns_provider):
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
return dns_provider_plugin.get_zones(account_number=account_number)
def get_dns_challenges(self, host, authorizations):
"""Get dns challenges for provided domain"""
domain_to_validate, is_wildcard = self.strip_wildcard(host)
dns_challenges = []
for authz in authorizations:
if not authz.body.identifier.value.lower() == domain_to_validate.lower():
continue
if is_wildcard and not authz.body.wildcard:
continue
if not is_wildcard and authz.body.wildcard:
continue
for combo in authz.body.challenges:
if isinstance(combo.chall, challenges.DNS01):
dns_challenges.append(combo)
return dns_challenges
def get_dns_provider(self, type):
provider_types = {
"cloudflare": cloudflare,
"dyn": dyn,
"route53": route53,
"ultradns": ultradns,
"powerdns": powerdns
}
provider = provider_types.get(type)
if not provider:
raise UnknownProvider("No such DNS provider: {}".format(type))
return provider
def start_dns_challenge(
self,
acme_client,
account_number,
domain,
target_domain,
dns_provider,
order,
dns_provider_options,
):
current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.")
change_ids = []
cname_delegation = domain != target_domain
dns_challenges = self.get_dns_challenges(domain, order.authorizations)
host_to_validate, _ = self.strip_wildcard(target_domain)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if not dns_challenges:
sentry.captureException()
metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1)
raise Exception("Unable to determine DNS challenges from authorizations")
for dns_challenge in dns_challenges:
if not cname_delegation:
host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
change_id = dns_provider.create_txt_record(
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
account_number,
)
change_ids.append(change_id)
return AuthorizationRecord(
domain, target_domain, order.authorizations, dns_challenges, change_ids, cname_delegation
)
def complete_dns_challenge(self, acme_client, authz_record):
current_app.logger.debug(
"Finalizing DNS challenge for {0}".format(
authz_record.authz[0].body.identifier.value
)
)
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
if not dns_providers:
metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1)
raise Exception(
"No DNS providers found for domain: {}".format(authz_record.target_domain)
)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
for change_id in authz_record.change_id:
try:
dns_provider_plugin.wait_for_dns_change(
change_id, account_number=account_number
)
except Exception:
metrics.send("complete_dns_challenge_error", "counter", 1)
sentry.captureException()
current_app.logger.debug(
f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
f"{account_number}",
exc_info=True,
)
raise
for dns_challenge in authz_record.dns_challenge:
response = dns_challenge.response(acme_client.client.net.key)
verified = response.simple_verify(
dns_challenge.chall,
authz_record.target_domain,
acme_client.client.net.key.public_key(),
)
if not verified:
metrics.send("complete_dns_challenge_verification_error", "counter", 1)
raise ValueError("Failed verification")
time.sleep(5)
res = acme_client.answer_challenge(dns_challenge, response)
current_app.logger.debug(f"answer_challenge response: {res}")
def get_authorizations(self, acme_client, order, order_info):
authorizations = []
for domain in order_info.domains:
# If CNAME exists, set host to the target address
target_domain = domain
if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False):
cname_result, _ = self.strip_wildcard(domain)
cname_result = challenges.DNS01().validation_domain_name(cname_result)
cname_result = self.get_cname(cname_result)
if cname_result:
target_domain = cname_result
self.autodetect_dns_providers(target_domain)
metrics.send(
"get_authorizations_cname_delegation_for_domain", "counter", 1, metric_tags={"domain": domain}
)
if not self.dns_providers_for_domain.get(target_domain):
metrics.send(
"get_authorizations_no_dns_provider_for_domain", "counter", 1
)
raise Exception("No DNS providers found for domain: {}".format(target_domain))
for dns_provider in self.dns_providers_for_domain[target_domain]:
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
authz_record = self.start_dns_challenge(
acme_client,
account_number,
domain,
target_domain,
dns_provider_plugin,
order,
dns_provider.options,
)
authorizations.append(authz_record)
return authorizations
def autodetect_dns_providers(self, domain):
"""
Get DNS providers associated with a domain when it has not been provided for certificate creation.
:param domain:
:return: dns_providers: List of DNS providers that have the correct zone.
"""
self.dns_providers_for_domain[domain] = []
match_length = 0
for dns_provider in self.all_dns_providers:
if not dns_provider.domains:
continue
for name in dns_provider.domains:
if name == domain or domain.endswith("." + name):
if len(name) > match_length:
self.dns_providers_for_domain[domain] = [dns_provider]
match_length = len(name)
elif len(name) == match_length:
self.dns_providers_for_domain[domain].append(dns_provider)
return self.dns_providers_for_domain
def finalize_authorizations(self, acme_client, authorizations):
for authz_record in authorizations:
self.complete_dns_challenge(acme_client, authz_record)
for authz_record in authorizations:
dns_challenges = authz_record.dns_challenge
for dns_challenge in dns_challenges:
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_plugin = self.get_dns_provider(
dns_provider.provider_type
)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if not authz_record.cname_delegation:
host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate)
dns_provider_plugin.delete_txt_record(
authz_record.change_id,
account_number,
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
)
return authorizations
def cleanup_dns_challenges(self, acme_client, authorizations):
"""
Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
on an exception
:param acme_client:
:param account_number:
:param dns_provider:
:param authorizations:
:param dns_provider_options:
:return:
"""
for authz_record in authorizations:
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_challenges = authz_record.dns_challenge
host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
host_to_validate = self.maybe_add_extension(
host_to_validate, dns_provider_options
)
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
for dns_challenge in dns_challenges:
if not authz_record.cname_delegation:
host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
try:
dns_provider_plugin.delete_txt_record(
authz_record.change_id,
account_number,
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
)
except Exception as e:
# If this fails, it's most likely because the record doesn't exist (It was already cleaned up)
# or we're not authorized to modify it.
metrics.send("cleanup_dns_challenges_error", "counter", 1)
sentry.captureException()
pass
def get_cname(self, domain):
"""
:param domain: Domain name to look up a CNAME for.
:return: First CNAME target or False if no CNAME record exists.
"""
try:
result = dns.resolver.query(domain, 'CNAME')
if len(result) > 0:
return str(result[0].target).rstrip('.')
except dns.exception.DNSException:
return False

260
lemur/plugins/lemur_acme/challenge_types.py Normal file
View File

@ -0,0 +1,260 @@
"""
.. module: lemur.plugins.lemur_acme.plugin
:platform: Unix
:synopsis: This module contains the different challenge types for ACME implementations
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Mathias Petermann <mathias.petermann@projektfokus.ch>
"""
import datetime
import json
from acme import challenges
from acme.messages import errors, STATUS_VALID, ERROR_CODES
from flask import current_app
from lemur.authorizations import service as authorization_service
from lemur.exceptions import LemurException, InvalidConfiguration
from lemur.plugins.base import plugins
from lemur.destinations import service as destination_service
from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler
class AcmeChallengeMissmatchError(LemurException):
pass
class AcmeChallenge(object):
"""
This is the base class, all ACME challenges will need to extend, allowing for future extendability
"""
def create_certificate(self, csr, issuer_options):
"""
Create the new certificate, using the provided CSR and issuer_options.
Right now this is basically a copy of the create_certificate methods in the AcmeHandlers, but should be cleaned
and tried to make use of the deploy and cleanup methods
:param csr:
:param issuer_options:
:return:
"""
pass
def deploy(self, challenge, acme_client, validation_target):
"""
In here the challenge validation is fetched and deployed somewhere that it can be validated by the provider
:param self:
:param challenge: the challenge object, must match for the challenge implementation
:param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on
:param validation_target: an identifier for the validation target, e.g. the name of a DNS provider
"""
raise NotImplementedError
def cleanup(self, challenge, acme_client, validation_target):
"""
Ideally the challenge should be cleaned up, after the validation is done
:param challenge: Needed to identify the challenge to be removed
:param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on
:param validation_target: Needed to remove the validation
"""
raise NotImplementedError
class AcmeHttpChallenge(AcmeChallenge):
challengeType = challenges.HTTP01
def create_certificate(self, csr, issuer_options):
"""
Creates an ACME certificate using the HTTP-01 challenge.
:param csr:
:param issuer_options:
:return: :raise Exception:
"""
self.acme = AcmeHandler()
authority = issuer_options.get("authority")
acme_client, registration = self.acme.setup_acme_client(authority)
orderr = acme_client.new_order(csr)
chall = []
deployed_challenges = []
all_pre_validated = True
for authz in orderr.authorizations:
# Choosing challenge.
# check if authorizations is already in a valid state
if authz.body.status != STATUS_VALID:
all_pre_validated = False
# authz.body.challenges is a set of ChallengeBody objects.
for i in authz.body.challenges:
# Find the supported challenge.
if isinstance(i.chall, challenges.HTTP01):
chall.append(i)
else:
current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value))
if len(chall) == 0 and not all_pre_validated:
raise Exception('HTTP-01 challenge was not offered by the CA server at {}'.format(orderr.uri))
elif not all_pre_validated:
validation_target = None
for option in json.loads(issuer_options["authority"].options):
if option["name"] == "tokenDestination":
validation_target = option["value"]
if validation_target is None:
raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge')
for challenge in chall:
try:
response = self.deploy(challenge, acme_client, validation_target)
deployed_challenges.append(challenge.chall.path)
acme_client.answer_challenge(challenge, response)
except Exception as e:
current_app.logger.error(e)
raise Exception('Failure while trying to deploy token to configure destination. See logs for more information')
current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order")
try:
finalized_orderr = acme_client.poll_and_finalize(orderr,
datetime.datetime.now() + datetime.timedelta(seconds=90))
except errors.ValidationError as validationError:
for authz in validationError.failed_authzrs:
for chall in authz.body.challenges:
if chall.error:
current_app.logger.error(
"ValidationError occured of type {}, with message {}".format(chall.error.typ,
ERROR_CODES[chall.error.code]))
raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.')
pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem)
if len(deployed_challenges) != 0:
for token_path in deployed_challenges:
self.cleanup(token_path, validation_target)
# validation is a random string, we use it as external id, to make it possible to implement revoke_certificate
return pem_certificate, pem_certificate_chain, None
def deploy(self, challenge, acme_client, validation_target):
if not isinstance(challenge.chall, challenges.HTTP01):
raise AcmeChallengeMissmatchError(
'The provided challenge is not of type HTTP01, but instead of type {}'.format(
challenge.__class__.__name__))
destination = destination_service.get(validation_target)
if destination is None:
raise Exception(
'Couldn\'t find the destination with name {}. Cant complete HTTP01 challenge'.format(validation_target))
destination_plugin = plugins.get(destination.plugin_name)
response, validation = challenge.response_and_validation(acme_client.net.key)
destination_plugin.upload_acme_token(challenge.chall.path, validation, destination.options)
current_app.logger.info("Uploaded HTTP-01 challenge token.")
return response
def cleanup(self, token_path, validation_target):
destination = destination_service.get(validation_target)
if destination is None:
current_app.logger.info(
'Couldn\'t find the destination with name {}, won\'t cleanup the challenge'.format(validation_target))
destination_plugin = plugins.get(destination.plugin_name)
destination_plugin.delete_acme_token(token_path, destination.options)
current_app.logger.info("Cleaned up HTTP-01 challenge token.")
class AcmeDnsChallenge(AcmeChallenge):
challengeType = challenges.DNS01
def create_certificate(self, csr, issuer_options):
"""
Creates an ACME certificate.
:param csr:
:param issuer_options:
:return: :raise Exception:
"""
self.acme = AcmeDnsHandler()
authority = issuer_options.get("authority")
create_immediately = issuer_options.get("create_immediately", False)
acme_client, registration = self.acme.setup_acme_client(authority)
dns_provider = issuer_options.get("dns_provider", {})
if dns_provider:
dns_provider_options = dns_provider.options
credentials = json.loads(dns_provider.credentials)
current_app.logger.debug(
"Using DNS provider: {0}".format(dns_provider.provider_type)
)
dns_provider_plugin = __import__(
dns_provider.provider_type, globals(), locals(), [], 1
)
account_number = credentials.get("account_id")
provider_type = dns_provider.provider_type
if provider_type == "route53" and not account_number:
error = "Route53 DNS Provider {} does not have an account number configured.".format(
dns_provider.name
)
current_app.logger.error(error)
raise InvalidConfiguration(error)
else:
dns_provider = {}
dns_provider_options = None
account_number = None
provider_type = None
domains = self.acme.get_domains(issuer_options)
if not create_immediately:
# Create pending authorizations that we'll need to do the creation
dns_authorization = authorization_service.create(
account_number, domains, provider_type
)
# Return id of the DNS Authorization
return None, None, dns_authorization.id
authorizations = self.acme.get_authorizations(
acme_client,
account_number,
domains,
dns_provider_plugin,
dns_provider_options,
)
self.acme.finalize_authorizations(
acme_client,
account_number,
dns_provider_plugin,
authorizations,
dns_provider_options,
)
pem_certificate, pem_certificate_chain = self.acme.request_certificate(
acme_client, authorizations, csr
)
# TODO add external ID (if possible)
return pem_certificate, pem_certificate_chain, None
def deploy(self, challenge, acme_client, validation_target):
pass
def cleanup(self, authorizations, acme_client, validation_target):
"""
Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
on an exception
:param authorizations: all the authorizations to be cleaned up
:param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on
:param validation_target: Unused right now
:return:
"""
acme = AcmeDnsHandler()
acme.cleanup_dns_challenges(acme_client, authorizations)

656
lemur/plugins/lemur_acme/plugin.py
View File

@ -11,465 +11,28 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
"""
import datetime
import json
import time
import OpenSSL.crypto
import dns.resolver
import josepy as jose
from acme import challenges, errors, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork
from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
from acme.errors import PollError, WildcardUnsupportedError
from acme.messages import Error as AcmeError
from botocore.exceptions import ClientError
from flask import current_app
from lemur.authorizations import service as authorization_service
from lemur.common.utils import generate_private_key
from lemur.constants import CRLReason
from lemur.dns_providers import service as dns_provider_service
from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider
from lemur.exceptions import InvalidConfiguration
from lemur.extensions import metrics, sentry
from lemur.plugins import lemur_acme as acme
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns
from lemur.authorities import service as authorities_service
from retrying import retry
class AuthorizationRecord(object):
def __init__(self, domain, target_domain, authz, dns_challenge, change_id):
self.domain = domain
self.target_domain = target_domain
self.authz = authz
self.dns_challenge = dns_challenge
self.change_id = change_id
class AcmeHandler(object):
def __init__(self):
self.dns_providers_for_domain = {}
try:
self.all_dns_providers = dns_provider_service.get_all_dns_providers()
except Exception as e:
metrics.send("AcmeHandler_init_error", "counter", 1)
sentry.captureException()
current_app.logger.error(f"Unable to fetch DNS Providers: {e}")
self.all_dns_providers = []
def get_dns_challenges(self, host, authorizations):
"""Get dns challenges for provided domain"""
domain_to_validate, is_wildcard = self.strip_wildcard(host)
dns_challenges = []
for authz in authorizations:
if not authz.body.identifier.value.lower() == domain_to_validate.lower():
continue
if is_wildcard and not authz.body.wildcard:
continue
if not is_wildcard and authz.body.wildcard:
continue
for combo in authz.body.challenges:
if isinstance(combo.chall, challenges.DNS01):
dns_challenges.append(combo)
return dns_challenges
def strip_wildcard(self, host):
"""Removes the leading *. and returns Host and whether it was removed or not (True/False)"""
prefix = "*."
if host.startswith(prefix):
return host[len(prefix):], True
return host, False
def maybe_add_extension(self, host, dns_provider_options):
if dns_provider_options and dns_provider_options.get(
"acme_challenge_extension"
):
host = host + dns_provider_options.get("acme_challenge_extension")
return host
def start_dns_challenge(
self,
acme_client,
account_number,
domain,
target_domain,
dns_provider,
order,
dns_provider_options,
):
current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.")
change_ids = []
dns_challenges = self.get_dns_challenges(domain, order.authorizations)
host_to_validate, _ = self.strip_wildcard(target_domain)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if not dns_challenges:
sentry.captureException()
metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1)
raise Exception("Unable to determine DNS challenges from authorizations")
for dns_challenge in dns_challenges:
# Only prepend '_acme-challenge' if not using CNAME redirection
if domain == target_domain:
host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
change_id = dns_provider.create_txt_record(
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
account_number,
)
change_ids.append(change_id)
return AuthorizationRecord(
domain, target_domain, order.authorizations, dns_challenges, change_ids
)
def complete_dns_challenge(self, acme_client, authz_record):
current_app.logger.debug(
"Finalizing DNS challenge for {0}".format(
authz_record.authz[0].body.identifier.value
)
)
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
if not dns_providers:
metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1)
raise Exception(
"No DNS providers found for domain: {}".format(authz_record.target_domain)
)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
for change_id in authz_record.change_id:
try:
dns_provider_plugin.wait_for_dns_change(
change_id, account_number=account_number
)
except Exception:
metrics.send("complete_dns_challenge_error", "counter", 1)
sentry.captureException()
current_app.logger.debug(
f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
f"{account_number}",
exc_info=True,
)
raise
for dns_challenge in authz_record.dns_challenge:
response = dns_challenge.response(acme_client.client.net.key)
verified = response.simple_verify(
dns_challenge.chall,
authz_record.target_domain,
acme_client.client.net.key.public_key(),
)
if not verified:
metrics.send("complete_dns_challenge_verification_error", "counter", 1)
raise ValueError("Failed verification")
time.sleep(5)
res = acme_client.answer_challenge(dns_challenge, response)
current_app.logger.debug(f"answer_challenge response: {res}")
def request_certificate(self, acme_client, authorizations, order):
for authorization in authorizations:
for authz in authorization.authz:
authorization_resource, _ = acme_client.poll(authz)
deadline = datetime.datetime.now() + datetime.timedelta(seconds=360)
try:
orderr = acme_client.poll_and_finalize(order, deadline)
except (AcmeError, TimeoutError):
sentry.captureException(extra={"order_url": str(order.uri)})
metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri})
current_app.logger.error(
f"Unable to resolve Acme order: {order.uri}", exc_info=True
)
raise
except errors.ValidationError:
if order.fullchain_pem:
orderr = order
else:
raise
metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri})
current_app.logger.info(
f"Successfully resolved Acme order: {order.uri}", exc_info=True
)
pem_certificate = OpenSSL.crypto.dump_certificate(
OpenSSL.crypto.FILETYPE_PEM,
OpenSSL.crypto.load_certificate(
OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem
),
).decode()
if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \
and datetime.datetime.now() < datetime.datetime.strptime(
current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'):
pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA")
else:
pem_certificate_chain = orderr.fullchain_pem[
len(pem_certificate) : # noqa
].lstrip()
current_app.logger.debug(
"{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))
)
return pem_certificate, pem_certificate_chain
@retry(stop_max_attempt_number=5, wait_fixed=5000)
def setup_acme_client(self, authority):
if not authority.options:
raise InvalidAuthority("Invalid authority. Options not set")
options = {}
for option in json.loads(authority.options):
options[option["name"]] = option.get("value")
email = options.get("email", current_app.config.get("ACME_EMAIL"))
tel = options.get("telephone", current_app.config.get("ACME_TEL"))
directory_url = options.get(
"acme_url", current_app.config.get("ACME_DIRECTORY_URL")
)
existing_key = options.get(
"acme_private_key", current_app.config.get("ACME_PRIVATE_KEY")
)
existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR"))
if existing_key and existing_regr:
current_app.logger.debug("Reusing existing ACME account")
# Reuse the same account for each certificate issuance
key = jose.JWK.json_loads(existing_key)
regr = messages.RegistrationResource.json_loads(existing_regr)
current_app.logger.debug(
"Connecting with directory at {0}".format(directory_url)
)
net = ClientNetwork(key, account=regr)
client = BackwardsCompatibleClientV2(net, key, directory_url)
return client, {}
else:
# Create an account for each certificate issuance
key = jose.JWKRSA(key=generate_private_key("RSA2048"))
current_app.logger.debug("Creating a new ACME account")
current_app.logger.debug(
"Connecting with directory at {0}".format(directory_url)
)
net = ClientNetwork(key, account=None, timeout=3600)
client = BackwardsCompatibleClientV2(net, key, directory_url)
registration = client.new_account_and_tos(
messages.NewRegistration.from_data(email=email)
)
# if store_account is checked, add the private_key and registration resources to the options
if options['store_account']:
new_options = json.loads(authority.options)
# the key returned by fields_to_partial_json is missing the key type, so we add it manually
key_dict = key.fields_to_partial_json()
key_dict["kty"] = "RSA"
acme_private_key = {
"name": "acme_private_key",
"value": json.dumps(key_dict)
}
new_options.append(acme_private_key)
acme_regr = {
"name": "acme_regr",
"value": json.dumps({"body": {}, "uri": registration.uri})
}
new_options.append(acme_regr)
authorities_service.update_options(authority.id, options=json.dumps(new_options))
current_app.logger.debug("Connected: {0}".format(registration.uri))
return client, registration
def get_domains(self, options):
"""
Fetches all domains currently requested
:param options:
:return:
"""
current_app.logger.debug("Fetching domains")
domains = [options["common_name"]]
if options.get("extensions"):
for dns_name in options["extensions"]["sub_alt_names"]["names"]:
if dns_name.value not in domains:
domains.append(dns_name.value)
current_app.logger.debug("Got these domains: {0}".format(domains))
return domains
def get_authorizations(self, acme_client, order, order_info):
authorizations = []
for domain in order_info.domains:
# If CNAME exists, set host to the target address
target_domain = domain
if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False):
cname_result, _ = self.strip_wildcard(domain)
cname_result = challenges.DNS01().validation_domain_name(cname_result)
cname_result = self.get_cname(cname_result)
if cname_result:
target_domain = cname_result
self.autodetect_dns_providers(target_domain)
if not self.dns_providers_for_domain.get(target_domain):
metrics.send(
"get_authorizations_no_dns_provider_for_domain", "counter", 1
)
raise Exception("No DNS providers found for domain: {}".format(target_domain))
for dns_provider in self.dns_providers_for_domain[target_domain]:
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
authz_record = self.start_dns_challenge(
acme_client,
account_number,
domain,
target_domain,
dns_provider_plugin,
order,
dns_provider.options,
)
authorizations.append(authz_record)
return authorizations
def autodetect_dns_providers(self, domain):
"""
Get DNS providers associated with a domain when it has not been provided for certificate creation.
:param domain:
:return: dns_providers: List of DNS providers that have the correct zone.
"""
self.dns_providers_for_domain[domain] = []
match_length = 0
for dns_provider in self.all_dns_providers:
if not dns_provider.domains:
continue
for name in dns_provider.domains:
if name == domain or domain.endswith("." + name):
if len(name) > match_length:
self.dns_providers_for_domain[domain] = [dns_provider]
match_length = len(name)
elif len(name) == match_length:
self.dns_providers_for_domain[domain].append(dns_provider)
return self.dns_providers_for_domain
def finalize_authorizations(self, acme_client, authorizations):
for authz_record in authorizations:
self.complete_dns_challenge(acme_client, authz_record)
for authz_record in authorizations:
dns_challenges = authz_record.dns_challenge
for dns_challenge in dns_challenges:
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_plugin = self.get_dns_provider(
dns_provider.provider_type
)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if authz_record.domain == authz_record.target_domain:
host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate)
dns_provider_plugin.delete_txt_record(
authz_record.change_id,
account_number,
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
)
return authorizations
def cleanup_dns_challenges(self, acme_client, authorizations):
"""
Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
on an exception
:param acme_client:
:param account_number:
:param dns_provider:
:param authorizations:
:param dns_provider_options:
:return:
"""
for authz_record in authorizations:
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_challenges = authz_record.dns_challenge
host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
host_to_validate = self.maybe_add_extension(
host_to_validate, dns_provider_options
)
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
for dns_challenge in dns_challenges:
if authz_record.domain == authz_record.target_domain:
host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
try:
dns_provider_plugin.delete_txt_record(
authz_record.change_id,
account_number,
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
)
except Exception as e:
# If this fails, it's most likely because the record doesn't exist (It was already cleaned up)
# or we're not authorized to modify it.
metrics.send("cleanup_dns_challenges_error", "counter", 1)
sentry.captureException()
pass
def get_dns_provider(self, type):
provider_types = {
"cloudflare": cloudflare,
"dyn": dyn,
"route53": route53,
"ultradns": ultradns,
"powerdns": powerdns
}
provider = provider_types.get(type)
if not provider:
raise UnknownProvider("No such DNS provider: {}".format(type))
return provider
def get_cname(self, domain):
"""
:param domain: Domain name to look up a CNAME for.
:return: First CNAME target or False if no CNAME record exists.
"""
try:
result = dns.resolver.query(domain, 'CNAME')
if len(result) > 0:
return str(result[0].target).rstrip('.')
except dns.exception.DNSException:
return False
from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler
from lemur.plugins.lemur_acme.challenge_types import AcmeHttpChallenge, AcmeDnsChallenge
class ACMEIssuerPlugin(IssuerPlugin):
title = "Acme"
slug = "acme-issuer"
description = (
"Enables the creation of certificates via ACME CAs (including Let's Encrypt)"
"Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the DNS-01 challenge"
)
version = acme.VERSION
@ -516,30 +79,8 @@ class ACMEIssuerPlugin(IssuerPlugin):
def __init__(self, *args, **kwargs):
super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
def get_dns_provider(self, type):
self.acme = AcmeHandler()
provider_types = {
"cloudflare": cloudflare,
"dyn": dyn,
"route53": route53,
"ultradns": ultradns,
"powerdns": powerdns
}
provider = provider_types.get(type)
if not provider:
raise UnknownProvider("No such DNS provider: {}".format(type))
return provider
def get_all_zones(self, dns_provider):
self.acme = AcmeHandler()
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
return dns_provider_plugin.get_zones(account_number=account_number)
def get_ordered_certificate(self, pending_cert):
self.acme = AcmeHandler()
self.acme = AcmeDnsHandler()
acme_client, registration = self.acme.setup_acme_client(pending_cert.authority)
order_info = authorization_service.get(pending_cert.external_id)
if pending_cert.dns_provider_id:
@ -585,7 +126,8 @@ class ACMEIssuerPlugin(IssuerPlugin):
return cert
def get_ordered_certificates(self, pending_certs):
self.acme = AcmeHandler()
self.acme = AcmeDnsHandler()
self.acme_dns_challenge = AcmeDnsChallenge()
pending = []
certs = []
for pending_cert in pending_certs:
@ -682,76 +224,22 @@ class ACMEIssuerPlugin(IssuerPlugin):
}
)
# Ensure DNS records get deleted
self.acme.cleanup_dns_challenges(
entry["acme_client"], entry["authorizations"]
self.acme_dns_challenge.cleanup(
entry["authorizations"], entry["acme_client"]
)
return certs
def create_certificate(self, csr, issuer_options):
"""
Creates an ACME certificate.
Creates an ACME certificate using the DNS-01 challenge.
:param csr:
:param issuer_options:
:return: :raise Exception:
"""
self.acme = AcmeHandler()
authority = issuer_options.get("authority")
create_immediately = issuer_options.get("create_immediately", False)
acme_client, registration = self.acme.setup_acme_client(authority)
dns_provider = issuer_options.get("dns_provider", {})
acme_dns_challenge = AcmeDnsChallenge()
if dns_provider:
dns_provider_options = dns_provider.options
credentials = json.loads(dns_provider.credentials)
current_app.logger.debug(
"Using DNS provider: {0}".format(dns_provider.provider_type)
)
dns_provider_plugin = __import__(
dns_provider.provider_type, globals(), locals(), [], 1
)
account_number = credentials.get("account_id")
provider_type = dns_provider.provider_type
if provider_type == "route53" and not account_number:
error = "Route53 DNS Provider {} does not have an account number configured.".format(
dns_provider.name
)
current_app.logger.error(error)
raise InvalidConfiguration(error)
else:
dns_provider = {}
dns_provider_options = None
account_number = None
provider_type = None
domains = self.acme.get_domains(issuer_options)
if not create_immediately:
# Create pending authorizations that we'll need to do the creation
dns_authorization = authorization_service.create(
account_number, domains, provider_type
)
# Return id of the DNS Authorization
return None, None, dns_authorization.id
authorizations = self.acme.get_authorizations(
acme_client,
account_number,
domains,
dns_provider_plugin,
dns_provider_options,
)
self.acme.finalize_authorizations(
acme_client,
account_number,
dns_provider_plugin,
authorizations,
dns_provider_options,
)
pem_certificate, pem_certificate_chain = self.acme.request_certificate(
acme_client, authorizations, csr
)
# TODO add external ID (if possible)
return pem_certificate, pem_certificate_chain, None
return acme_dns_challenge.create_certificate(csr, issuer_options)
@staticmethod
def create_authority(options):
@ -779,3 +267,117 @@ class ACMEIssuerPlugin(IssuerPlugin):
def cancel_ordered_certificate(self, pending_cert, **kwargs):
# Needed to override issuer function.
pass
def revoke_certificate(self, certificate, reason):
self.acme = AcmeDnsHandler()
crl_reason = CRLReason.unspecified
if "crl_reason" in reason:
crl_reason = CRLReason[reason["crl_reason"]]
return self.acme.revoke_certificate(certificate, crl_reason.value)
class ACMEHttpIssuerPlugin(IssuerPlugin):
title = "Acme HTTP-01"
slug = "acme-http-issuer"
description = (
"Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the HTTP-01 challenge"
)
version = acme.VERSION
author = "Netflix"
author_url = "https://github.com/netflix/lemur.git"
options = [
{
"name": "acme_url",
"type": "str",
"required": True,
"validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/",
"helpMessage": "Must be a valid web url starting with http[s]://",
},
{
"name": "telephone",
"type": "str",
"default": "",
"helpMessage": "Telephone to use",
},
{
"name": "email",
"type": "str",
"default": "",
"validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/",
"helpMessage": "Email to use",
},
{
"name": "certificate",
"type": "textarea",
"default": "",
"validation": "/^-----BEGIN CERTIFICATE-----/",
"helpMessage": "Certificate to use",
},
{
"name": "store_account",
"type": "bool",
"required": False,
"helpMessage": "Disable to create a new account for each ACME request",
"default": False,
},
{
"name": "tokenDestination",
"type": "destinationSelect",
"required": True,
"helpMessage": "The destination to use to deploy the token.",
},
]
def __init__(self, *args, **kwargs):
super(ACMEHttpIssuerPlugin, self).__init__(*args, **kwargs)
def create_certificate(self, csr, issuer_options):
"""
Creates an ACME certificate using the HTTP-01 challenge.
:param csr:
:param issuer_options:
:return: :raise Exception:
"""
acme_http_challenge = AcmeHttpChallenge()
return acme_http_challenge.create_certificate(csr, issuer_options)
@staticmethod
def create_authority(options):
"""
Creates an authority, this authority is then used by Lemur to allow a user
to specify which Certificate Authority they want to sign their certificate.
:param options:
:return:
"""
role = {"username": "", "password": "", "name": "acme"}
plugin_options = options.get("plugin", {}).get("plugin_options")
if not plugin_options:
error = "Invalid options for lemur_acme plugin: {}".format(options)
current_app.logger.error(error)
raise InvalidConfiguration(error)
# Define static acme_root based off configuration variable by default. However, if user has passed a
# certificate, use this certificate as the root.
acme_root = current_app.config.get("ACME_ROOT")
for option in plugin_options:
if option.get("name") == "certificate":
acme_root = option.get("value")
return acme_root, "", [role]
def cancel_ordered_certificate(self, pending_cert, **kwargs):
# Needed to override issuer function.
pass
def revoke_certificate(self, certificate, reason):
self.acme = AcmeHandler()
crl_reason = CRLReason.unspecified
if "crl_reason" in reason:
crl_reason = CRLReason[reason["crl_reason"]]
return self.acme.revoke_certificate(certificate, crl_reason.value)

228
lemur/plugins/lemur_acme/tests/test_acme.py → lemur/plugins/lemur_acme/tests/test_acme_dns.py
View File

@ -5,15 +5,16 @@ import josepy as jose
from cryptography.x509 import DNSName
from flask import Flask
from lemur.plugins.lemur_acme import plugin
from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord
from lemur.common.utils import generate_private_key
from mock import MagicMock
class TestAcme(unittest.TestCase):
@patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
class TestAcmeDns(unittest.TestCase):
@patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
def setUp(self, mock_dns_provider_service):
self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin()
self.acme = plugin.AcmeHandler()
self.acme = plugin.AcmeDnsHandler()
mock_dns_provider = Mock()
mock_dns_provider.name = "cloudflare"
mock_dns_provider.credentials = "{}"
@ -50,36 +51,19 @@ class TestAcme(unittest.TestCase):
result = yield self.acme.get_dns_challenges(host, mock_authz)
self.assertEqual(result, mock_entry)
def test_strip_wildcard(self):
expected = ("example.com", False)
result = self.acme.strip_wildcard("example.com")
self.assertEqual(expected, result)
expected = ("example.com", True)
result = self.acme.strip_wildcard("*.example.com")
self.assertEqual(expected, result)
def test_authz_record(self):
a = plugin.AuthorizationRecord("domain", "host", "authz", "challenge", "id")
self.assertEqual(type(a), plugin.AuthorizationRecord)
@patch("acme.client.Client")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.plugin.len", return_value=1)
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges")
def test_start_dns_challenge(
self, mock_get_dns_challenges, mock_len, mock_app, mock_acme
self, mock_get_dns_challenges, mock_len, mock_acme
):
assert mock_len
mock_order = Mock()
mock_app.logger.debug = Mock()
mock_authz = Mock()
mock_authz.body.resolved_combinations = []
mock_entry = MagicMock()
from acme import challenges
c = challenges.DNS01()
mock_entry.chall = TestAcme.test_complete_dns_challenge_fail
mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail
mock_authz.body.resolved_combinations.append(mock_entry)
mock_acme.request_domain_challenges = Mock(return_value=mock_authz)
mock_dns_provider = Mock()
@ -92,14 +76,13 @@ class TestAcme(unittest.TestCase):
result = self.acme.start_dns_challenge(
mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {}
)
self.assertEqual(type(result), plugin.AuthorizationRecord)
self.assertEqual(type(result), AuthorizationRecord)
@patch("acme.client.Client")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change")
@patch("time.sleep")
def test_complete_dns_challenge_success(
self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme
self, mock_sleep, mock_wait_for_dns_change, mock_acme
):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
@ -120,10 +103,9 @@ class TestAcme(unittest.TestCase):
self.acme.complete_dns_challenge(mock_acme, mock_authz)
@patch("acme.client.Client")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change")
def test_complete_dns_challenge_fail(
self, mock_wait_for_dns_change, mock_current_app, mock_acme
self, mock_wait_for_dns_change, mock_acme
):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
@ -150,11 +132,9 @@ class TestAcme(unittest.TestCase):
@patch("acme.client.Client")
@patch("OpenSSL.crypto", return_value="mock_cert")
@patch("josepy.util.ComparableX509")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges")
def test_request_certificate(
self,
mock_current_app,
mock_get_dns_challenges,
mock_jose,
mock_crypto,
@ -171,7 +151,6 @@ class TestAcme(unittest.TestCase):
mock_acme.fetch_chain = Mock(return_value="mock_chain")
mock_crypto.dump_certificate = Mock(return_value=b"chain")
mock_order = Mock()
mock_current_app.config = {}
self.acme.request_certificate(mock_acme, [], mock_order)
def test_setup_acme_client_fail(self):
@ -180,10 +159,9 @@ class TestAcme(unittest.TestCase):
with self.assertRaises(Exception):
self.acme.setup_acme_client(mock_authority)
@patch("lemur.plugins.lemur_acme.plugin.jose.JWK.json_loads")
@patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_setup_acme_client_success_load_account_from_authority(self, mock_current_app, mock_acme, mock_key_json_load):
@patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads")
@patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
def test_setup_acme_client_success_load_account_from_authority(self, mock_acme, mock_key_json_load):
mock_authority = Mock()
mock_authority.id = 2
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
@ -192,7 +170,6 @@ class TestAcme(unittest.TestCase):
'{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]'
mock_client = Mock()
mock_acme.return_value = mock_client
mock_current_app.config = {}
mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048"))
@ -202,11 +179,10 @@ class TestAcme(unittest.TestCase):
assert result_client
assert not result_registration
@patch("lemur.plugins.lemur_acme.plugin.jose.JWKRSA.fields_to_partial_json")
@patch("lemur.plugins.lemur_acme.plugin.authorities_service")
@patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_setup_acme_client_success_store_new_account(self, mock_current_app, mock_acme, mock_authorities_service,
@patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWKRSA.fields_to_partial_json")
@patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service")
@patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
def test_setup_acme_client_success_store_new_account(self, mock_acme, mock_authorities_service,
mock_key_generation):
mock_authority = Mock()
mock_authority.id = 2
@ -219,7 +195,6 @@ class TestAcme(unittest.TestCase):
mock_client.agree_to_tos = Mock(return_value=True)
mock_client.new_account_and_tos.return_value = mock_registration
mock_acme.return_value = mock_client
mock_current_app.config = {}
mock_key_generation.return_value = {"n": "PwIOkViO"}
@ -232,10 +207,9 @@ class TestAcme(unittest.TestCase):
'{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, '
'{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]')
@patch("lemur.plugins.lemur_acme.plugin.authorities_service")
@patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_setup_acme_client_success(self, mock_current_app, mock_acme, mock_authorities_service):
@patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service")
@patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
def test_setup_acme_client_success(self, mock_acme, mock_authorities_service):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
'{"name": "store_account", "value": false}]'
@ -245,20 +219,17 @@ class TestAcme(unittest.TestCase):
mock_client.register = mock_registration
mock_client.agree_to_tos = Mock(return_value=True)
mock_acme.return_value = mock_client
mock_current_app.config = {}
result_client, result_registration = self.acme.setup_acme_client(mock_authority)
mock_authorities_service.update_options.assert_not_called()
assert result_client
assert result_registration
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_get_domains_single(self, mock_current_app):
def test_get_domains_single(self):
options = {"common_name": "test.netflix.net"}
result = self.acme.get_domains(options)
self.assertEqual(result, [options["common_name"]])
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_get_domains_multiple(self, mock_current_app):
def test_get_domains_multiple(self):
options = {
"common_name": "test.netflix.net",
"extensions": {
@ -270,8 +241,7 @@ class TestAcme(unittest.TestCase):
result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"]
)
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_get_domains_san(self, mock_current_app):
def test_get_domains_san(self):
options = {
"common_name": "test.netflix.net",
"extensions": {
@ -283,9 +253,63 @@ class TestAcme(unittest.TestCase):
result, [options["common_name"], "test2.netflix.net"]
)
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", return_value="test")
@patch("lemur.plugins.lemur_acme.plugin.current_app", return_value=False)
def test_get_authorizations(self, mock_current_app, mock_start_dns_challenge):
def test_create_authority(self):
options = {
"plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}
}
acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options)
self.assertEqual(acme_root, "123")
self.assertEqual(b, "")
self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}])
@patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
def test_get_dns_provider(self, mock_dns_provider_service):
provider = plugin.AcmeDnsHandler()
route53 = provider.get_dns_provider("route53")
assert route53
cloudflare = provider.get_dns_provider("cloudflare")
assert cloudflare
dyn = provider.get_dns_provider("dyn")
assert dyn
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.challenge_types.authorization_service")
def test_create_certificate(
self,
mock_authorization_service,
mock_request_certificate,
mock_finalize_authorizations,
mock_get_authorizations,
mock_dns_provider_service,
mock_acme,
):
provider = plugin.ACMEIssuerPlugin()
mock_authority = Mock()
mock_client = Mock()
mock_acme.return_value = (mock_client, "")
mock_dns_provider = Mock()
mock_dns_provider.credentials = '{"account_id": 1}'
mock_dns_provider.provider_type = "route53"
mock_dns_provider_service.get.return_value = mock_dns_provider
issuer_options = {
"authority": mock_authority,
"dns_provider": mock_dns_provider,
"common_name": "test.netflix.net",
}
csr = "123"
mock_request_certificate.return_value = ("pem_certificate", "chain")
result = provider.create_certificate(csr, issuer_options)
assert result
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", return_value="test")
def test_get_authorizations(self, mock_start_dns_challenge):
mock_order = Mock()
mock_order.body.identifiers = []
mock_domain = Mock()
@ -299,7 +323,7 @@ class TestAcme(unittest.TestCase):
self.assertEqual(result, ["test"])
@patch(
"lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge",
"lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.complete_dns_challenge",
return_value="test",
)
def test_finalize_authorizations(self, mock_complete_dns_challenge):
@ -317,51 +341,21 @@ class TestAcme(unittest.TestCase):
result = self.acme.finalize_authorizations(mock_acme_client, mock_authz)
self.assertEqual(result, mock_authz)
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_create_authority(self, mock_current_app):
mock_current_app.config = Mock()
options = {
"plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}
}
acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options)
self.assertEqual(acme_root, "123")
self.assertEqual(b, "")
self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}])
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.dyn.current_app")
@patch("lemur.plugins.lemur_acme.cloudflare.current_app")
@patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
def test_get_dns_provider(
self,
mock_dns_provider_service,
mock_current_app_cloudflare,
mock_current_app_dyn,
mock_current_app,
):
provider = plugin.ACMEIssuerPlugin()
route53 = provider.get_dns_provider("route53")
assert route53
cloudflare = provider.get_dns_provider("cloudflare")
assert cloudflare
dyn = provider.get_dns_provider("dyn")
assert dyn
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.plugin.authorization_service")
@patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
@patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate")
def test_get_ordered_certificate(
self,
mock_request_certificate,
mock_finalize_authorizations,
mock_get_authorizations,
mock_dns_provider_service_p,
mock_dns_provider_service,
mock_authorization_service,
mock_current_app,
mock_acme,
):
mock_client = Mock()
@ -379,20 +373,20 @@ class TestAcme(unittest.TestCase):
)
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.plugin.authorization_service")
@patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
@patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate")
def test_get_ordered_certificates(
self,
mock_request_certificate,
mock_finalize_authorizations,
mock_get_authorizations,
mock_dns_provider_service,
mock_dns_provider_service_p,
mock_authorization_service,
mock_current_app,
mock_acme,
):
mock_client = Mock()
@ -417,41 +411,3 @@ class TestAcme(unittest.TestCase):
result[1]["cert"],
{"body": "pem_certificate", "chain": "chain", "external_id": "2"},
)
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.plugin.authorization_service")
def test_create_certificate(
self,
mock_authorization_service,
mock_request_certificate,
mock_finalize_authorizations,
mock_get_authorizations,
mock_current_app,
mock_dns_provider_service,
mock_acme,
):
provider = plugin.ACMEIssuerPlugin()
mock_authority = Mock()
mock_client = Mock()
mock_acme.return_value = (mock_client, "")
mock_dns_provider = Mock()
mock_dns_provider.credentials = '{"account_id": 1}'
mock_dns_provider.provider_type = "route53"
mock_dns_provider_service.get.return_value = mock_dns_provider
issuer_options = {
"authority": mock_authority,
"dns_provider": mock_dns_provider,
"common_name": "test.netflix.net",
}
csr = "123"
mock_request_certificate.return_value = ("pem_certificate", "chain")
result = provider.create_certificate(csr, issuer_options)
assert result

112
lemur/plugins/lemur_acme/tests/test_acme_handler.py Normal file
View File

@ -0,0 +1,112 @@
import unittest
from unittest.mock import patch, Mock
from flask import Flask
from cryptography.x509 import DNSName
from lemur.plugins.lemur_acme import acme_handlers
class TestAcmeHandler(unittest.TestCase):
def setUp(self):
self.acme = acme_handlers.AcmeHandler()
# Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
# needed to run tests in dev environment without getting error 'Working outside of application context'.
_app = Flask('lemur_test_acme')
self.ctx = _app.app_context()
assert self.ctx
self.ctx.push()
def tearDown(self):
self.ctx.pop()
def test_strip_wildcard(self):
expected = ("example.com", False)
result = self.acme.strip_wildcard("example.com")
self.assertEqual(expected, result)
expected = ("example.com", True)
result = self.acme.strip_wildcard("*.example.com")
self.assertEqual(expected, result)
def test_authz_record(self):
a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id", "cname_delegation")
self.assertEqual(type(a), acme_handlers.AuthorizationRecord)
def test_setup_acme_client_fail(self):
mock_authority = Mock()
mock_authority.options = []
with self.assertRaises(Exception):
self.acme.setup_acme_client(mock_authority)
def test_reuse_account_not_defined(self):
mock_authority = Mock()
mock_authority.options = []
with self.assertRaises(Exception):
self.acme.reuse_account(mock_authority)
def test_reuse_account_from_authority(self):
mock_authority = Mock()
mock_authority.options = '[{"name": "acme_private_key", "value": "PRIVATE_KEY"}, {"name": "acme_regr", "value": "ACME_REGR"}]'
self.assertTrue(self.acme.reuse_account(mock_authority))
@patch("lemur.plugins.lemur_acme.acme_handlers.current_app")
def test_reuse_account_from_config(self, mock_current_app):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
mock_current_app.config = {"ACME_PRIVATE_KEY": "PRIVATE_KEY", "ACME_REGR": "ACME_REGR"}
self.assertTrue(self.acme.reuse_account(mock_authority))
def test_reuse_account_no_configuration(self):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
self.assertFalse(self.acme.reuse_account(mock_authority))
@patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service")
@patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
def test_setup_acme_client_success(self, mock_acme, mock_authorities_service):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
'{"name": "store_account", "value": false}]'
mock_client = Mock()
mock_registration = Mock()
mock_registration.uri = "http://test.com"
mock_client.register = mock_registration
mock_client.agree_to_tos = Mock(return_value=True)
mock_acme.return_value = mock_client
result_client, result_registration = self.acme.setup_acme_client(mock_authority)
mock_authorities_service.update_options.assert_not_called()
assert result_client
assert result_registration
def test_get_domains_single(self):
options = {"common_name": "test.netflix.net"}
result = self.acme.get_domains(options)
self.assertEqual(result, [options["common_name"]])
def test_get_domains_multiple(self):
options = {
"common_name": "test.netflix.net",
"extensions": {
"sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]}
},
}
result = self.acme.get_domains(options)
self.assertEqual(
result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"]
)
def test_get_domains_san(self):
options = {
"common_name": "test.netflix.net",
"extensions": {
"sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]}
},
}
result = self.acme.get_domains(options)
self.assertEqual(
result, [options["common_name"], "test2.netflix.net"]
)

311
lemur/plugins/lemur_acme/tests/test_acme_http.py Normal file
View File

@ -0,0 +1,311 @@
import unittest
from unittest.mock import patch, Mock
from acme import challenges
from flask import Flask
from lemur.plugins.lemur_acme import plugin
class TestAcmeHttp(unittest.TestCase):
def setUp(self):
self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin()
self.acme = plugin.AcmeHandler()
# Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
# needed to run tests in dev environment without getting error 'Working outside of application context'.
_app = Flask('lemur_test_acme')
self.ctx = _app.app_context()
assert self.ctx
self.ctx.push()
def tearDown(self):
self.ctx.pop()
def test_create_authority(self):
options = {
"plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}
}
acme_root, b, role = self.ACMEHttpIssuerPlugin.create_authority(options)
self.assertEqual(acme_root, "123")
self.assertEqual(b, "")
self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}])
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.base.manager.PluginManager.get")
@patch("lemur.plugins.lemur_acme.challenge_types.destination_service")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.plugin.authorization_service")
def test_create_certificate(
self,
mock_authorization_service,
mock_request_certificate,
mock_destination_service,
mock_plugin_manager_get,
mock_acme,
):
provider = plugin.ACMEHttpIssuerPlugin()
mock_authority = Mock()
mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]'
mock_order_resource = Mock()
mock_order_resource.authorizations = [Mock()]
mock_order_resource.authorizations[0].body.challenges = [Mock()]
mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (
Mock(), "Anything-goes")
mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(
token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
mock_client = Mock()
mock_client.new_order.return_value = mock_order_resource
mock_client.answer_challenge.return_value = True
mock_finalized_order = Mock()
mock_finalized_order.fullchain_pem = """
-----BEGIN CERTIFICATE-----
MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw
GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2
MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq
hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ
diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP
xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG
TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj
EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd
O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa
aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0
A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr
IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe
Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb
Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50
qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA
A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln
uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H
sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm
dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd
oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV
/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ
zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc
VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1
Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4
8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c
idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng==
-----END CERTIFICATE-----
"""
mock_client.poll_and_finalize.return_value = mock_finalized_order
mock_acme.return_value = (mock_client, "")
mock_destination = Mock()
mock_destination.label = "mock-sftp-destination"
mock_destination.plugin_name = "SFTPDestinationPlugin"
mock_destination_service.get.return_value = mock_destination
mock_destination_plugin = Mock()
mock_destination_plugin.upload_acme_token.return_value = True
mock_plugin_manager_get.return_value = mock_destination_plugin
issuer_options = {
"authority": mock_authority,
"tokenDestination": "mock-sftp-destination",
"common_name": "test.netflix.net",
}
csr = "123"
mock_request_certificate.return_value = ("pem_certificate", "chain")
pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options)
self.assertEqual(pem_certificate, "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n")
self.assertEqual(pem_certificate_chain, """-----BEGIN CERTIFICATE-----
MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw
GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2
MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq
hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ
diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP
xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG
TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj
EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd
O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa
aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0
A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr
IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe
Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb
Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50
qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA
A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln
uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H
sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm
dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd
oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV
/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ
zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc
VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1
Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4
8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c
idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng==
-----END CERTIFICATE-----
""")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.base.manager.PluginManager.get")
@patch("lemur.plugins.lemur_acme.challenge_types.destination_service")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.plugin.authorization_service")
def test_create_certificate_missing_destination_token(
self,
mock_authorization_service,
mock_request_certificate,
mock_destination_service,
mock_plugin_manager_get,
mock_acme,
):
provider = plugin.ACMEHttpIssuerPlugin()
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
mock_order_resource = Mock()
mock_order_resource.authorizations = [Mock()]
mock_order_resource.authorizations[0].body.challenges = [Mock()]
mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(
token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
mock_client = Mock()
mock_client.new_order.return_value = mock_order_resource
mock_acme.return_value = (mock_client, "")
mock_destination = Mock()
mock_destination.label = "mock-sftp-destination"
mock_destination.plugin_name = "SFTPDestinationPlugin"
mock_destination_service.get_by_label.return_value = mock_destination
mock_destination_plugin = Mock()
mock_destination_plugin.upload_acme_token.return_value = True
mock_plugin_manager_get.return_value = mock_destination_plugin
issuer_options = {
"authority": mock_authority,
"tokenDestination": "mock-sftp-destination",
"common_name": "test.netflix.net",
}
csr = "123"
mock_request_certificate.return_value = ("pem_certificate", "chain")
with self.assertRaisesRegex(Exception, "No token_destination configured"):
provider.create_certificate(csr, issuer_options)
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
@patch("lemur.plugins.base.manager.PluginManager.get")
@patch("lemur.plugins.lemur_acme.challenge_types.destination_service")
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
@patch("lemur.plugins.lemur_acme.plugin.authorization_service")
def test_create_certificate_missing_http_challenge(
self,
mock_authorization_service,
mock_request_certificate,
mock_destination_service,
mock_plugin_manager_get,
mock_acme,
):
provider = plugin.ACMEHttpIssuerPlugin()
mock_authority = Mock()
mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]'
mock_order_resource = Mock()
mock_order_resource.authorizations = [Mock()]
mock_order_resource.authorizations[0].body.challenges = [Mock()]
mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.DNS01(
token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
mock_client = Mock()
mock_client.new_order.return_value = mock_order_resource
mock_acme.return_value = (mock_client, "")
issuer_options = {
"authority": mock_authority,
"tokenDestination": "mock-sftp-destination",
"common_name": "test.netflix.net",
}
csr = "123"
mock_request_certificate.return_value = ("pem_certificate", "chain")
with self.assertRaisesRegex(Exception, "HTTP-01 challenge was not offered"):
provider.create_certificate(csr, issuer_options)

13
lemur/plugins/lemur_adcs/plugin.py
View File

@ -59,8 +59,8 @@ class ADCSIssuerPlugin(IssuerPlugin):
)
return cert, chain, None
def revoke_certificate(self, certificate, comments):
raise NotImplementedError("Not implemented\n", self, certificate, comments)
def revoke_certificate(self, certificate, reason):
raise NotImplementedError("Not implemented\n", self, certificate, reason)
def get_ordered_certificate(self, order_id):
raise NotImplementedError("Not implemented\n", self, order_id)
@ -77,15 +77,6 @@ class ADCSSourcePlugin(SourcePlugin):
author = "sirferl"
author_url = "https://github.com/sirferl/lemur"
options = [
{
"name": "dummy",
"type": "str",
"required": False,
"validation": "/^[0-9]{12,12}$/",
"helpMessage": "Just to prevent error",
}
]
def get_certificates(self, options, **kwargs):
adcs_server = current_app.config.get("ADCS_SERVER")

32
lemur/plugins/lemur_aws/elb.py
View File

@ -149,6 +149,38 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
raise
@sts_client("elbv2")
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=5)
def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs):
"""
Get a load balancer ARN from an endpoint.
:param endpoint_name:
:return:
"""
try:
client = kwargs.pop("client")
elbs = client.describe_load_balancers(Names=[endpoint_name])
if "LoadBalancers" in elbs and elbs["LoadBalancers"]:
return elbs["LoadBalancers"][0]["LoadBalancerArn"]
except Exception as e: # noqa
metrics.send(
"get_load_balancer_arn_from_endpoint",
"counter",
1,
metric_tags={
"error": str(e),
"endpoint_name": endpoint_name,
},
)
sentry.captureException(
extra={
"endpoint_name": str(endpoint_name),
}
)
raise
@sts_client("elb")
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def get_elbs(**kwargs):

88
lemur/plugins/lemur_aws/plugin.py
View File

@ -300,6 +300,41 @@ class AWSSourcePlugin(SourcePlugin):
)
return None
def get_endpoint_certificate_names(self, endpoint):
options = endpoint.source.options
account_number = self.get_option("accountNumber", options)
region = get_region_from_dns(endpoint.dnsname)
certificate_names = []
if endpoint.type == "elb":
elb_details = elb.get_elbs(account_number=account_number,
region=region,
LoadBalancerNames=[endpoint.name],)
for lb_description in elb_details["LoadBalancerDescriptions"]:
for listener_description in lb_description["ListenerDescriptions"]:
listener = listener_description.get("Listener")
if not listener.get("SSLCertificateId"):
continue
certificate_names.append(iam.get_name_from_arn(listener.get("SSLCertificateId")))
elif endpoint.type == "elbv2":
listeners = elb.describe_listeners_v2(
account_number=account_number,
region=region,
LoadBalancerArn=elb.get_load_balancer_arn_from_endpoint(endpoint.name,
account_number=account_number,
region=region),
)
for listener in listeners["Listeners"]:
if not listener.get("Certificates"):
continue
for certificate in listener["Certificates"]:
certificate_names.append(iam.get_name_from_arn(certificate["CertificateArn"]))
return certificate_names
class AWSDestinationPlugin(DestinationPlugin):
title = "AWS"
@ -344,6 +379,10 @@ class AWSDestinationPlugin(DestinationPlugin):
def deploy(self, elb_name, account, region, certificate):
pass
def clean(self, certificate, options, **kwargs):
account_number = self.get_option("accountNumber", options)
iam.delete_cert(certificate.name, account_number=account_number)
class S3DestinationPlugin(ExportDestinationPlugin):
title = "AWS-S3"
@ -411,7 +450,8 @@ class S3DestinationPlugin(ExportDestinationPlugin):
def upload_acme_token(self, token_path, token, options, **kwargs):
"""
This is called from the acme http challenge
This is called from the acme http challenge
:param self:
:param token_path:
:param token:
@ -419,7 +459,7 @@ class S3DestinationPlugin(ExportDestinationPlugin):
:param kwargs:
:return:
"""
current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
current_app.logger.debug("S3 destination plugin is started to upload HTTP-01 challenge")
function = f"{__name__}.{sys._getframe().f_code.co_name}"
@ -431,16 +471,16 @@ class S3DestinationPlugin(ExportDestinationPlugin):
if not prefix.endswith("/"):
prefix + "/"
res = s3.put(bucket_name=bucket_name,
region_name=region,
prefix=prefix + filename,
data=token,
encrypt=False,
account_number=account_number)
res = "Success" if res else "Failure"
response = s3.put(bucket_name=bucket_name,
region_name=region,
prefix=prefix + filename,
data=token,
encrypt=False,
account_number=account_number)
res = "Success" if response else "Failure"
log_data = {
"function": function,
"message": "check if any valid certificate is revoked",
"message": "upload acme token challenge",
"result": res,
"bucket_name": bucket_name,
"filename": filename
@ -449,6 +489,34 @@ class S3DestinationPlugin(ExportDestinationPlugin):
metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
"bucket_name": bucket_name,
"filename": filename})
return response
def delete_acme_token(self, token_path, options, **kwargs):
current_app.logger.debug("S3 destination plugin is started to delete HTTP-01 challenge")
function = f"{__name__}.{sys._getframe().f_code.co_name}"
account_number = self.get_option("accountNumber", options)
bucket_name = self.get_option("bucket", options)
prefix = self.get_option("prefix", options)
filename = token_path.split("/")[-1]
response = s3.delete(bucket_name=bucket_name,
prefixed_object_name=prefix + filename,
account_number=account_number)
res = "Success" if response else "Failure"
log_data = {
"function": function,
"message": "delete acme token challenge",
"result": res,
"bucket_name": bucket_name,
"filename": filename
}
current_app.logger.info(log_data)
metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
"bucket_name": bucket_name,
"filename": filename})
return response
class SNSNotificationPlugin(ExpirationNotificationPlugin):

2
lemur/plugins/lemur_aws/sns.py
View File

@ -50,6 +50,8 @@ def format_message(certificate, notification_type):
json_message = {
"notification_type": notification_type,
"certificate_name": certificate["name"],
"issuer": certificate["issuer"],
"id": certificate["id"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), # 2047-12-31T22:00:00
"endpoints_detected": len(certificate["endpoints"]),
"owner": certificate["owner"],

106
lemur/plugins/lemur_aws/tests/test_elb.py
View File

@ -1,5 +1,5 @@
import boto3
from moto import mock_sts, mock_elb
from moto import mock_sts, mock_ec2, mock_elb, mock_elbv2, mock_iam
@mock_sts()
@ -27,3 +27,107 @@ def test_get_all_elbs(app, aws_credentials):
elbs = get_all_elbs(account_number="123456789012", region="us-east-1")
assert elbs
@mock_sts()
@mock_ec2
@mock_elbv2()
@mock_iam
def test_create_elb_with_https_listener_miscellaneous(app, aws_credentials):
from lemur.plugins.lemur_aws import iam, elb
endpoint_name = "example-lbv2"
account_number = "123456789012"
region_ue1 = "us-east-1"
client = boto3.client("elbv2", region_name="us-east-1")
ec2 = boto3.resource("ec2", region_name="us-east-1")
# Create VPC
vpc = ec2.create_vpc(CidrBlock="172.28.7.0/24")
# Create LB (elbv2) in above VPC
assert create_load_balancer(client, ec2, vpc.id, endpoint_name)
# Create target group
target_group_arn = create_target_group(client, vpc.id)
assert target_group_arn
# Test get_load_balancer_arn_from_endpoint
lb_arn = elb.get_load_balancer_arn_from_endpoint(endpoint_name,
account_number=account_number,
region=region_ue1)
assert lb_arn
# Test describe_listeners_v2
listeners = elb.describe_listeners_v2(account_number=account_number,
region=region_ue1,
LoadBalancerArn=lb_arn)
assert listeners
assert not listeners["Listeners"]
# Upload cert
response = iam.upload_cert("LemurTestCert", "testCert", "cert1", "cert2",
account_number=account_number)
assert response
cert_arn = response["ServerCertificateMetadata"]["Arn"]
assert cert_arn
# Create https listener using above cert
listeners = client.create_listener(
LoadBalancerArn=lb_arn,
Protocol="HTTPS",
Port=443,
Certificates=[{"CertificateArn": cert_arn}],
DefaultActions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
)
assert listeners
listener_arn = listeners["Listeners"][0]["ListenerArn"]
assert listener_arn
assert listeners["Listeners"]
for listener in listeners["Listeners"]:
if listener["Port"] == 443:
assert listener["Certificates"]
assert cert_arn == listener["Certificates"][0]["CertificateArn"]
# Test get_listener_arn_from_endpoint
assert listener_arn == elb.get_listener_arn_from_endpoint(
endpoint_name,
443,
account_number=account_number,
region=region_ue1,
)
@mock_sts()
@mock_elb()
def test_get_all_elbs_v2():
from lemur.plugins.lemur_aws.elb import get_all_elbs_v2
elbs = get_all_elbs_v2(account_number="123456789012",
region="us-east-1")
assert elbs
def create_load_balancer(client, ec2, vpc_id, endpoint_name):
subnet1 = ec2.create_subnet(
VpcId=vpc_id,
CidrBlock="172.28.7.192/26",
AvailabilityZone="us-east-1a"
)
return client.create_load_balancer(
Name=endpoint_name,
Subnets=[
subnet1.id,
],
)
def create_target_group(client, vpc_id):
response = client.create_target_group(
Name="a-target",
Protocol="HTTPS",
Port=443,
VpcId=vpc_id,
)
return response.get("TargetGroups")[0]["TargetGroupArn"]

14
lemur/plugins/lemur_aws/tests/test_plugin.py
View File

@ -68,10 +68,11 @@ def test_upload_acme_token(app):
s3_client.create_bucket(Bucket=bucket)
p = plugins.get("aws-s3")
p.upload_acme_token(token_path=token_path,
token_content=token_content,
token=token_content,
options=additional_options)
response = p.upload_acme_token(token_path=token_path,
token_content=token_content,
token=token_content,
options=additional_options)
assert response
response = get(bucket_name=bucket,
prefixed_object_name=prefix + token_name,
@ -80,3 +81,8 @@ def test_upload_acme_token(app):
# put data, and getting the same data
assert (response == token_content)
response = p.delete_acme_token(token_path=token_path,
options=additional_options,
account_number=account)
assert response

2
lemur/plugins/lemur_aws/tests/test_sns.py
View File

@ -21,6 +21,8 @@ def test_format(certificate, endpoint):
"notification_type": "expiration",
"certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
"issuer": certificate["issuer"],
"id": certificate["id"],
"endpoints_detected": 0,
"owner": certificate["owner"],
"details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])

4
lemur/plugins/lemur_azure_dest/__init__.py Normal file
View File

@ -0,0 +1,4 @@
try:
VERSION = __import__("pkg_resources").get_distribution(__name__).version
except Exception as e:
VERSION = "unknown"

184
lemur/plugins/lemur_azure_dest/plugin.py Executable file
View File

@ -0,0 +1,184 @@
"""
.. module: lemur.plugins.lemur_azure_dest.plugin
:platform: Unix
:copyright: (c) 2019
:license: Apache, see LICENCE for more details.
Plugin for uploading certificates and private key as secret to azure key-vault
that can be pulled down by end point nodes.
.. moduleauthor:: sirferl
"""
from flask import current_app
from lemur.common.defaults import common_name, bitstrength
from lemur.common.utils import parse_certificate, parse_private_key
from lemur.plugins.bases import DestinationPlugin
from cryptography.hazmat.primitives import serialization
import requests
import json
import sys
def handle_response(my_response):
"""
Helper function for parsing responses from the Entrust API.
:param my_response:
:return: :raise Exception:
"""
msg = {
200: "The request was successful.",
400: "Keyvault Error"
}
try:
data = json.loads(my_response.content)
except ValueError:
# catch an empty jason object here
data = {'response': 'No detailed message'}
status_code = my_response.status_code
if status_code > 399:
raise Exception(f"AZURE error: {msg.get(status_code, status_code)}\n{data}")
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Response",
"status": status_code,
"response": data
}
current_app.logger.info(log_data)
if data == {'response': 'No detailed message'}:
# status if no data
return status_code
else:
# return data from the response
return data
def get_access_token(tenant, appID, password, self):
"""
Gets the access token with the appid and the password and returns it
Improvment option: we can try to save it and renew it only when necessary
:param tenant: Tenant used
:param appID: Application ID from Azure
:param password: password for Application ID
:return: Access token to post to the keyvault
"""
# prepare the call for the access_token
auth_url = f"https://login.microsoftonline.com/{tenant}/oauth2/token"
post_data = {
'grant_type': 'client_credentials',
'client_id': appID,
'client_secret': password,
'resource': 'https://vault.azure.net'
}
try:
response = self.session.post(auth_url, data=post_data)
except requests.exceptions.RequestException as e:
current_app.logger.exception(f"AZURE: Error for POST {e}")
access_token = json.loads(response.content)["access_token"]
return access_token
class AzureDestinationPlugin(DestinationPlugin):
"""Azure Keyvault Destination plugin for Lemur"""
title = "Azure"
slug = "azure-keyvault-destination"
description = "Allow the uploading of certificates to Azure key vault"
author = "Sirferl"
author_url = "https://github.com/sirferl/lemur"
options = [
{
"name": "vaultUrl",
"type": "str",
"required": True,
"validation": "^https?://[a-zA-Z0-9.:-]+$",
"helpMessage": "Valid URL to Azure key vault instance",
},
{
"name": "azureTenant",
"type": "str",
"required": True,
"validation": "^([a-zA-Z0-9/-/?)+$",
"helpMessage": "Tenant for the Azure Key Vault",
},
{
"name": "appID",
"type": "str",
"required": True,
"validation": "^([a-zA-Z0-9/-/?)+$",
"helpMessage": "AppID for the Azure Key Vault",
},
{
"name": "azurePassword",
"type": "str",
"required": True,
"validation": "[0-9a-zA-Z.:_-~]+",
"helpMessage": "Tenant password for the Azure Key Vault",
}
]
def __init__(self, *args, **kwargs):
self.session = requests.Session()
super(AzureDestinationPlugin, self).__init__(*args, **kwargs)
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
"""
Upload certificate and private key
:param private_key:
:param cert_chain:
:return:
"""
# we use the common name to identify the certificate
# Azure does not allow "." in the certificate name we replace them with "-"
cert = parse_certificate(body)
certificate_name = common_name(cert).replace(".", "-")
vault_URI = self.get_option("vaultUrl", options)
tenant = self.get_option("azureTenant", options)
app_id = self.get_option("appID", options)
password = self.get_option("azurePassword", options)
access_token = get_access_token(tenant, app_id, password, self)
cert_url = f"{vault_URI}/certificates/{certificate_name}/import?api-version=7.1"
post_header = {
"Authorization": f"Bearer {access_token}"
}
key_pkcs8 = parse_private_key(private_key).private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
)
key_pkcs8 = key_pkcs8.decode("utf-8").replace('\\n', '\n')
cert_package = f"{body}\n{key_pkcs8}"
post_body = {
"value": cert_package,
"policy": {
"key_props": {
"exportable": True,
"kty": "RSA",
"key_size": bitstrength(cert),
"reuse_key": True
},
"secret_props": {
"contentType": "application/x-pem-file"
}
}
}
try:
response = self.session.post(cert_url, headers=post_header, json=post_body)
except requests.exceptions.RequestException as e:
current_app.logger.exception(f"AZURE: Error for POST {e}")
return_value = handle_response(response)

1
lemur/plugins/lemur_azure_dest/tests/conftest.py Normal file
View File

@ -0,0 +1 @@
from lemur.tests.conftest import * # noqa

12
lemur/plugins/lemur_cfssl/plugin.py
View File

@ -18,6 +18,7 @@ from flask import current_app
from lemur.common.utils import parse_certificate
from lemur.common.utils import get_authority_key
from lemur.constants import CRLReason
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins import lemur_cfssl as cfssl
from lemur.extensions import metrics
@ -102,16 +103,23 @@ class CfsslIssuerPlugin(IssuerPlugin):
role = {"username": "", "password": "", "name": "cfssl"}
return current_app.config.get("CFSSL_ROOT"), "", [role]
def revoke_certificate(self, certificate, comments):
def revoke_certificate(self, certificate, reason):
"""Revoke a CFSSL certificate."""
base_url = current_app.config.get("CFSSL_URL")
create_url = "{0}/api/v1/cfssl/revoke".format(base_url)
crl_reason = CRLReason.unspecified
if "crl_reason" in reason:
crl_reason = CRLReason[reason["crl_reason"]]
data = (
'{"serial": "'
+ certificate.external_id
+ '","authority_key_id": "'
+ get_authority_key(certificate.body)
+ '", "reason": "superseded"}'
+ '", "reason": "'
+ crl_reason
+ '"}'
)
current_app.logger.debug("Revoking cert: {0}".format(data))
response = self.session.post(

13
lemur/plugins/lemur_digicert/plugin.py
View File

@ -368,7 +368,7 @@ class DigiCertIssuerPlugin(IssuerPlugin):
certificate_id,
)
def revoke_certificate(self, certificate, comments):
def revoke_certificate(self, certificate, reason):
"""Revoke a Digicert certificate."""
base_url = current_app.config.get("DIGICERT_URL")
@ -376,6 +376,11 @@ class DigiCertIssuerPlugin(IssuerPlugin):
create_url = "{0}/services/v2/certificate/{1}/revoke".format(
base_url, certificate.external_id
)
comments = reason["comments"] if "comments" in reason else ''
if "crl_reason" in reason:
comments += '(' + reason["crl_reason"] + ')'
metrics.send("digicert_revoke_certificate", "counter", 1)
response = self.session.put(create_url, data=json.dumps({"comments": comments}))
return handle_response(response)
@ -575,7 +580,7 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
data["id"],
)
def revoke_certificate(self, certificate, comments):
def revoke_certificate(self, certificate, reason):
"""Revoke a Digicert certificate."""
base_url = current_app.config.get("DIGICERT_CIS_URL")
@ -584,6 +589,10 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
base_url, certificate.external_id
)
metrics.send("digicert_revoke_certificate_success", "counter", 1)
comments = reason["comments"] if "comments" in reason else ''
if "crl_reason" in reason:
comments += '(' + reason["crl_reason"] + ')'
response = self.session.put(revoke_url, data=json.dumps({"comments": comments}))
if response.status_code != 204:

11
lemur/plugins/lemur_email/plugin.py
View File

@ -105,8 +105,11 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
@staticmethod
def send(notification_type, message, targets, options, **kwargs):
if not targets:
return
subject = "Lemur: {0} Notification".format(notification_type.capitalize())
readable_notification_type = ' '.join(map(lambda x: x.capitalize(), notification_type.split('_')))
subject = f"Lemur: {readable_notification_type} Notification"
body = render_html(notification_type, options, message)
@ -119,11 +122,9 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
send_via_smtp(subject, body, targets)
@staticmethod
def filter_recipients(options, excluded_recipients, **kwargs):
def get_recipients(options, additional_recipients, **kwargs):
notification_recipients = get_plugin_option("recipients", options)
if notification_recipients:
notification_recipients = notification_recipients.split(",")
# removing owner and security_email from notification_recipient
notification_recipients = [i for i in notification_recipients if i not in excluded_recipients]
return notification_recipients
return list(set(notification_recipients + additional_recipients))

179
lemur/plugins/lemur_email/templates/authority_expiration.html Normal file
View File

@ -0,0 +1,179 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
<title>Lemur</title>
</head>
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr height="32px"></tr>
<tr align="center">
<td width="32px"></td>
<td>
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
<tbody>
<tr>
<td>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
Lemur
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td>
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
<tbody>
<tr>
<td height="72px" colspan="3"></td>
</tr>
<tr>
<td width="32px"></td>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
Your CA certificate(s) are expiring in {{ message.options | interval }} days!
</td>
<td width="32px"></td>
</tr>
<tr>
<td height="18px" colspan="3"></td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td>
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
<tbody>
<tr height="16px">
<td width="32px" rowspan="3"></td>
<td></td>
<td width="32px" rowspan="3"></td>
</tr>
<tr>
<td>
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Hi,
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a Lemur CA certificate expiration notice. The following CA certificates are expiring soon;
please take manual action to renew them if necessary. Note that rotating a root CA requires
advanced planing and the respective trustStores need to be updated. A sub-CA, on the other hand,
does not require any changes to the trustStore. You may also disable notifications via the
Notify toggle in Lemur if they are no longer in use.
<table border="0" cellspacing="0" cellpadding="0"
style="margin-top:48px;margin-bottom:48px">
<tbody>
{% for certificate in message.certificates %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
<br>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
{% if certificate.self_signed %}
<b>Root</b>
{% else %}
Intermediate
{% endif %} CA
<br>{{ certificate.issued_cert_count }} issued certificates
<br>{{ certificate.owner }}
<br>{{ certificate.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
{% if not loop.last %}
<tr valign="middle">
<td width="32px" height="24px"></td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Your action is required if the above CA certificates are still needed.
</td>
</tr>
<tr>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>Best,<br><span class="il">Lemur</span>
</td>
</tr>
<tr height="16px"></tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
<tbody>
<tr>
<td>*All expiration times are in UTC<br></td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
</tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
<tbody>
<tr>
<td>You received this mandatory email announcement to update you about
important changes to your <span class="il">TLS certificate</span>.
</td>
</tr>
<tr>
<td>
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span></div>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
<td width="32px"></td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</div>

193
lemur/plugins/lemur_email/templates/expiration_summary.html Normal file
View File

@ -0,0 +1,193 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
<title>Lemur</title>
</head>
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr height="32px"></tr>
<tr align="center">
<td width="32px"></td>
<td>
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
<tbody>
<tr>
<td>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td align="left"
style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
Lemur
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td>
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
<tbody>
<tr>
<td height="72px" colspan="3"></td>
</tr>
<tr>
<td width="32px"></td>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
Lemur certificate expiration summary
</td>
<td width="32px"></td>
</tr>
<tr>
<td height="18px" colspan="3"></td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td>
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
<tbody>
<tr height="16px">
<td width="32px" rowspan="3"></td>
<td></td>
<td width="32px" rowspan="3"></td>
</tr>
<tr>
<td>
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Hi,
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a summary of all certificates expiring soon.
Certificates with notifications disabled have been omitted.
<table border="0" cellspacing="0" cellpadding="0" style="margin-top:12px;margin-bottom:48px">
<tbody>
{% for interval_and_certs in message["certificates"] | sort(attribute="interval") %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#202020">
<br>Expiring in {{ interval_and_certs["interval"] + 1 }} days<br>
</span>
</td>
</tr>
<tr valign="middle">
<td width="32px" height="12"></td>
</tr>
{% for certificate in interval_and_certs["certificates"] %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:16px;color:#202020">
{{ certificate.name }}
</span>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
{% if certificate.endpoints | length > 0 %} <!-- highlight in red if > 0 -->
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#F44336">
<br>{{ certificate.endpoints | length }} Endpoints
</span>
{% else %}
<br>{{ certificate.endpoints | length }} Endpoints
{% endif %}
<br>{{ certificate.owner }}
<br>{{ certificate.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
{% if not loop.last %}
<tr valign="middle">
<td width="32px" height="24px"></td>
</tr>
{% endif %}
{% endfor %}
{% endfor %}
</tbody>
</table>
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Please take action if any of the above certificates are still needed.
</td>
</tr>
<tr>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>Best,<br><span class="il">Lemur</span>
</td>
</tr>
<tr height="16px"></tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
<tbody>
<tr>
<td>*All expiration times are in UTC<br></td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
</tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
<tbody>
<tr>
<td>You received this mandatory email announcement to update you about
important changes to your <span class="il">TLS certificate</span>.
</td>
</tr>
<tr>
<td>
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span>
</div>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
<td width="32px"></td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</div>

24
lemur/plugins/lemur_email/tests/test_email.py
View File

@ -21,7 +21,6 @@ def get_options():
def test_render_expiration(certificate, endpoint):
new_cert = CertificateFactory()
new_cert.replaces.append(certificate)
@ -54,7 +53,7 @@ def test_send_expiration_notification():
certificate.notifications[0].options = get_options()
verify_sender_email()
assert send_expiration_notifications([]) == (3, 0) # owner, recipients (only counted as 1), and security
assert send_expiration_notifications([]) == (4, 0) # owner (1), recipients (2), and security (1)
@mock_ses
@ -76,15 +75,20 @@ def test_send_pending_failure_notification(user, pending_certificate, async_issu
verify_sender_email()
assert send_pending_failure_notification(pending_certificate)
assert send_pending_failure_notification(pending_certificate, True, True)
assert send_pending_failure_notification(pending_certificate, True, False)
assert send_pending_failure_notification(pending_certificate, False, True)
assert send_pending_failure_notification(pending_certificate, False, False)
def test_filter_recipients(certificate, endpoint):
def test_get_recipients(certificate, endpoint):
from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin
options = [{"name": "recipients", "value": "security@example.com,bob@example.com,joe@example.com"}]
assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@example.com", "bob@example.com",
"joe@example.com"]
assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com"]) == ["bob@example.com",
"joe@example.com"]
assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com", "bob@example.com",
"joe@example.com"]) == []
options = [{"name": "recipients", "value": "security@example.com,joe@example.com"}]
two_emails = sorted(["security@example.com", "joe@example.com"])
assert sorted(EmailNotificationPlugin.get_recipients(options, [])) == two_emails
assert sorted(EmailNotificationPlugin.get_recipients(options, ["security@example.com"])) == two_emails
three_emails = sorted(["security@example.com", "bob@example.com", "joe@example.com"])
assert sorted(EmailNotificationPlugin.get_recipients(options, ["bob@example.com"])) == three_emails
assert sorted(EmailNotificationPlugin.get_recipients(options, ["security@example.com", "bob@example.com",
"joe@example.com"])) == three_emails

167
lemur/plugins/lemur_entrust/plugin.py
View File

@ -5,10 +5,11 @@ import sys
from flask import current_app
from retrying import retry
from lemur.constants import CRLReason
from lemur.plugins import lemur_entrust as entrust
from lemur.plugins.bases import IssuerPlugin, SourcePlugin
from lemur.extensions import metrics
from lemur.common.utils import validate_conf
from lemur.common.utils import validate_conf, get_key_type_from_certificate
def log_status_code(r, *args, **kwargs):
@ -20,13 +21,14 @@ def log_status_code(r, *args, **kwargs):
:param kwargs:
:return:
"""
log_data = {
"reason": (r.reason if r.reason else ""),
"status_code": r.status_code,
"url": (r.url if r.url else ""),
}
metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
current_app.logger.info(log_data)
if r.status_code != 200:
log_data = {
"reason": (r.reason if r.reason else ""),
"status_code": r.status_code,
"url": (r.url if r.url else ""),
}
metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
current_app.logger.info(log_data)
def determine_end_date(end_date):
@ -45,7 +47,7 @@ def determine_end_date(end_date):
return end_date.format('YYYY-MM-DD')
def process_options(options):
def process_options(options, client_id):
"""
Processes and maps the incoming issuer options to fields/options that
Entrust understands
@ -78,11 +80,50 @@ def process_options(options):
"eku": "SERVER_AND_CLIENT_AUTH",
"certType": product_type,
"certExpiryDate": validity_end,
"tracking": tracking_data
"tracking": tracking_data,
"org": options.get("organization"),
"clientId": client_id
}
return data
@retry(stop_max_attempt_number=5, wait_fixed=1000)
def get_client_id(session, organization):
"""
Helper function for looking up clientID pased on Organization and parsing the response.
:param session:
:param organization: the validated org with Entrust, for instance "Company, Inc."
:return: ClientID
:raise Exception:
"""
# get the organization ID
url = current_app.config.get("ENTRUST_URL") + "/organizations"
try:
response = session.get(url, timeout=(15, 40))
except requests.exceptions.Timeout:
raise Exception("Timeout for Getting Organizations")
except requests.exceptions.RequestException as e:
raise Exception(f"Error for Getting Organization {e}")
# parse the response
try:
d = json.loads(response.content)
except ValueError:
# catch an empty json object here
d = {'response': 'No detailed message'}
found = False
for y in d["organizations"]:
if y["name"] == organization:
found = True
client_id = y["clientId"]
if found:
return client_id
else:
raise Exception(f"Error on Organization - Use on of the List: {d['organizations']}")
def handle_response(my_response):
"""
Helper function for parsing responses from the Entrust API.
@ -192,9 +233,20 @@ class EntrustIssuerPlugin(IssuerPlugin):
}
current_app.logger.info(log_data)
if current_app.config.get("ENTRUST_USE_DEFAULT_CLIENT_ID"):
# The ID of the primary client is 1.
client_id = 1
else:
client_id = get_client_id(self.session, issuer_options.get("organization"))
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": f"Organization id: {client_id}"
}
current_app.logger.info(log_data)
url = current_app.config.get("ENTRUST_URL") + "/certificates"
data = process_options(issuer_options)
data = process_options(issuer_options, client_id)
data["csr"] = csr
response_dict = order_and_download_certificate(self.session, url, data)
@ -202,11 +254,16 @@ class EntrustIssuerPlugin(IssuerPlugin):
external_id = response_dict['trackingId']
cert = response_dict['endEntityCert']
if len(response_dict['chainCerts']) < 2:
# certificate signed by CA directly, no ICA included ini the chain
# certificate signed by CA directly, no ICA included in the chain
chain = None
else:
chain = response_dict['chainCerts'][1]
if current_app.config.get("ENTRUST_CROSS_SIGNED_RSA_L1K") and get_key_type_from_certificate(cert) == "RSA2048":
chain = current_app.config.get("ENTRUST_CROSS_SIGNED_RSA_L1K")
if current_app.config.get("ENTRUST_CROSS_SIGNED_ECC_L1F") and get_key_type_from_certificate(cert) == "ECCPRIME256V1":
chain = current_app.config.get("ENTRUST_CROSS_SIGNED_ECC_L1F")
log_data["message"] = "Received Chain"
log_data["options"] = f"chain: {chain}"
current_app.logger.info(log_data)
@ -214,16 +271,20 @@ class EntrustIssuerPlugin(IssuerPlugin):
return cert, chain, external_id
@retry(stop_max_attempt_number=3, wait_fixed=1000)
def revoke_certificate(self, certificate, comments):
def revoke_certificate(self, certificate, reason):
"""Revoke an Entrust certificate."""
base_url = current_app.config.get("ENTRUST_URL")
# make certificate revoke request
revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations"
if not comments or comments == '':
if "comments" not in reason or reason["comments"] == '':
comments = "revoked via API"
crl_reason = CRLReason.unspecified
if "crl_reason" in reason:
crl_reason = CRLReason[reason["crl_reason"]]
data = {
"crlReason": "superseded", # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation)
"crlReason": crl_reason, # per RFC 5280 section 5.3.1
"revocationComment": comments
}
response = self.session.post(revoke_url, json=data)
@ -272,9 +333,81 @@ class EntrustSourcePlugin(SourcePlugin):
author = "sirferl"
author_url = "https://github.com/sirferl/lemur"
def __init__(self, *args, **kwargs):
"""Initialize the issuer with the appropriate details."""
required_vars = [
"ENTRUST_API_CERT",
"ENTRUST_API_KEY",
"ENTRUST_API_USER",
"ENTRUST_API_PASS",
"ENTRUST_URL",
"ENTRUST_ROOT",
"ENTRUST_NAME",
"ENTRUST_EMAIL",
"ENTRUST_PHONE",
]
validate_conf(current_app, required_vars)
self.session = requests.Session()
cert_file = current_app.config.get("ENTRUST_API_CERT")
key_file = current_app.config.get("ENTRUST_API_KEY")
user = current_app.config.get("ENTRUST_API_USER")
password = current_app.config.get("ENTRUST_API_PASS")
self.session.cert = (cert_file, key_file)
self.session.auth = (user, password)
self.session.hooks = dict(response=log_status_code)
super(EntrustSourcePlugin, self).__init__(*args, **kwargs)
def get_certificates(self, options, **kwargs):
# Not needed for ENTRUST
raise NotImplementedError("Not implemented\n", self, options, **kwargs)
""" Fetch all Entrust certificates """
base_url = current_app.config.get("ENTRUST_URL")
host = base_url.replace('/enterprise/v2', '')
get_url = f"{base_url}/certificates"
certs = []
processed_certs = 0
offset = 0
while True:
response = self.session.get(get_url,
params={
"status": "ACTIVE",
"isThirdParty": "false",
"fields": "uri,dn",
"offset": offset
}
)
try:
data = json.loads(response.content)
except ValueError:
# catch an empty jason object here
data = {'response': 'No detailed message'}
status_code = response.status_code
if status_code > 399:
raise Exception(f"ENTRUST error: {status_code}\n{data['errors']}")
for c in data["certificates"]:
download_url = "{0}{1}".format(
host, c["uri"]
)
cert_response = self.session.get(download_url)
certificate = json.loads(cert_response.content)
# normalize serial
serial = str(int(certificate["serialNumber"], 16))
cert = {
"body": certificate["endEntityCert"],
"serial": serial,
"external_id": str(certificate["trackingId"]),
"csr": certificate["csr"],
"owner": certificate["tracking"]["requesterEmail"],
"description": f"Imported by Lemur; Type: Entrust {certificate['certType']}\nExtended Key Usage: {certificate['eku']}"
}
certs.append(cert)
processed_certs += 1
if data["summary"]["limit"] * offset >= data["summary"]["total"]:
break
else:
offset += 1
current_app.logger.info(f"Retrieved {processed_certs} ertificates")
return certs
def get_endpoints(self, options, **kwargs):
# There are no endpoints in ENTRUST

7
lemur/plugins/lemur_entrust/tests/test_entrust.py
View File

@ -56,7 +56,10 @@ def test_process_options(mock_current_app, authority):
"requesterName": mock_current_app.config.get("ENTRUST_NAME"),
"requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"),
"requesterPhone": mock_current_app.config.get("ENTRUST_PHONE")
}
},
"org": "Example, Inc.",
"clientId": 1
}
assert expected == plugin.process_options(options)
client_id = 1
assert expected == plugin.process_options(options, client_id)

11
lemur/plugins/lemur_kubernetes/plugin.py
View File

@ -10,7 +10,6 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
"""
import base64
import itertools
import os
@ -18,7 +17,7 @@ import requests
from flask import current_app
from lemur.common.defaults import common_name
from lemur.common.utils import parse_certificate
from lemur.common.utils import parse_certificate, base64encode
from lemur.plugins.bases import DestinationPlugin
DEFAULT_API_VERSION = "v1"
@ -73,12 +72,6 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V
)
# Performs Base64 encoding of string to string using the base64.b64encode() function
# which encodes bytes to bytes.
def base64encode(string):
return base64.b64encode(string.encode()).decode()
def build_secret(secret_format, secret_name, body, private_key, cert_chain):
secret = {
"apiVersion": "v1",
@ -96,7 +89,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain):
if secret_format == "TLS":
secret["type"] = "kubernetes.io/tls"
secret["data"] = {
"tls.crt": base64encode(body),
"tls.crt": base64encode("%s\n%s" % (body, cert_chain)),
"tls.key": base64encode(private_key),
}
if secret_format == "Certificate":

202
lemur/plugins/lemur_sftp/plugin.py
View File

@ -16,8 +16,10 @@
.. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov
"""
from os import path
import paramiko
from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError
from flask import current_app
from lemur.plugins import lemur_sftp
@ -95,33 +97,15 @@ class SFTPDestinationPlugin(DestinationPlugin):
},
]
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
current_app.logger.debug("SFTP destination plugin is started")
cn = common_name(parse_certificate(body))
def open_sftp_connection(self, options):
host = self.get_option("host", options)
port = self.get_option("port", options)
user = self.get_option("user", options)
password = self.get_option("password", options)
ssh_priv_key = self.get_option("privateKeyPath", options)
ssh_priv_key_pass = self.get_option("privateKeyPass", options)
dst_path = self.get_option("destinationPath", options)
export_format = self.get_option("exportFormat", options)
# prepare files for upload
files = {cn + ".key": private_key, cn + ".pem": body}
if cert_chain:
if export_format == "NGINX":
# assemble body + chain in the single file
files[cn + ".pem"] += "\n" + cert_chain
elif export_format == "Apache":
# store chain in the separate file
files[cn + ".ca.bundle.pem"] = cert_chain
# upload files
# delete files
try:
current_app.logger.debug(
"Connecting to {0}@{1}:{2}".format(user, host, port)
@ -145,50 +129,170 @@ class SFTPDestinationPlugin(DestinationPlugin):
current_app.logger.error(
"No password or private key provided. Can't proceed"
)
raise paramiko.ssh_exception.AuthenticationException
raise AuthenticationException
# open the sftp session inside the ssh connection
sftp = ssh.open_sftp()
return ssh.open_sftp(), ssh
# make sure that the destination path exist
try:
current_app.logger.debug("Creating {0}".format(dst_path))
sftp.mkdir(dst_path)
except IOError:
current_app.logger.debug("{0} already exist, resuming".format(dst_path))
try:
dst_path_cn = dst_path + "/" + cn
current_app.logger.debug("Creating {0}".format(dst_path_cn))
sftp.mkdir(dst_path_cn)
except IOError:
current_app.logger.debug(
"{0} already exist, resuming".format(dst_path_cn)
)
except AuthenticationException as e:
current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.")
except NoValidConnectionsError as e:
current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
raise NoValidConnectionsError("Couldn't connect to {0}, possible timeout or invalid hostname")
# upload certificate files to the sftp destination
for filename, data in files.items():
# this is called when using this as a default destination plugin
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
current_app.logger.debug("SFTP destination plugin is started")
cn = common_name(parse_certificate(body))
dst_path = self.get_option("destinationPath", options)
dst_path_cn = dst_path + "/" + cn
export_format = self.get_option("exportFormat", options)
# prepare files for upload
files = {cn + ".key": private_key, cn + ".pem": body}
if cert_chain:
if export_format == "NGINX":
# assemble body + chain in the single file
files[cn + ".pem"] += "\n" + cert_chain
elif export_format == "Apache":
# store chain in the separate file
files[cn + ".ca.bundle.pem"] = cert_chain
self.upload_file(dst_path_cn, files, options)
# this is called from the acme http challenge
def upload_acme_token(self, token_path, token, options, **kwargs):
current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge")
dst_path = self.get_option("destinationPath", options)
_, filename = path.split(token_path)
# prepare files for upload
files = {filename: token}
self.upload_file(dst_path, files, options)
# this is called from the acme http challenge
def delete_acme_token(self, token_path, options, **kwargs):
dst_path = self.get_option("destinationPath", options)
_, filename = path.split(token_path)
# prepare files for upload
files = {filename: None}
self.delete_file(dst_path, files, options)
# here the file is deleted
def delete_file(self, dst_path, files, options):
try:
# open the ssh and sftp sessions
sftp, ssh = self.open_sftp_connection(options)
# delete files
for filename, _ in files.items():
current_app.logger.debug(
"Uploading {0} to {1}".format(filename, dst_path_cn)
"Deleting {0} from {1}".format(filename, dst_path)
)
try:
with sftp.open(dst_path_cn + "/" + filename, "w") as f:
f.write(data)
except (PermissionError) as permerror:
sftp.remove(path.join(dst_path, filename))
except PermissionError as permerror:
if permerror.errno == 13:
current_app.logger.debug(
"Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn)
"Deleting {0} from {1} returned Permission Denied Error, making file writable and retrying".format(
filename, dst_path)
)
sftp.chmod(dst_path_cn + "/" + filename, 0o600)
with sftp.open(dst_path_cn + "/" + filename, "w") as f:
f.write(data)
# read only for owner, -r--------
sftp.chmod(dst_path_cn + "/" + filename, 0o400)
sftp.chmod(path.join(dst_path, filename), 0o600)
sftp.remove(path.join(dst_path, filename))
ssh.close()
except (AuthenticationException, NoValidConnectionsError) as e:
raise e
except Exception as e:
current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
try:
ssh.close()
except BaseException:
pass
# here the file is uploaded for real, this helps to keep this class DRY
def upload_file(self, dst_path, files, options):
try:
# open the ssh and sftp sessions
sftp, ssh = self.open_sftp_connection(options)
# split the path into it's segments, so we can create it recursively
allparts = []
path_copy = dst_path
while True:
parts = path.split(path_copy)
if parts[0] == path_copy: # sentinel for absolute paths
allparts.insert(0, parts[0])
break
elif parts[1] == path_copy: # sentinel for relative paths
allparts.insert(0, parts[1])
break
else:
path_copy = parts[0]
allparts.insert(0, parts[1])
# make sure that the destination path exists, recursively
remote_path = allparts[0]
for part in allparts:
try:
if part != "/" and part != "":
remote_path = path.join(remote_path, part)
sftp.stat(remote_path)
except IOError:
current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path))
try:
sftp.mkdir(remote_path)
except IOError as ioerror:
current_app.logger.debug(
"Couldn't create {0}, error message: {1}".format(remote_path, ioerror))
# upload certificate files to the sftp destination
for filename, data in files.items():
current_app.logger.debug(
"Uploading {0} to {1}".format(filename, dst_path)
)
try:
with sftp.open(path.join(dst_path, filename), "w") as f:
f.write(data)
except PermissionError as permerror:
if permerror.errno == 13:
current_app.logger.debug(
"Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(
filename, dst_path)
)
sftp.chmod(path.join(dst_path, filename), 0o600)
with sftp.open(path.join(dst_path, filename), "w") as f:
f.write(data)
# most likely the upload user isn't the webuser, -rw-r--r--
sftp.chmod(path.join(dst_path, filename), 0o644)
ssh.close()
except (AuthenticationException, NoValidConnectionsError) as e:
raise e
except Exception as e:
current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
try:
ssh.close()
except BaseException:
pass
message = ''
if hasattr(e, 'errors'):
for _, error in e.errors.items():
message = error.strerror
raise Exception(
'Couldn\'t upload file to {}, error message: {}'.format(self.get_option("host", options), message))

144
lemur/plugins/lemur_sftp/tests/test_sftp.py Normal file
View File

@ -0,0 +1,144 @@
import unittest
from unittest.mock import patch, Mock, MagicMock, mock_open
from flask import Flask
from lemur.plugins.lemur_sftp import plugin
from paramiko.ssh_exception import AuthenticationException
class TestSftp(unittest.TestCase):
def setUp(self):
self.sftp_destination = plugin.SFTPDestinationPlugin()
# Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
# needed to run tests in dev environment without getting error 'Working outside of application context'.
_app = Flask('lemur_test_sftp')
self.ctx = _app.app_context()
assert self.ctx
self.ctx.push()
def tearDown(self):
self.ctx.pop()
def test_failing_ssh_connection(self):
dst_path = '/var/non-existent'
files = {'first-file': 'data'}
options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
{'name': 'user', 'value': 'test_acme'}]
with self.assertRaises(AuthenticationException):
self.sftp_destination.upload_file(dst_path, files, options)
@patch("lemur.plugins.lemur_sftp.plugin.paramiko")
def test_upload_file_single_with_password(self, mock_paramiko):
dst_path = '/var/non-existent'
files = {'first-file': 'data'}
options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
{'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}]
mock_sftp = Mock()
mock_sftp.open = mock_open()
mock_ssh = mock_paramiko.SSHClient.return_value
mock_ssh.connect = MagicMock()
mock_ssh.open_sftp.return_value = mock_sftp
self.sftp_destination.upload_file(dst_path, files, options)
mock_sftp.open.assert_called_once_with('/var/non-existent/first-file', 'w')
handle = mock_sftp.open()
handle.write.assert_called_once_with('data')
mock_ssh.close.assert_called_once()
mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
password='test_password')
@patch("lemur.plugins.lemur_sftp.plugin.paramiko")
def test_upload_file_multiple_with_key(self, mock_paramiko):
dst_path = '/var/non-existent'
files = {'first-file': 'data', 'second-file': 'data2'}
options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
{'name': 'user', 'value': 'test_acme'}, {'name': 'privateKeyPath', 'value': '/var/id_rsa'},
{'name': 'privateKeyPass', 'value': 'ssh-key-password'}]
mock_sftp = Mock()
mock_sftp.open = mock_open()
mock_paramiko.RSAKey.from_private_key_file.return_value = 'ssh-rsa test-key'
mock_ssh = mock_paramiko.SSHClient.return_value
mock_ssh.connect = MagicMock()
mock_ssh.open_sftp.return_value = mock_sftp
self.sftp_destination.upload_file(dst_path, files, options)
mock_sftp.open.assert_called_with('/var/non-existent/second-file', 'w')
handle = mock_sftp.open()
handle.write.assert_called_with('data2')
mock_ssh.close.assert_called_once()
mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/var/id_rsa', 'ssh-key-password')
mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
pkey='ssh-rsa test-key')
@patch("lemur.plugins.lemur_sftp.plugin.paramiko")
def test_upload_acme_token(self, mock_paramiko):
token_path = './well-known/acme-challenge/some-token-path'
token = 'token-data'
options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
{'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'},
{'name': 'destinationPath', 'value': '/var/destination-path'}]
mock_sftp = Mock()
mock_sftp.open = mock_open()
mock_ssh = mock_paramiko.SSHClient.return_value
mock_ssh.connect = MagicMock()
mock_ssh.open_sftp.return_value = mock_sftp
self.sftp_destination.upload_acme_token(token_path, token, options)
mock_sftp.open.assert_called_once_with('/var/destination-path/some-token-path', 'w')
handle = mock_sftp.open()
handle.write.assert_called_once_with('token-data')
mock_ssh.close.assert_called_once()
mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
password='test_password')
@patch("lemur.plugins.lemur_sftp.plugin.paramiko")
def test_delete_file_with_password(self, mock_paramiko):
dst_path = '/var/non-existent'
files = {'first-file': None}
options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
{'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}]
mock_sftp = Mock()
mock_ssh = mock_paramiko.SSHClient.return_value
mock_ssh.connect = MagicMock()
mock_ssh.open_sftp.return_value = mock_sftp
self.sftp_destination.delete_file(dst_path, files, options)
mock_sftp.remove.assert_called_once_with('/var/non-existent/first-file')
mock_ssh.close.assert_called_once()
mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
password='test_password')
@patch("lemur.plugins.lemur_sftp.plugin.paramiko")
def test_delete_acme_token(self, mock_paramiko):
token_path = './well-known/acme-challenge/some-token-path'
options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
{'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'},
{'name': 'destinationPath', 'value': '/var/destination-path'}]
mock_sftp = Mock()
mock_ssh = mock_paramiko.SSHClient.return_value
mock_ssh.connect = MagicMock()
mock_ssh.open_sftp.return_value = mock_sftp
self.sftp_destination.delete_acme_token(token_path, options)
mock_sftp.remove.assert_called_once_with('/var/destination-path/some-token-path')
mock_ssh.close.assert_called_once()
mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
password='test_password')

11
lemur/roles/service.py
View File

@ -12,6 +12,7 @@
from lemur import database
from lemur.roles.models import Role
from lemur.users.models import User
from lemur.logs import service as log_service
def update(role_id, name, description, users):
@ -29,6 +30,8 @@ def update(role_id, name, description, users):
role.description = description
role.users = users
database.update(role)
log_service.audit_log("update_role", name, f"Role with id {role_id} updated")
return role
@ -44,6 +47,8 @@ def set_third_party(role_id, third_party_status=False):
role = get(role_id)
role.third_party = third_party_status
database.update(role)
log_service.audit_log("update_role", role.name, f"Updated third_party_status={third_party_status}")
return role
@ -71,6 +76,7 @@ def create(
if users:
role.users = users
log_service.audit_log("create_role", name, "Creating new role")
return database.create(role)
@ -101,7 +107,10 @@ def delete(role_id):
:param role_id:
:return:
"""
return database.delete(get(role_id))
role = get(role_id)
log_service.audit_log("delete_role", role.name, "Deleting role")
return database.delete(role)
def render(args):

2
lemur/roles/views.py
View File

@ -106,6 +106,7 @@ class RolesList(AuthenticatedResource):
POST /roles HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "role3",
@ -265,6 +266,7 @@ class Roles(AuthenticatedResource):
PUT /roles/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"name": "role1",

21
lemur/sources/views.py
View File

@ -106,6 +106,7 @@ class SourcesList(AuthenticatedResource):
POST /sources HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"options": [
@ -156,12 +157,19 @@ class SourcesList(AuthenticatedResource):
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
"""
return service.create(
data["label"],
data["plugin"]["slug"],
data["plugin"]["plugin_options"],
data["description"],
)
if "plugin_options" in data["plugin"]:
return service.create(
data["label"],
data["plugin"]["slug"],
data["plugin"]["plugin_options"],
data["description"],
)
else:
return service.create(
data["label"],
data["plugin"]["slug"],
data["description"],
)
class Sources(AuthenticatedResource):
@ -230,6 +238,7 @@ class Sources(AuthenticatedResource):
POST /sources/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"options": [

4
lemur/static/app/angular/api_keys/api_key/api_key.tpl.html
View File

@ -51,8 +51,8 @@
<div class="modal-body" ng-show="jwt">
<h4>Pass the following token on every Lemur API request:</h4>
<pre><code>{{ jwt }}</code></pre>
<h4>Example usuage:</h4>
<pre><code>curl -i {{ origin }}/certificates -H "Authorization: Bearer {{ jwt }}</code></pre>
<h4>Example usage:</h4>
<pre><code>curl -i {{ origin }}/api/1/certificates -H "Authorization: Bearer {{ jwt }}"</code></pre>
</div>
<div class="modal-footer" ng-show="!jwt">
<button ng-click="save(apiKey)" type="submit" ng-disabled="createForm.$invalid" class="btn btn-primary">Save</button>

8
lemur/static/app/angular/authorities/authority/authority.js vendored
View File

@ -34,7 +34,7 @@ angular.module('lemur')
};
})
.controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster) {
.controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster, DestinationService) {
$scope.authority = LemurRestangular.restangularizeElement(null, {}, 'authorities');
// set the defaults
AuthorityService.getDefaults($scope.authority).then(function () {
@ -52,6 +52,12 @@ angular.module('lemur')
});
});
$scope.getDestinations = function() {
return DestinationService.findDestinationsByName('').then(function(destinations) {
$scope.destinations = destinations;
});
};
$scope.getAuthoritiesByName = function (value) {
return AuthorityService.findAuthorityByName(value).then(function (authorities) {
$scope.authorities = authorities;

17
lemur/static/app/angular/authorities/authority/options.tpl.html
View File

@ -66,11 +66,28 @@
<div class="col-sm-10">
<input name="sub" ng-if="item.type == 'int'" type="number" ng-pattern="item.validation?item.validation:'^[0-9]+$'"
class="form-control" ng-model="item.value"/>
<select name="sub" ng-if="item.type == 'select'" class="form-control" ng-options="i for i in item.available"
ng-model="item.value"></select>
<!-- DestSelect options -->
<ui-select class="input-md" ng-model="item.value" theme="bootstrap" title="choose a destination" ng-if="item.type == 'destinationSelect'">
<ui-select-match placeholder="select an destination...">{{$select.selected.label}}</ui-select-match>
<ui-select-choices class="form-control"
refresh="getDestinations()"
refresh-delay="300"
repeat="destination.id as destination in destinations | filter: $select.search">
<div ng-bind-html="destination.label | highlight: $select.search"></div>
<small>
<span ng-bind-html="''+destination.description | highlight: $select.search"></span>
</small>
</ui-select-choices>
</ui-select>
<input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox" ng-model="item.value">
<input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value"/>
<textarea name="sub" ng-if="item.type == 'textarea'" class="form-control" ng-model="item.value"></textarea>
<div ng-if="item.type == 'export-plugin'">
<form name="exportForm" class="form-horizontal" role="form" novalidate>
<select class="form-control" ng-model="item.value"

4
lemur/static/app/angular/certificates/certificate/certificate.js vendored
View File

@ -419,8 +419,8 @@ angular.module('lemur')
$uibModalInstance.dismiss('cancel');
};
$scope.revoke = function (certificate) {
CertificateService.revoke(certificate).then(
$scope.revoke = function (certificate, crlReason) {
CertificateService.revoke(certificate, crlReason).then(
function () {
toaster.pop({
type: 'success',

1
lemur/static/app/angular/certificates/certificate/export.tpl.html
View File

@ -25,6 +25,7 @@
<input name="sub" ng-if="item.type == 'int'" type="number" ng-pattern="/^[0-9]{12,12}$/"
class="form-control" ng-model="item.value"/>
<select name="sub" ng-if="item.type == 'select'" class="form-control"
ng-init="item.value = item.available[0]"
ng-options="i for i in item.available" ng-model="item.value"></select>
<input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox"
ng-model="item.value">

2
lemur/static/app/angular/certificates/certificate/options.tpl.html
View File

@ -33,7 +33,7 @@
<div class="col-sm-10">
<select class="form-control" ng-model="certificate.keyType"
ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1']"
ng-init="certificate.keyType = 'RSA2048'"></select>
ng-init="certificate.keyType = 'ECCPRIME256V1'"></select>
</div>
</div>
<div class="form-group">

32
lemur/static/app/angular/certificates/certificate/revoke.tpl.html
View File

@ -4,13 +4,13 @@
<h3 class="modal-title">Revoke <span class="text-muted"><small>{{ certificate.name }}</small></span></h3>
</div>
<div class="modal-body">
<form name="revokeForm" ng-if="!certificate.endpoints.length" novalidate>
<form name="revokeForm" novalidate>
<p><strong>Certificate revocation is final. Once revoked the certificate is no longer valid.</strong></p>
<div class="form-horizontal">
<div class="form-group"
ng-class="{'has-error': revokeForm.confirm.$invalid, 'has-success': !revokeForm.$invalid&&revokeForm.confirm.$dirty}">
<label class="control-label col-sm-2">
Confirm Revocation
Confirm Certificate Name
</label>
<div class="col-sm-10">
<input name="confirm" ng-model="confirm" placeholder='{{ certificate.name }}'
@ -23,12 +23,32 @@
You must confirm certificate revocation.</p>
</div>
</div>
<div class="form-group">
<label class="control-label col-sm-2">
Reason
</label>
<div class="col-sm-10">
<select class="form-control" ng-model="crlReason"
ng-options="option.value as option.name for option in [
{'name': 'Unspecified', 'value': 'unspecified'},
{'name': 'Key Compromise', 'value': 'keyCompromise'},
{'name': 'CA Compromise', 'value': 'cACompromise'},
{'name': 'Affiliation Changed', 'value': 'affiliationChanged'},
{'name': 'Superseded', 'value': 'superseded'},
{'name': 'Cessation of Operation', 'value': 'cessationOfOperation'},
{'name': 'Certificate Hold', 'value': 'certificateHold'},
{'name': 'Remove from CRL', 'value': 'removeFromCRL'},
{'name': 'Privilege Withdrawn', 'value': 'privilegeWithdrawn'},
{'name': 'Attribute Authority Compromise', 'value': 'aACompromise'}]"
ng-init="crlReason = 'unspecified'"></select>
</div>
</div>
</div>
</form>
<div ng-if="certificate.endpoints.length">
<p><strong>Certificate cannot be revoked, it is associated with the following endpoints. Disassociate this
certificate
before revoking.</strong></p>
<p><strong>Certificate might be associated with the following endpoints. Disassociate this
certificate before revoking or continue if you've already done so.</strong></p>
<ul class="list-group">
<li class="list-group-item" ng-repeat="endpoint in certificate.endpoints">
<span class="pull-right"><label class="label label-default">{{ endpoint.type }}</label></span>
@ -41,7 +61,7 @@
</div>
</div>
<div class="modal-footer">
<button type="submit" ng-click="revoke(certificate)" ng-disabled="revokeForm.confirm.$invalid"
<button type="submit" ng-click="revoke(certificate, crlReason)" ng-disabled="revokeForm.confirm.$invalid"
class="btn btn-danger">Revoke
</button>
<button ng-click="cancel()" class="btn">Cancel</button>

4
lemur/static/app/angular/certificates/services.js vendored
View File

@ -313,8 +313,8 @@ angular.module('lemur')
return certificate.customPOST(certificate.exportOptions, 'export');
};
CertificateService.revoke = function (certificate) {
return certificate.customPUT({}, 'revoke');
CertificateService.revoke = function (certificate, crlReason) {
return certificate.customPUT({'crlReason':crlReason}, 'revoke');
};
return CertificateService;

2
lemur/static/app/angular/notifications/view/view.tpl.html
View File

@ -1,7 +1,7 @@
<div class="row">
<div class="col-md-12">
<h2 class="featurette-heading">Notifications
<span class="text-muted"><small>you have to speak up son!</small></span></h2>
<span class="text-muted"><small>you have to speak up!</small></span></h2>
<div class="panel panel-default">
<div class="panel-heading">
<div class="btn-group pull-right">

12
lemur/tests/conftest.py
View File

@ -56,7 +56,7 @@ def pytest_runtest_makereport(item, call):
parent._previousfailed = item
@pytest.yield_fixture(scope="session")
@pytest.fixture(scope="session")
def app(request):
"""
Creates a new Flask application for a test duration.
@ -73,7 +73,7 @@ def app(request):
ctx.pop()
@pytest.yield_fixture(scope="session")
@pytest.fixture(scope="session")
def db(app, request):
_db.drop_all()
_db.engine.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
@ -92,7 +92,7 @@ def db(app, request):
_db.drop_all()
@pytest.yield_fixture(scope="function")
@pytest.fixture(scope="function")
def session(db, request):
"""
Creates a new database session with (with working transaction)
@ -103,7 +103,7 @@ def session(db, request):
db.session.rollback()
@pytest.yield_fixture(scope="function")
@pytest.fixture(scope="function")
def client(app, session, client):
yield client
@ -276,14 +276,14 @@ def source_plugin():
return TestSourcePlugin
@pytest.yield_fixture(scope="function")
@pytest.fixture(scope="function")
def logged_in_user(session, app):
with app.test_request_context():
identity_changed.send(current_app._get_current_object(), identity=Identity(1))
yield
@pytest.yield_fixture(scope="function")
@pytest.fixture(scope="function")
def logged_in_admin(session, app):
with app.test_request_context():
identity_changed.send(current_app._get_current_object(), identity=Identity(2))

62
lemur/tests/test_certificates.py
View File

@ -103,6 +103,30 @@ def test_delete_cert(session):
assert not cert_exists
def test_cleanup_after_revoke(session, issuer_plugin, crypto_authority):
from lemur.certificates.service import cleanup_after_revoke, get
from lemur.tests.factories import CertificateFactory
revoke_this = CertificateFactory(name="REVOKEME")
session.commit()
to_be_revoked = get(revoke_this.id)
assert to_be_revoked
to_be_revoked.notify = True
to_be_revoked.rotation = True
# Assuming the cert is revoked by corresponding issuer, update the records in lemur
cleanup_after_revoke(to_be_revoked)
revoked_cert = get(to_be_revoked.id)
# then not exist after delete
assert revoked_cert
assert revoked_cert.status == "revoked"
assert not revoked_cert.notify
assert not revoked_cert.rotation
assert not revoked_cert.destinations
def test_get_by_attributes(session, certificate):
from lemur.certificates.service import get_by_attributes
@ -301,6 +325,7 @@ def test_certificate_input_schema(client, authority):
# make sure the defaults got set
assert data["common_name"] == "test.example.com"
assert data["country"] == "US"
assert data["key_type"] == "ECCPRIME256V1"
assert len(data.keys()) == 19
@ -325,10 +350,12 @@ def test_certificate_input_with_extensions(client, authority):
},
},
"dnsProvider": None,
"keyType": "RSA2048"
}
data, errors = CertificateInputSchema().load(input_data)
assert not errors
assert data["key_type"] == "RSA2048"
def test_certificate_input_schema_parse_csr(authority):
@ -363,9 +390,11 @@ def test_certificate_input_schema_parse_csr(authority):
data, errors = CertificateInputSchema().load(input_data)
assert not errors
for san in data["extensions"]["sub_alt_names"]["names"]:
assert san.value == test_san_dns
assert not errors
assert data["key_type"] == "RSA2048"
def test_certificate_out_of_range_date(client, authority):
@ -658,6 +687,23 @@ def test_certificate_upload_schema_wrong_chain_2nd(client):
}
def test_certificate_revoke_schema():
from lemur.certificates.schemas import CertificateRevokeSchema
input = {
"comments": "testing certificate revoke schema",
"crl_reason": "cessationOfOperation"
}
data, errors = CertificateRevokeSchema().load(input)
assert not errors
input["crl_reason"] = "fakeCrlReason"
data, errors = CertificateRevokeSchema().load(input)
assert errors == {
"crl_reason": ['Not a valid choice.']
}
def test_create_basic_csr(client):
csr_config = dict(
common_name="example.com",
@ -1336,3 +1382,17 @@ def test_boolean_filter(client):
headers=VALID_ADMIN_HEADER_TOKEN,
)
assert resp.status_code == 200
def test_issued_cert_count_for_authority(authority):
from lemur.tests.factories import CertificateFactory
from lemur.certificates.service import get_issued_cert_count_for_authority
assert get_issued_cert_count_for_authority(authority) == 0
# create a few certs issued by the authority
CertificateFactory(authority=authority, name="test_issued_cert_count_for_authority1")
CertificateFactory(authority=authority, name="test_issued_cert_count_for_authority2")
CertificateFactory(authority=authority, name="test_issued_cert_count_for_authority3")
assert get_issued_cert_count_for_authority(authority) == 3

83
lemur/tests/test_messaging.py
View File

@ -5,6 +5,7 @@ import boto3
import pytest
from freezegun import freeze_time
from moto import mock_ses
from lemur.tests.factories import AuthorityFactory, CertificateFactory, EndpointFactory
@mock_ses
@ -111,6 +112,28 @@ def test_send_expiration_notification_with_no_notifications(
assert send_expiration_notifications([]) == (0, 0)
@mock_ses
def test_send_expiration_summary_notification(certificate, notification, notification_plugin):
from lemur.notifications.messaging import send_security_expiration_summary
verify_sender_email()
# we don't actually test the email contents, but adding an assortment of certs here is useful for step debugging
# to confirm the produced email body looks like we expect
create_cert_that_expires_in_days(14)
create_cert_that_expires_in_days(12)
create_cert_that_expires_in_days(9)
create_cert_that_expires_in_days(7)
create_cert_that_expires_in_days(7)
create_cert_that_expires_in_days(2)
create_cert_that_expires_in_days(30)
create_cert_that_expires_in_days(15)
create_cert_that_expires_in_days(20)
create_cert_that_expires_in_days(1)
create_cert_that_expires_in_days(100)
assert send_security_expiration_summary([])
@mock_ses
def test_send_rotation_notification(notification_plugin, certificate):
from lemur.notifications.messaging import send_rotation_notification
@ -125,3 +148,63 @@ def test_send_pending_failure_notification(notification_plugin, async_issuer_plu
verify_sender_email()
assert send_pending_failure_notification(pending_certificate)
def test_get_authority_certificates():
from lemur.notifications.messaging import get_expiring_authority_certificates
certificate_1 = create_ca_cert_that_expires_in_days(180)
certificate_2 = create_ca_cert_that_expires_in_days(365)
create_ca_cert_that_expires_in_days(364)
create_ca_cert_that_expires_in_days(366)
create_ca_cert_that_expires_in_days(179)
create_ca_cert_that_expires_in_days(181)
create_ca_cert_that_expires_in_days(1)
assert set(get_expiring_authority_certificates()) == {certificate_1, certificate_2}
@mock_ses
def test_send_authority_expiration_notifications():
from lemur.notifications.messaging import send_authority_expiration_notifications
verify_sender_email()
create_ca_cert_that_expires_in_days(180)
create_ca_cert_that_expires_in_days(180) # two on the same day results in a single email
create_ca_cert_that_expires_in_days(365)
create_ca_cert_that_expires_in_days(364)
create_ca_cert_that_expires_in_days(366)
create_ca_cert_that_expires_in_days(179)
create_ca_cert_that_expires_in_days(181)
create_ca_cert_that_expires_in_days(1)
assert send_authority_expiration_notifications() == (2, 0)
def create_ca_cert_that_expires_in_days(days):
now = arrow.utcnow()
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
authority = AuthorityFactory()
certificate = CertificateFactory()
certificate.not_after = not_after
certificate.notify = True
certificate.root_authority_id = authority.id
certificate.authority_id = None
return certificate
def create_cert_that_expires_in_days(days):
from random import randrange
now = arrow.utcnow()
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
certificate = CertificateFactory()
certificate.not_after = not_after
certificate.notify = True
endpoints = []
for i in range(0, randrange(0, 5)):
endpoints.append(EndpointFactory())
certificate.endpoints = endpoints
return certificate

14
lemur/users/service.py
View File

@ -8,6 +8,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from lemur import database
from lemur.logs import service as log_service
from lemur.users.models import User
@ -31,6 +32,7 @@ def create(username, password, email, active, profile_picture, roles):
profile_picture=profile_picture,
)
user.roles = roles
log_service.audit_log("create_user", username, "Creating new user")
return database.create(user)
@ -52,6 +54,8 @@ def update(user_id, username, email, active, profile_picture, roles):
user.active = active
user.profile_picture = profile_picture
update_roles(user, roles)
log_service.audit_log("update_user", username, f"Updating user with id {user_id}")
return database.update(user)
@ -64,19 +68,29 @@ def update_roles(user, roles):
:param user:
:param roles:
"""
removed_roles = []
for ur in user.roles:
for r in roles:
if r.id == ur.id:
break
else:
user.roles.remove(ur)
removed_roles.append(ur.name)
if removed_roles:
log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
added_roles = []
for r in roles:
for ur in user.roles:
if r.id == ur.id:
break
else:
user.roles.append(r)
added_roles.append(r.name)
if added_roles:
log_service.audit_log("assign_role", user.username, f"Assigning roles {added_roles}")
def get(user_id):

2
lemur/users/views.py
View File

@ -108,6 +108,7 @@ class UsersList(AuthenticatedResource):
POST /users HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"username": "user3",
@ -208,6 +209,7 @@ class Users(AuthenticatedResource):
PUT /users/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
Content-Type: application/json;charset=UTF-8
{
"username": "user1",