Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into ADCS-plugin

Browse Source
This commit is contained in:
sirferl
2019-01-28 17:57:46 +01:00
committed by GitHub
parent a1ca61d813 4842bb0880
commit c77ccdf46e
19 changed files with 267 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lemur/authorities/service.py
View File

@ -15,6 +15,7 @@ from lemur import database
from lemur.common.utils import truthiness
from lemur.extensions import metrics
from lemur.authorities.models import Authority
from lemur.certificates.models import Certificate
from lemur.roles import service as role_service
from lemur.certificates.service import upload
@ -179,7 +180,12 @@ def render(args):
if 'active' in filt:
query = query.filter(Authority.active == truthiness(terms[1]))
elif 'cn' in filt:
query = query.join(Authority.active == truthiness(terms[1]))
term = '%{0}%'.format(terms[1])
sub_query = database.session_query(Certificate.root_authority_id) \
.filter(Certificate.cn.ilike(term)) \
.subquery()
query = query.filter(Authority.id.in_(sub_query))
else:
query = database.filter(query, Authority, terms)

14
lemur/certificates/models.py
View File

@ -19,7 +19,7 @@ from sqlalchemy.sql.expression import case, extract
from sqlalchemy_utils.types.arrow import ArrowType
from werkzeug.utils import cached_property
from lemur.common import defaults, utils
from lemur.common import defaults, utils, validators
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.database import db
from lemur.domains.models import Domain
@ -186,6 +186,18 @@ class Certificate(db.Model):
for domain in defaults.domains(cert):
self.domains.append(Domain(name=domain))
# Check integrity before saving anything into the database.
# For user-facing API calls, validation should also be done in schema validators.
self.check_integrity()
def check_integrity(self):
"""
Integrity checks: Does the cert have a matching private key?
"""
if self.private_key:
validators.verify_private_key_match(utils.parse_private_key(self.private_key), self.parsed_cert,
error_class=AssertionError)
@cached_property
def parsed_cert(self):
assert self.body, "Certificate body not set"

26
lemur/certificates/schemas.py
View File

@ -10,7 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.common import validators, missing
from lemur.common import missing, utils, validators
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.constants import CERTIFICATE_KEY_TYPES
@ -242,8 +242,8 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
authority = fields.Nested(AssociatedAuthoritySchema, required=False)
notify = fields.Boolean(missing=True)
external_id = fields.String(missing=None, allow_none=True)
private_key = fields.String(validate=validators.private_key)
body = fields.String(required=True, validate=validators.public_certificate)
private_key = fields.String()
body = fields.String(required=True)
chain = fields.String(validate=validators.public_certificate, missing=None,
allow_none=True) # TODO this could be multiple certificates
@ -258,6 +258,26 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
if not data.get('private_key'):
raise ValidationError('Destinations require private key.')
@validates_schema
def validate_cert_private_key(self, data):
cert = None
key = None
if data.get('body'):
try:
cert = utils.parse_certificate(data['body'])
except ValueError:
raise ValidationError("Public certificate presented is not valid.", field_names=['body'])
if data.get('private_key'):
try:
key = utils.parse_private_key(data['private_key'])
except ValueError:
raise ValidationError("Private key presented is not valid.", field_names=['private_key'])
if cert and key:
# Throws ValidationError
validators.verify_private_key_match(key, cert)
class CertificateExportInputSchema(LemurInputSchema):
plugin = fields.Nested(PluginInputSchema)

9
lemur/certificates/service.py
View File

@ -20,7 +20,6 @@ from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.extensions import metrics, sentry, signals
from lemur.models import certificate_associations
from lemur.notifications.models import Notification
from lemur.pending_certificates.models import PendingCertificate
from lemur.plugins.base import plugins
@ -307,7 +306,7 @@ def render(args):
if filt:
terms = filt.split(';')
term = '%{0}%'.format(terms[1])
term = '{0}%'.format(terms[1])
# Exact matches for quotes. Only applies to name, issuer, and cn
if terms[1].startswith('"') and terms[1].endswith('"'):
term = terms[1][1:-1]
@ -341,13 +340,13 @@ def render(args):
elif 'id' in terms:
query = query.filter(Certificate.id == cast(terms[1], Integer))
elif 'name' in terms:
query = query.outerjoin(certificate_associations).outerjoin(Domain).filter(
query = query.filter(
or_(
Certificate.name.ilike(term),
Domain.name.ilike(term),
Certificate.domains.any(Domain.name.ilike(term)),
Certificate.cn.ilike(term),
)
).group_by(Certificate.id)
)
else:
query = database.filter(query, Certificate, terms)

26
lemur/common/utils.py
View File

@ -12,7 +12,9 @@ import string
import sqlalchemy
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import rsa, ec
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from flask_restful.reqparse import RequestParser
from sqlalchemy import and_, func
@ -52,6 +54,20 @@ def parse_certificate(body):
return x509.load_pem_x509_certificate(body, default_backend())
def parse_private_key(private_key):
"""
Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm).
Raises ValueError for an invalid string.
:param private_key: String containing PEM private key
"""
if isinstance(private_key, str):
private_key = private_key.encode('utf8')
return load_pem_private_key(private_key, password=None, backend=default_backend())
def parse_csr(csr):
"""
Helper function that parses a CSR.
@ -211,3 +227,13 @@ def truthiness(s):
"""If input string resembles something truthy then return True, else False."""
return s.lower() in ('true', 'yes', 'on', 't', '1')
def find_matching_certificates_by_hash(cert, matching_certs):
"""Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
determine if any of the certificate hashes match and return the matches."""
matching = []
for c in matching_certs:
if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()):
matching.append(c)
return matching

33
lemur/common/validators.py
View File

@ -2,14 +2,12 @@ import re
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.x509 import NameOID
from flask import current_app
from marshmallow.exceptions import ValidationError
from lemur.auth.permissions import SensitiveDomainPermission
from lemur.common.utils import parse_certificate, is_weekend
from lemur.domains import service as domain_service
def public_certificate(body):
@ -26,22 +24,6 @@ def public_certificate(body):
raise ValidationError('Public certificate presented is not valid.')
def private_key(key):
"""
User to validate that a given string is a RSA private key
:param key:
:return: :raise ValueError:
"""
try:
if isinstance(key, bytes):
serialization.load_pem_private_key(key, None, backend=default_backend())
else:
serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
except Exception:
raise ValidationError('Private key presented is not valid.')
def common_name(value):
"""If the common name could be a domain name, apply domain validation rules."""
# Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
@ -66,6 +48,9 @@ def sensitive_domain(domain):
raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
'Contact an administrator to issue the certificate.'.format(domain))
# Avoid circular import.
from lemur.domains import service as domain_service
if any(d.sensitive for d in domain_service.get_by_name(domain)):
raise ValidationError('Domain {0} has been marked as sensitive. '
'Contact an administrator to issue the certificate.'.format(domain))
@ -141,3 +126,15 @@ def dates(data):
raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
return data
def verify_private_key_match(key, cert, error_class=ValidationError):
"""
Checks that the supplied private key matches the certificate.
:param cert: Parsed certificate
:param key: Parsed private key
:param error_class: Exception class to raise on error
"""
if key.public_key().public_numbers() != cert.public_key().public_numbers():
raise error_class("Private key does not match certificate.")

8
lemur/plugins/lemur_acme/dyn.py
View File

@ -5,7 +5,7 @@ import dns.exception
import dns.name
import dns.query
import dns.resolver
from dyn.tm.errors import DynectCreateError
from dyn.tm.errors import DynectCreateError, DynectGetError
from dyn.tm.session import DynectSession
from dyn.tm.zones import Node, Zone, get_all_zones
from flask import current_app
@ -119,7 +119,11 @@ def delete_txt_record(change_id, account_number, domain, token):
zone = Zone(zone_name)
node = Node(zone_name, fqdn)
all_txt_records = node.get_all_records_by_type('TXT')
try:
all_txt_records = node.get_all_records_by_type('TXT')
except DynectGetError:
# No Text Records remain or host is not in the zone anymore because all records have been deleted.
return
for txt_record in all_txt_records:
if txt_record.txtdata == ("{}".format(token)):
current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))

18
lemur/plugins/lemur_aws/elb.py
View File

@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
"""
Get a listener ARN from an endpoint.
@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_elbs(**kwargs):
"""
Fetches one page elb objects for a given account and region.
@ -123,7 +123,7 @@ def get_elbs(**kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_elbs_v2(**kwargs):
"""
Fetches one page of elb objects for a given account and region.
@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_listeners_v2(**kwargs):
"""
Fetches one page of listener objects for a given elb arn.
@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
"""
Fetching all policies currently associated with an ELB.
@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_ssl_policies_v2(policy_names, **kwargs):
"""
Fetching all policies currently associated with an ELB.
@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_load_balancer_types(policies, **kwargs):
"""
Describe the policies with policy details.
@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def attach_certificate(name, port, certificate_id, **kwargs):
"""
Attaches a certificate to a listener, throws exception
@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def attach_certificate_v2(listener_arn, port, certificates, **kwargs):
"""
Attaches a certificate to a listener, throws exception

8
lemur/plugins/lemur_aws/iam.py
View File

@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
"""
Upload a certificate to AWS
@ -95,7 +95,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def delete_cert(cert_name, **kwargs):
"""
Delete a certificate from AWS
@ -112,7 +112,7 @@ def delete_cert(cert_name, **kwargs):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_certificate(name, **kwargs):
"""
Retrieves an SSL certificate.
@ -126,7 +126,7 @@ def get_certificate(name, **kwargs):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_certificates(**kwargs):
"""
Fetches one page of certificate objects for a given account.

16
lemur/plugins/lemur_aws/sts.py
View File

@ -9,14 +9,22 @@ from functools import wraps
import boto3
from botocore.config import Config
from flask import current_app
config = Config(
retries=dict(
max_attempts=20
)
)
def sts_client(service, service_type='client'):
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
sts = boto3.client('sts')
sts = boto3.client('sts', config=config)
arn = 'arn:aws:iam::{0}:role/{1}'.format(
kwargs.pop('account_number'),
current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
@ -31,7 +39,8 @@ def sts_client(service, service_type='client'):
region_name=kwargs.pop('region', 'us-east-1'),
aws_access_key_id=role['Credentials']['AccessKeyId'],
aws_secret_access_key=role['Credentials']['SecretAccessKey'],
aws_session_token=role['Credentials']['SessionToken']
aws_session_token=role['Credentials']['SessionToken'],
config=config
)
kwargs['client'] = client
elif service_type == 'resource':
@ -40,7 +49,8 @@ def sts_client(service, service_type='client'):
region_name=kwargs.pop('region', 'us-east-1'),
aws_access_key_id=role['Credentials']['AccessKeyId'],
aws_secret_access_key=role['Credentials']['SecretAccessKey'],
aws_session_token=role['Credentials']['SessionToken']
aws_session_token=role['Credentials']['SessionToken'],
config=config
)
kwargs['resource'] = resource
return f(*args, **kwargs)

5
lemur/sources/service.py
View File

@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service
from lemur.destinations import service as destination_service
from lemur.certificates.schemas import CertificateUploadInputSchema
from lemur.common.utils import parse_certificate
from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate
from lemur.common.defaults import serial
from lemur.plugins.base import plugins
@ -126,7 +126,8 @@ def sync_certificates(source, user):
if not exists:
cert = parse_certificate(certificate['body'])
exists = certificate_service.get_by_serial(serial(cert))
matching_serials = certificate_service.get_by_serial(serial(cert))
exists = find_matching_certificates_by_hash(cert, matching_serials)
if not certificate.get('owner'):
certificate['owner'] = user.email

9
lemur/tests/conftest.py
View File

@ -15,7 +15,7 @@ from lemur.tests.vectors import SAN_CERT_KEY, INTERMEDIATE_KEY
from .factories import ApiKeyFactory, AuthorityFactory, NotificationFactory, DestinationFactory, \
CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, \
RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory
RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory, CryptoAuthorityFactory
def pytest_runtest_setup(item):
@ -91,6 +91,13 @@ def authority(session):
return a
@pytest.fixture
def crypto_authority(session):
a = CryptoAuthorityFactory()
session.commit()
return a
@pytest.fixture
def async_authority(session):
a = AsyncAuthorityFactory()

5
lemur/tests/factories.py
View File

@ -168,6 +168,11 @@ class AsyncAuthorityFactory(AuthorityFactory):
authority_certificate = SubFactory(CertificateFactory)
class CryptoAuthorityFactory(AuthorityFactory):
"""Authority factory based on 'cryptography' plugin."""
plugin = {'slug': 'cryptography-issuer'}
class DestinationFactory(BaseFactory):
"""Destination factory."""
plugin_name = 'test-destination'

88
lemur/tests/test_certificates.py
View File

@ -18,7 +18,7 @@ from lemur.domains.models import Domain
from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \
INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY
INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
def test_get_or_increase_name(session, certificate):
@ -365,6 +365,85 @@ def test_certificate_sensitive_name(client, authority, session, logged_in_user):
assert errors['common_name'][0].startswith("Domain sensitive.example.com has been marked as sensitive")
def test_certificate_upload_schema_ok(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'name': 'Jane',
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
'privateKey': SAN_CERT_KEY,
'chain': INTERMEDIATE_CERT_STR,
'external_id': '1234',
}
data, errors = CertificateUploadInputSchema().load(data)
assert not errors
def test_certificate_upload_schema_minimal(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
}
data, errors = CertificateUploadInputSchema().load(data)
assert not errors
def test_certificate_upload_schema_long_chain(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
'chain': INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR
}
data, errors = CertificateUploadInputSchema().load(data)
assert not errors
def test_certificate_upload_schema_invalid_body(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': 'Hereby I certify that this is a valid body',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'body': ['Public certificate presented is not valid.']}
def test_certificate_upload_schema_invalid_pkey(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
'privateKey': 'Look at me Im a private key!!111',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'private_key': ['Private key presented is not valid.']}
def test_certificate_upload_schema_invalid_chain(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'body': SAN_CERT_STR,
'chain': 'CHAINSAW',
'owner': 'pwner@example.com',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'chain': ['Public certificate presented is not valid.']}
def test_certificate_upload_schema_wrong_pkey(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'body': SAN_CERT_STR,
'privateKey': ROOTCA_KEY,
'chain': INTERMEDIATE_CERT_STR,
'owner': 'pwner@example.com',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'_schema': ['Private key does not match certificate.']}
def test_create_basic_csr(client):
csr_config = dict(
common_name='example.com',
@ -462,8 +541,11 @@ def test_create_certificate(issuer_plugin, authority, user):
assert cert.name == 'ACustomName1'
def test_reissue_certificate(issuer_plugin, authority, certificate):
def test_reissue_certificate(issuer_plugin, crypto_authority, certificate, logged_in_user):
from lemur.certificates.service import reissue_certificate
# test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead.
certificate.authority = crypto_authority
new_cert = reissue_certificate(certificate)
assert new_cert
@ -487,7 +569,7 @@ def test_import(user):
assert str(cert.not_after) == '2047-12-31T22:00:00+00:00'
assert str(cert.not_before) == '2017-12-31T22:00:00+00:00'
assert cert.issuer == 'LemurTrustUnittestsClass1CA2018'
assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2'
assert cert.name.startswith('SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231')
cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, owner='joe@example.com', name='ACustomName2', creator=user['user'])
assert cert.name == 'ACustomName2'

22
lemur/tests/test_validators.py
View File

@ -1,16 +1,28 @@
import pytest
from datetime import datetime
from .vectors import SAN_CERT_KEY
import pytest
from marshmallow.exceptions import ValidationError
from lemur.common.utils import parse_private_key
from lemur.common.validators import verify_private_key_match
from lemur.tests.vectors import INTERMEDIATE_CERT, SAN_CERT, SAN_CERT_KEY
def test_private_key(session):
from lemur.common.validators import private_key
parse_private_key(SAN_CERT_KEY)
private_key(SAN_CERT_KEY)
with pytest.raises(ValueError):
parse_private_key('invalid_private_key')
def test_validate_private_key(session):
key = parse_private_key(SAN_CERT_KEY)
verify_private_key_match(key, SAN_CERT)
with pytest.raises(ValidationError):
private_key('invalid_private_key')
# Wrong key for certificate
verify_private_key_match(key, INTERMEDIATE_CERT)
def test_sub_alt_type(session):