Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Kevin Glisson
2015-06-22 13:47:27 -07:00
commit 4330ac9c05
228 changed files with 16656 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
lemur/certificates/__init__.py Normal file
View File

87
lemur/certificates/exceptions.py Normal file
View File

@ -0,0 +1,87 @@
"""
.. module: lemur.certificates.exceptions
:synopsis: Defines all monterey specific exceptions
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
from lemur.exceptions import LemurException
class UnknownAuthority(LemurException):
def __init__(self, authority):
self.code = 404
self.authority = authority
self.data = {"message": "The authority specified '{}' is not a valid authority".format(self.authority)}
current_app.logger.warning(self)
def __str__(self):
return repr(self.data['message'])
class InsufficientDomains(LemurException):
def __init__(self):
self.code = 400
self.data = {"message": "Need at least one domain specified in order create a certificate"}
current_app.logger.warning(self)
def __str__(self):
return repr(self.data['message'])
class InvalidCertificate(LemurException):
def __init__(self):
self.code = 400
self.data = {"message": "Need at least one domain specified in order create a certificate"}
current_app.logger.warning(self)
def __str__(self):
return repr(self.data['message'])
class UnableToCreateCSR(LemurException):
def __init__(self):
self.code = 500
self.data = {"message": "Unable to generate CSR"}
current_app.logger.error(self)
def __str__(self):
return repr(self.data['message'])
class UnableToCreatePrivateKey(LemurException):
def __init__(self):
self.code = 500
self.data = {"message": "Unable to generate Private Key"}
current_app.logger.error(self)
def __str__(self):
return repr(self.data['message'])
class MissingFiles(LemurException):
def __init__(self, path):
self.code = 500
self.path = path
self.data = {"path": self.path, "message": "Expecting missing files"}
current_app.logger.error(self)
def __str__(self):
return repr(self.data['message'])
class NoPersistanceFound(LemurException):
def __init__(self):
self.code = 500
self.data = {"code": 500, "message": "No peristence method found, Lemur cannot persist sensitive information"}
current_app.logger.error(self)
def __str__(self):
return repr(self.data['message'])

307
lemur/certificates/models.py Normal file
View File

@ -0,0 +1,307 @@
"""
.. module: lemur.certificates.models
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import os
import datetime
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from flask import current_app
from sqlalchemy.orm import relationship
from sqlalchemy import Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean
from sqlalchemy_utils import EncryptedType
from lemur.database import db
from lemur.domains.models import Domain
from lemur.users import service as user_service
from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE, NONSTANDARD_NAMING_TEMPLATE
from lemur.models import certificate_associations, certificate_account_associations
def cert_get_cn(cert):
"""
Attempts to get a sane common name from a given certificate.
:param cert:
:return: Common name or None
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_COMMON_NAME
)[0].value.strip()
except Exception as e:
current_app.logger.error("Unable to get CN! {0}".format(e))
def cert_get_domains(cert):
"""
Attempts to get an domains listed in a certificate.
If 'subjectAltName' extension is not available we simply
return the common name.
:param cert:
:return: List of domains
"""
domains = []
try:
ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
entries = ext.get_values_for(x509.DNSName)
for entry in entries:
domains.append(entry.split(":")[1].strip(", "))
except Exception as e:
current_app.logger.warning("Failed to get SubjectAltName: {0}".format(e))
domains.append(cert_get_cn(cert))
return domains
def cert_get_serial(cert):
"""
Fetch the serial number from the certificate.
:param cert:
:return: serial number
"""
return cert.serial
def cert_is_san(cert):
"""
Determines if a given certificate is a SAN certificate.
SAN certificates are simply certificates that cover multiple domains.
:param cert:
:return: Bool
"""
domains = cert_get_domains(cert)
if len(domains) > 1:
return True
return False
def cert_is_wildcard(cert):
"""
Determines if certificate is a wildcard certificate.
:param cert:
:return: Bool
"""
domains = cert_get_domains(cert)
if len(domains) == 1 and domains[0][0:1] == "*":
return True
return False
def cert_get_bitstrength(cert):
"""
Calculates a certificates public key bit length.
:param cert:
:return: Integer
"""
return cert.public_key().key_size * 8
def cert_get_issuer(cert):
"""
Gets a sane issuer from a given certificate.
:param cert:
:return: Issuer
"""
try:
return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[0].value
except Exception as e:
current_app.logger.error("Unable to get issuer! {0}".format(e))
def cert_is_internal(cert):
"""
Uses an internal resource in order to determine if
a given certificate was issued by an 'internal' certificate
authority.
:param cert:
:return: Bool
"""
if cert_get_issuer(cert) in current_app.config.get('INTERNAL_CA', []):
return True
return False
def cert_get_not_before(cert):
"""
Gets the naive datetime of the certificates 'not_before' field.
This field denotes the first date in time which the given certificate
is valid.
:param cert:
:return: Datetime
"""
return cert.not_valid_before
def cert_get_not_after(cert):
"""
Gets the naive datetime of the certificates 'not_after' field.
This field denotes the last date in time which the given certificate
is valid.
:param cert:
:return: Datetime
"""
return cert.not_valid_after
def get_name_from_arn(arn):
"""
Extract the certificate name from an arn.
:param arn: IAM SSL arn
:return: name of the certificate as uploaded to AWS
"""
return arn.split("/", 1)[1]
def get_account_number(arn):
"""
Extract the account number from an arn.
:param arn: IAM SSL arn
:return: account number associated with ARN
"""
return arn.split(":")[4]
class Certificate(db.Model):
__tablename__ = 'certificates'
id = Column(Integer, primary_key=True)
owner = Column(String(128))
body = Column(Text())
private_key = Column(EncryptedType(String, os.environ.get('LEMUR_ENCRYPTION_KEY')))
challenge = Column(EncryptedType(String, os.environ.get('LEMUR_ENCRYPTION_KEY')))
csr_config = Column(Text())
status = Column(String(128))
deleted = Column(Boolean, index=True)
name = Column(String(128))
chain = Column(Text())
bits = Column(Integer())
issuer = Column(String(128))
serial = Column(String(128))
cn = Column(String(128))
description = Column(String(1024))
active = Column(Boolean, default=True)
san = Column(String(1024))
not_before = Column(DateTime)
not_after = Column(DateTime)
date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
user_id = Column(Integer, ForeignKey('users.id'))
authority_id = Column(Integer, ForeignKey('authorities.id'))
accounts = relationship("Account", secondary=certificate_account_associations, backref='certificate')
domains = relationship("Domain", secondary=certificate_associations, backref="certificate")
elb_listeners = relationship("Listener", lazy='dynamic', backref='certificate')
def __init__(self, body, private_key=None, challenge=None, chain=None, csr_config=None):
self.body = body
# We encrypt the private_key on creation
self.private_key = private_key
self.chain = chain
self.csr_config = csr_config
self.challenge = challenge
cert = x509.load_pem_x509_certificate(str(self.body), default_backend())
self.bits = cert_get_bitstrength(cert)
self.issuer = cert_get_issuer(cert)
self.serial = cert_get_serial(cert)
self.cn = cert_get_cn(cert)
self.san = cert_is_san(cert)
self.not_before = cert_get_not_before(cert)
self.not_after = cert_get_not_after(cert)
self.name = self.create_name
for domain in cert_get_domains(cert):
self.domains.append(Domain(name=domain))
@property
def create_name(self):
"""
Create a name for our certificate. A naming standard
is based on a series of templates. The name includes
useful information such as Common Name, Validation dates,
and Issuer.
:rtype : str
:return:
"""
# aws doesn't allow special chars
if self.cn:
subject = self.cn.replace('*', "WILDCARD")
if self.san:
t = SAN_NAMING_TEMPLATE
else:
t = DEFAULT_NAMING_TEMPLATE
temp = t.format(
subject=subject,
issuer=self.issuer,
not_before=self.not_before.strftime('%Y%m%d'),
not_after=self.not_after.strftime('%Y%m%d')
)
else:
t = NONSTANDARD_NAMING_TEMPLATE
temp = t.format(
issuer=self.issuer,
not_before=self.not_before.strftime('%Y%m%d'),
not_after=self.not_after.strftime('%Y%m%d')
)
return temp
@property
def is_expired(self):
if self.not_after < datetime.datetime.now():
return True
@property
def is_unused(self):
if self.elb_listeners.count() == 0:
return True
@property
def is_revoked(self):
# we might not yet know the condition of the cert
if self.status:
if 'revoked' in self.status:
return True
def get_arn(self, account_number):
"""
Generate a valid AWS IAM arn
:rtype : str
:param account_number:
:return:
"""
return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)
def as_dict(self):
return {c.name: getattr(self, c.name) for c in self.__table__.columns}
def serialize(self):
blob = self.as_dict()
# TODO this should be done with relationships
user = user_service.get(self.user_id)
if user:
blob['creator'] = user.username
return blob

446
lemur/certificates/service.py Normal file
View File

@ -0,0 +1,446 @@
"""
.. module: service
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import os
import arrow
import string
import random
import hashlib
import datetime
import subprocess
from sqlalchemy import func, or_
from flask import g, current_app
from lemur import database
from lemur.common.services.aws import iam
from lemur.common.services.issuers.manager import get_plugin_by_name
from lemur.certificates.models import Certificate
from lemur.certificates.exceptions import UnableToCreateCSR, \
UnableToCreatePrivateKey, MissingFiles
from lemur.accounts.models import Account
from lemur.accounts import service as account_service
from lemur.authorities.models import Authority
from lemur.roles.models import Role
def get(cert_id):
"""
Retrieves certificate by it's ID.
:param cert_id:
:return:
"""
return database.get(Certificate, cert_id)
def get_by_name(name):
"""
Retrieves certificate by it's Name.
:param name:
:return:
"""
return database.get(Certificate, name, field='name')
def delete(cert_id):
"""
Delete's a certificate.
:param cert_id:
"""
database.delete(get(cert_id))
def disassociate_aws_account(certs, account):
"""
Removes the account association from a certificate. We treat AWS as a completely
external service. Certificates are added and removed from this service but a record
of that certificate is always kept and tracked by Lemur. This allows us to migrate
certificates to different accounts with ease.
:param certs:
:param account:
"""
account_certs = Certificate.query.filter(Certificate.accounts.any(Account.id == 1)).\
filter(~Certificate.body.in_(certs)).all()
for a_cert in account_certs:
try:
a_cert.accounts.remove(account)
except Exception as e:
current_app.logger.debug("Skipping {0} account {1} is already disassociated".format(a_cert.name, account.label))
continue
database.update(a_cert)
def get_all_certs():
"""
Retrieves all certificates within Lemur.
:return:
"""
return Certificate.query.all()
def find_duplicates(cert_body):
"""
Finds certificates that already exist within Lemur. We do this by looking for
certificate bodies that are the same. This is the most reliable way to determine
if a certificate is already being tracked by Lemur.
:param cert_body:
:return:
"""
return Certificate.query.filter_by(body=cert_body).all()
def update(cert_id, owner, active):
"""
Updates a certificate.
:param cert_id:
:param owner:
:param active:
:return:
"""
cert = get(cert_id)
cert.owner = owner
cert.active = active
return database.update(cert)
def mint(issuer_options):
"""
Minting is slightly different for each authority.
Support for multiple authorities is handled by individual plugins.
:param issuer_options:
"""
authority = issuer_options['authority']
issuer = get_plugin_by_name(authority.plugin_name)
# NOTE if we wanted to support more issuers it might make sense to
# push CSR creation down to the plugin
path = create_csr(issuer.get_csr_config(issuer_options))
challenge, csr, csr_config, private_key = load_ssl_pack(path)
issuer_options['challenge'] = challenge
issuer_options['creator'] = g.user.email
cert_body, cert_chain = issuer.create_certificate(csr, issuer_options)
cert = save_cert(cert_body, private_key, cert_chain, challenge, csr_config, issuer_options.get('accounts'))
cert.user = g.user
cert.authority = authority
database.update(cert)
# securely delete pack after saving it to RDS and IAM (if applicable)
delete_ssl_pack(path)
return cert, private_key, cert_chain,
def import_certificate(**kwargs):
"""
Uploads already minted certificates and pulls the required information into Lemur.
This is to be used for certificates that are reated outside of Lemur but
should still be tracked.
Internally this is used to bootstrap Lemur with external
certificates, and used when certificates are 'discovered' through various discovery
techniques. was still in aws.
:param kwargs:
"""
cert = Certificate(kwargs['public_certificate'])
cert.owner = kwargs.get('owner', )
cert.creator = kwargs.get('creator', 'Lemur')
# NOTE existing certs may not follow our naming standard we will
# overwrite the generated name with the actual cert name
if kwargs.get('name'):
cert.name = kwargs.get('name')
if kwargs.get('user'):
cert.user = kwargs.get('user')
if kwargs.get('account'):
cert.accounts.append(kwargs.get('account'))
cert = database.create(cert)
return cert
def save_cert(cert_body, private_key, cert_chain, challenge, csr_config, accounts):
"""
Determines if the certificate needs to be uploaded to AWS or other services.
:param cert_body:
:param private_key:
:param cert_chain:
:param challenge:
:param csr_config:
:param account_ids:
"""
cert = Certificate(cert_body, private_key, challenge, cert_chain, csr_config)
# if we have an AWS accounts lets upload them
if accounts:
for account in accounts:
account = account_service.get(account['id'])
iam.upload_cert(account.account_number, cert, private_key, cert_chain)
cert.accounts.append(account)
return cert
def upload(**kwargs):
"""
Allows for pre-made certificates to be imported into Lemur.
"""
# save this cert the same way we save all of our certs, including uploading
# to aws if necessary
cert = save_cert(
kwargs.get('public_cert'),
kwargs.get('private_key'),
kwargs.get('intermediate_cert'),
None,
None,
kwargs.get('accounts')
)
cert.owner = kwargs['owner']
cert = database.create(cert)
g.user.certificates.append(cert)
return cert
def create(**kwargs):
"""
Creates a new certificate.
"""
cert, private_key, cert_chain = mint(kwargs)
cert.owner = kwargs['owner']
database.create(cert)
g.user.certificates.append(cert)
database.update(g.user)
return cert
def render(args):
"""
Helper function that allows use to render our REST Api.
:param args:
:return:
"""
query = database.session_query(Certificate)
time_range = args.pop('time_range')
account_id = args.pop('account_id')
show = args.pop('show')
owner = args.pop('owner')
creator = args.pop('creator') # TODO we should enabling filtering by owner
filt = args.pop('filter')
if filt:
terms = filt.split(';')
if 'issuer' in terms:
# we can't rely on issuer being correct in the cert directly so we combine queries
sub_query = database.session_query(Authority.id)\
.filter(Authority.name.ilike('%{0}%'.format(terms[1])))\
.subquery()
query = query.filter(
or_(
Certificate.issuer.ilike('%{0}%'.format(terms[1])),
Certificate.authority_id.in_(sub_query)
)
)
return database.sort_and_page(query, Certificate, args)
if 'account' in terms:
query = query.filter(Certificate.accounts.any(Account.id == terms[1]))
elif 'active' in filt: # this is really weird but strcmp seems to not work here??
query = query.filter(Certificate.active == terms[1])
else:
query = database.filter(query, Certificate, terms)
if show:
sub_query = database.session_query(Role.name).filter(Role.user_id == g.user.id).subquery()
query = query.filter(
or_(
Certificate.user_id == g.user.id,
Certificate.owner.in_(sub_query)
)
)
if account_id:
query = query.filter(Certificate.accounts.any(Account.id == account_id))
if time_range:
to = arrow.now().replace(weeks=+time_range).format('YYYY-MM-DD')
now = arrow.now().format('YYYY-MM-DD')
query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
return database.sort_and_page(query, Certificate, args)
def create_csr(csr_config):
"""
Given a list of domains create the appropriate csr
for those domains
:param csr_config:
"""
# we create a no colliding file name
path = create_path(hashlib.md5(csr_config).hexdigest())
challenge = create_challenge()
challenge_path = os.path.join(path, 'challenge.txt')
with open(challenge_path, 'w') as c:
c.write(challenge)
csr_path = os.path.join(path, 'csr_config.txt')
with open(csr_path, 'w') as f:
f.write(csr_config)
#TODO use cloudCA to seed a -rand file for each call
#TODO replace openssl shell calls with cryptograph
with open('/dev/null', 'w') as devnull:
code = subprocess.call(['openssl', 'genrsa',
'-out', os.path.join(path, 'private.key'), '2048'],
stdout=devnull, stderr=devnull)
if code != 0:
raise UnableToCreatePrivateKey(code)
with open('/dev/null', 'w') as devnull:
code = subprocess.call(['openssl', 'req', '-new', '-sha256', '-nodes',
'-config', csr_path, "-key", os.path.join(path, 'private.key'),
"-out", os.path.join(path, 'request.csr')], stdout=devnull, stderr=devnull)
if code != 0:
raise UnableToCreateCSR(code)
return path
def create_path(domain_hash):
"""
:param domain_hash:
:return:
"""
path = os.path.join('/tmp', domain_hash)
try:
os.mkdir(path)
except OSError as e:
now = datetime.datetime.now()
path = os.path.join('/tmp', "{}.{}".format(domain_hash, now.strftime('%s')))
os.mkdir(path)
current_app.logger.warning(e)
current_app.logger.debug("Writing ssl files to: {}".format(path))
return path
def load_ssl_pack(path):
"""
Loads the information created by openssl to be used by other functions.
:param path:
"""
if len(os.listdir(path)) != 4:
raise MissingFiles(path)
with open(os.path.join(path, 'challenge.txt')) as c:
challenge = c.read()
with open(os.path.join(path, 'request.csr')) as r:
csr = r.read()
with open(os.path.join(path, 'csr_config.txt')) as config:
csr_config = config.read()
with open(os.path.join(path, 'private.key')) as key:
private_key = key.read()
return (challenge, csr, csr_config, private_key,)
def delete_ssl_pack(path):
"""
Removes the temporary files associated with CSR creation.
:param path:
"""
subprocess.check_call(['srm', '-r', path])
def create_challenge():
"""
Create a random and strongish csr challenge.
"""
challenge = ''.join(random.choice(string.ascii_uppercase) for x in range(6))
challenge += ''.join(random.choice("~!@#$%^&*()_+") for x in range(6))
challenge += ''.join(random.choice(string.ascii_lowercase) for x in range(6))
challenge += ''.join(random.choice(string.digits) for x in range(6))
return challenge
def stats(**kwargs):
"""
Helper that defines some useful statistics about certifications.
:param kwargs:
:return:
"""
query = database.session_query(Certificate)
if kwargs.get('active') == 'true':
query = query.filter(Certificate.elb_listeners.any())
if kwargs.get('account_id'):
query = query.filter(Certificate.accounts.any(Account.id == kwargs.get('account_id')))
if kwargs.get('metric') == 'not_after':
start = arrow.utcnow()
end = start.replace(weeks=+32)
items = database.db.session.query(Certificate.issuer, func.count(Certificate.id))\
.group_by(Certificate.issuer)\
.filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
.filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()
else:
attr = getattr(Certificate, kwargs.get('metric'))
query = database.db.session.query(attr, func.count(attr))
# TODO this could be cleaned up
if kwargs.get('active') == 'true':
query = query.filter(Certificate.elb_listeners.any())
items = query.group_by(attr).all()
keys = []
values = []
for key, count in items:
keys.append(key)
values.append(count)
return {'labels': keys, 'values': values}

169
lemur/certificates/sync.py Normal file
View File

@ -0,0 +1,169 @@
"""
.. module: sync
:platform: Unix
:synopsis: This module contains various certificate syncing operations.
Because of the nature of the SSL environment there are multiple ways
a certificate could be created without Lemur's knowledge. Lemur attempts
to 'sync' with as many different datasources as possible to try and track
any certificate that may be in use.
This include querying AWS for certificates attached to ELBs, querying our own
internal CA for certificates issued. As well as some rudimentary source code
scraping that attempts to find certificates checked into source code.
These operations are typically run on a periodic basis from either the command
line or a cron job.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import requests
from bs4 import BeautifulSoup
from flask import current_app
from lemur.users import service as user_service
from lemur.accounts import service as account_service
from lemur.certificates import service as cert_service
from lemur.certificates.models import Certificate, get_name_from_arn
from lemur.common.services.aws.iam import get_all_server_certs
from lemur.common.services.aws.iam import get_cert_from_arn
from lemur.common.services.issuers.manager import get_plugin_by_name
def aws():
"""
Attempts to retrieve all certificates located in known AWS accounts
:raise e:
"""
new = 0
updated = 0
# all certificates 'discovered' by lemur are tracked by the lemur
# user
user = user_service.get_by_email('lemur@nobody')
# we don't need to check regions as IAM is a global service
for account in account_service.get_all():
certificate_bodies = []
try:
cert_arns = get_all_server_certs(account.account_number)
except Exception as e:
current_app.logger.error("Failed to to get Certificates from '{}/{}' reason {}".format(
account.label, account.account_number, e.message)
)
raise e
current_app.logger.info("found {} certs from '{}/{}' ... ".format(
len(cert_arns), account.account_number, account.label)
)
for cert in cert_arns:
cert_body = get_cert_from_arn(cert.arn)[0]
certificate_bodies.append(cert_body)
existing = cert_service.find_duplicates(cert_body)
if not existing:
cert_service.import_certificate(
**{'owner': 'secops@netflix.com',
'creator': 'Lemur',
'name': get_name_from_arn(cert.arn),
'account': account,
'user': user,
'public_certificate': cert_body
}
)
new += 1
elif len(existing) == 1: # we check to make sure we know about the current account for this certificate
for e_account in existing[0].accounts:
if e_account.account_number == account.account_number:
break
else: # we have a new account
existing[0].accounts.append(account)
updated += 1
else:
current_app.logger.error(
"Multiple certificates with the same body found, unable to correctly determine which entry to update"
)
# make sure we remove any certs that have been removed from AWS
cert_service.disassociate_aws_account(certificate_bodies, account)
current_app.logger.info("found {} new certificates in aws {}".format(new, account.label))
def cloudca():
"""
Attempts to retrieve all certificates that are stored in CloudCA
"""
user = user_service.get_by_email('lemur@nobody')
# sync all new certificates/authorities not created through lemur
issuer = get_plugin_by_name('cloudca')
authorities = issuer.get_authorities()
total = 0
new = 1
for authority in authorities:
certs = issuer.get_cert(ca_name=authority)
for cert in certs:
total += 1
cert['user'] = user
existing = cert_service.find_duplicates(cert['public_certificate'])
if not existing:
new += 1
try:
cert_service.import_certificate(**cert)
except NameError as e:
current_app.logger.error("Cannot import certificate {0}".format(cert))
current_app.logger.debug("Found {0} total certificates in cloudca".format(total))
current_app.logger.debug("Found {0} new certificates in cloudca".format(new))
def source():
"""
Attempts to track certificates that are stored in Source Code
"""
new = 0
keywords = ['"--- Begin Certificate ---"']
endpoint = current_app.config.get('LEMUR_SOURCE_SEARCH')
maxresults = 25000
current_app.logger.info("Searching {0} for new certificates".format(endpoint))
for keyword in keywords:
current_app.logger.info("Looking for keyword: {0}".format(keyword))
url = "{}/source/s?n={}&start=1&sort=relevancy&q={}&project=github%2Cperforce%2Cstash".format(endpoint, maxresults, keyword)
current_app.logger.debug("Request url: {0}".format(url))
r = requests.get(url, timeout=20)
if r.status_code != 200:
current_app.logger.error("Unable to retrieve: {0} Status Code: {1}".format(url, r.status_code))
continue
soup = BeautifulSoup(r.text, "lxml")
results = soup.find_all(title='Download')
for result in results:
parts = result['href'].split('/')
path = "/".join(parts[:-1])
filename = parts[-1:][0]
r = requests.get("{0}{1}/{2}".format(endpoint, path, filename))
if r.status_code != 200:
current_app.logger.error("Unable to retrieve: {0} Status Code: {1}".format(url, r.status_code))
continue
try:
# validate we have a real certificate
cert = Certificate(r.content)
# do a lookup to see if we know about this certificate
existing = cert_service.find_duplicates(r.content)
if not existing:
current_app.logger.debug(cert.name)
cert_service.import_certificate()
new += 1
except Exception as e:
current_app.logger.debug("Could not parse the following 'certificate': {0} Reason: {1}".format(r.content, e))

135
lemur/certificates/verify.py Normal file
View File

@ -0,0 +1,135 @@
"""
.. module: lemur.certificates.verify
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import os
import re
import hashlib
import requests
import subprocess
from OpenSSL import crypto
from flask import current_app
def ocsp_verify(cert_path, issuer_chain_path):
"""
Attempts to verify a certificate via OCSP. OCSP is a more modern version
of CRL in that it will query the OCSP URI in order to determine if the
certificate as been revoked
:param cert_path:
:param issuer_chain_path:
:return bool: True if certificate is valid, False otherwise
"""
command = ['openssl', 'x509', '-noout', '-ocsp_uri', '-in', cert_path]
p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
url, err = p1.communicate()
p2 = subprocess.Popen(['openssl', 'ocsp', '-issuer', issuer_chain_path,
'-cert', cert_path, "-url", url.strip()], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
message, err = p2.communicate()
if 'error' in message or 'Error' in message:
raise Exception("Got error when parsing OCSP url")
elif 'revoked' in message:
return
elif 'good' not in message:
raise Exception("Did not receive a valid response")
return True
def crl_verify(cert_path):
"""
Attempts to verify a certificate using CRL.
:param cert_path:
:return: True if certificate is valid, False otherwise
:raise Exception: If certificate does not have CRL
"""
s = "(http(s)?\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}/\S*?$)"
regex = re.compile(s, re.MULTILINE)
x509 = crypto.load_certificate(crypto.FILETYPE_PEM, open(cert_path, 'rt').read())
for x in range(x509.get_extension_count()):
ext = x509.get_extension(x)
if ext.get_short_name() == 'crlDistributionPoints':
r = regex.search(ext.get_data())
points = r.groups()
break
else:
raise Exception("Certificate does not have a CRL distribution point")
for point in points:
if point:
response = requests.get(point)
crl = crypto.load_crl(crypto.FILETYPE_ASN1, response.content)
revoked = crl.get_revoked()
for r in revoked:
if x509.get_serial_number() == r.get_serial():
return
return True
def verify(cert_path, issuer_chain_path):
"""
Verify a certificate using OCSP and CRL
:param cert_path:
:param issuer_chain_path:
:return: True if valid, False otherwise
"""
# OCSP is our main source of truth, in a lot of cases CRLs
# have been deprecated and are no longer updated
try:
return ocsp_verify(cert_path, issuer_chain_path)
except Exception as e:
current_app.logger.debug("Could not use OCSP: {0}".format(e))
try:
return crl_verify(cert_path)
except Exception as e:
current_app.logger.debug("Could not use CRL: {0}".format(e))
raise Exception("Failed to verify")
raise Exception("Failed to verify")
def make_tmp_file(string):
"""
Creates a temporary file for a given string
:param string:
:return: Full file path to created file
"""
m = hashlib.md5()
m.update(string)
hexdigest = m.hexdigest()
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), hexdigest)
with open(path, 'w') as f:
f.write(string)
return path
def verify_string(cert_string, issuer_string):
"""
Verify a certificate given only it's string value
:param cert_string:
:param issuer_string:
:return: True if valid, False otherwise
"""
cert_path = make_tmp_file(cert_string)
issuer_path = make_tmp_file(issuer_string)
status = verify(cert_path, issuer_path)
remove_tmp_file(cert_path)
remove_tmp_file(issuer_path)
return status
def remove_tmp_file(file_path):
os.remove(file_path)

575
lemur/certificates/views.py Normal file
View File

@ -0,0 +1,575 @@
"""
.. module: lemur.certificates.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import Blueprint, make_response, jsonify
from flask.ext.restful import reqparse, Api, fields
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from lemur.certificates import service
from lemur.authorities.models import Authority
from lemur.auth.service import AuthenticatedResource
from lemur.auth.permissions import ViewKeyPermission, AuthorityPermission, UpdateCertificatePermission
from lemur.roles import service as role_service
from lemur.common.utils import marshal_items, paginated_parser
mod = Blueprint('certificates', __name__)
api = Api(mod)
FIELDS = {
'name': fields.String,
'id': fields.Integer,
'bits': fields.Integer,
'deleted': fields.String,
'issuer': fields.String,
'serial': fields.String,
'owner': fields.String,
'chain': fields.String,
'san': fields.String,
'active': fields.Boolean,
'description': fields.String,
'notBefore': fields.DateTime(dt_format='iso8601', attribute='not_before'),
'notAfter': fields.DateTime(dt_format='iso8601', attribute='not_after'),
'cn': fields.String,
'status': fields.String,
'body': fields.String
}
def valid_authority(authority_options):
"""
Defends against invalid authorities
:param authority_name:
:return: :raise ValueError:
"""
name = authority_options['name']
authority = Authority.query.filter(Authority.name == name).one()
if not authority:
raise ValueError("Unable to find authority specified")
if not authority.active:
raise ValueError("Selected authority [{0}] is not currently active".format(name))
return authority
def pem_str(value, name):
"""
Used to validate that the given string is a PEM formatted string
:param value:
:param name:
:return: :raise ValueError:
"""
try:
x509.load_pem_x509_certificate(str(value), default_backend())
except Exception as e:
raise ValueError("The parameter '{0}' needs to be a valid PEM string".format(name))
return value
def private_key_str(value, name):
"""
User to validate that a given string is a RSA private key
:param value:
:param name:
:return: :raise ValueError:
"""
try:
serialization.load_pem_private_key(str(value), backend=default_backend())
except Exception as e:
raise ValueError("The parameter '{0}' needs to be a valid RSA private key".format(name))
return value
class CertificatesList(AuthenticatedResource):
""" Defines the 'certificates' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(CertificatesList, self).__init__()
@marshal_items(FIELDS)
def get(self):
"""
.. http:get:: /certificates
The current list of certificates
**Example request**:
.. sourcecode:: http
GET /certificates HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"items": [
{
"id": 1,
"name": "cert1",
"description": "this is cert1",
"bits": 2048,
"deleted": false,
"issuer": "ExampeInc.",
"serial": "123450",
"chain": "-----Begin ...",
"body": "-----Begin ...",
"san": true,
"owner": 'bob@example.com",
"active": true,
"notBefore": "2015-06-05T17:09:39",
"notAfter": "2015-06-10T17:09:39",
"cn": "example.com",
"status": "unknown"
}
]
"total": 1
}
:query sortBy: field to sort on
:query sortDir: acs or desc
:query page: int. default is 1
:query filter: key value pair. format is k=v;
:query limit: limit number. default is 10
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
parser = paginated_parser.copy()
parser.add_argument('timeRange', type=int, dest='time_range', location='args')
parser.add_argument('owner', type=bool, location='args')
parser.add_argument('id', type=str, location='args')
parser.add_argument('active', type=bool, location='args')
parser.add_argument('accountId', type=int, dest="account_id", location='args')
parser.add_argument('creator', type=str, location='args')
parser.add_argument('show', type=str, location='args')
args = parser.parse_args()
return service.render(args)
@marshal_items(FIELDS)
def post(self):
"""
.. http:post:: /certificates
Creates a new certificate
**Example request**:
.. sourcecode:: http
POST /certificates HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
{
"country": "US",
"state": "CA",
"location": "A Place",
"organization": "ExampleInc.",
"organizationalUnit": "Operations",
"owner": "bob@example.com",
"description": "test",
"selectedAuthority": "timetest2",
"authority": {
"body": "-----BEGIN...",
"name": "timetest2",
"chain": "",
"notBefore": "2015-06-05T15:20:59",
"active": true,
"id": 50,
"notAfter": "2015-06-17T15:21:08",
"description": "dsfdsf"
},
"extensions": {
"basicConstraints": {},
"keyUsage": {
"isCritical": true,
"useKeyEncipherment": true,
"useDigitalSignature": true
},
"extendedKeyUsage": {
"isCritical": true,
"useServerAuthentication": true
},
"subjectKeyIdentifier": {
"includeSKI": true
},
"subAltNames": {
"names": []
}
},
"commonName": "test",
"validityStart": "2015-06-05T07:00:00.000Z",
"validityEnd": "2015-06-16T07:00:00.000Z"
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"id": 1,
"name": "cert1",
"description": "this is cert1",
"bits": 2048,
"deleted": false,
"issuer": "ExampeInc.",
"serial": "123450",
"chain": "-----Begin ...",
"body": "-----Begin ...",
"san": true,
"owner": "jimbob@example.com",
"active": false,
"notBefore": "2015-06-05T17:09:39",
"notAfter": "2015-06-10T17:09:39",
"cn": "example.com",
"status": "unknown"
}
:arg extensions: extensions to be used in the certificate
:arg description: description for new certificate
:arg owner: owner email
:arg validityStart: when the certificate should start being valid
:arg validityEnd: when the certificate should expire
:arg authority: authority that should issue the certificate
:arg country: country for the CSR
:arg state: state for the CSR
:arg location: location for the CSR
:arg organization: organization for CSR
:arg commonName: certiifcate common name
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
self.reqparse.add_argument('extensions', type=dict, location='json')
self.reqparse.add_argument('accounts', type=list, location='json')
self.reqparse.add_argument('elbs', type=list, location='json')
self.reqparse.add_argument('owner', type=str, location='json')
self.reqparse.add_argument('validityStart', type=str, location='json') # parse date
self.reqparse.add_argument('validityEnd', type=str, location='json') # parse date
self.reqparse.add_argument('authority', type=valid_authority, location='json')
self.reqparse.add_argument('description', type=str, location='json')
self.reqparse.add_argument('country', type=str, location='json')
self.reqparse.add_argument('state', type=str, location='json')
self.reqparse.add_argument('location', type=str, location='json')
self.reqparse.add_argument('organization', type=str, location='json')
self.reqparse.add_argument('organizationalUnit', type=str, location='json')
self.reqparse.add_argument('owner', type=str, location='json')
self.reqparse.add_argument('commonName', type=str, location='json')
args = self.reqparse.parse_args()
authority = args['authority']
role = role_service.get_by_name(authority.owner)
# all the authority role members should be allowed
roles = [x.name for x in authority.roles]
# allow "owner" roles by team DL
roles.append(role)
permission = AuthorityPermission(authority.id, roles)
if permission.can():
return service.create(**args)
return dict(message="You are not authorized to use {0}".format(args['authority'].name)), 403
class CertificatesUpload(AuthenticatedResource):
""" Defines the 'certificates' upload endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(CertificatesUpload, self).__init__()
@marshal_items(FIELDS)
def post(self):
"""
.. http:post:: /certificates/upload
Upload a certificate
**Example request**:
.. sourcecode:: http
POST /certificates/upload HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
{
"owner": "joe@exmaple.com",
"publicCert": "---Begin Public...",
"intermediateCert": "---Begin Public...",
"privateKey": "---Begin Private..."
"accounts": []
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"id": 1,
"name": "cert1",
"description": "this is cert1",
"bits": 2048,
"deleted": false,
"issuer": "ExampeInc.",
"serial": "123450",
"chain": "-----Begin ...",
"body": "-----Begin ...",
"san": true,
"owner": "joe@example.com",
"active": true,
"notBefore": "2015-06-05T17:09:39",
"notAfter": "2015-06-10T17:09:39",
"cn": "example.com",
"status": "unknown"
}
:arg owner: owner email for certificate
:arg publicCert: valid PEM public key for certificate
:arg intermediateCert valid PEM intermediate key for certificate
:arg privateKey: valid PEM private key for certificate
:arg accounts: list of aws accounts to upload the certificate to
:reqheader Authorization: OAuth token to authenticate
:statuscode 403: unauthenticated
:statuscode 200: no error
"""
self.reqparse.add_argument('owner', type=str, required=True, location='json')
self.reqparse.add_argument('publicCert', type=pem_str, required=True, dest='public_cert', location='json')
self.reqparse.add_argument('accounts', type=list, dest='accounts', location='json')
self.reqparse.add_argument('intermediateCert', type=pem_str, dest='intermediate_cert', location='json')
self.reqparse.add_argument('privateKey', type=private_key_str, dest='private_key', location='json')
args = self.reqparse.parse_args()
if args.get('accounts'):
if args.get('private_key'):
return service.upload(**args)
else:
raise Exception("Private key must be provided in order to upload certificate to AWS")
return service.upload(**args)
class CertificatesStats(AuthenticatedResource):
""" Defines the 'certificates' stats endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(CertificatesStats, self).__init__()
def get(self):
self.reqparse.add_argument('metric', type=str, location='args')
self.reqparse.add_argument('range', default=32, type=int, location='args')
self.reqparse.add_argument('accountId', dest='account_id', location='args')
self.reqparse.add_argument('active', type=str, default='true', location='args')
args = self.reqparse.parse_args()
items = service.stats(**args)
return dict(items=items, total=len(items))
class CertificatePrivateKey(AuthenticatedResource):
def __init__(self):
super(CertificatePrivateKey, self).__init__()
def get(self, certificate_id):
"""
.. http:get:: /certificates/1/key
Retrieves the private key for a given certificate
**Example request**:
.. sourcecode:: http
GET /certificates/1/key HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"key": "----Begin ...",
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
cert = service.get(certificate_id)
role = role_service.get_by_name(cert.owner)
permission = ViewKeyPermission(certificate_id, hasattr(role, 'id'))
if permission.can():
response = make_response(jsonify(key=cert.private_key), 200)
response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
response.headers['pragma'] = 'no-cache'
return response
return dict(message='You are not authorized to view this key'), 403
class Certificates(AuthenticatedResource):
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(Certificates, self).__init__()
@marshal_items(FIELDS)
def get(self, certificate_id):
"""
.. http:get:: /certificates/1
One certificate
**Example request**:
.. sourcecode:: http
GET /certificates/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"id": 1,
"name": "cert1",
"description": "this is cert1",
"bits": 2048,
"deleted": false,
"issuer": "ExampeInc.",
"serial": "123450",
"chain": "-----Begin ...",
"body": "-----Begin ...",
"san": true,
"owner": "bob@example.com",
"active": true,
"notBefore": "2015-06-05T17:09:39",
"notAfter": "2015-06-10T17:09:39",
"cn": "example.com",
"status": "unknown"
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
return service.get(certificate_id)
@marshal_items(FIELDS)
def put(self, certificate_id):
"""
.. http:put:: /certificates/1
Update a certificate
**Example request**:
.. sourcecode:: http
PUT /certificates/1 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
{
"owner": "jimbob@example.com",
"active": false
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"id": 1,
"name": "cert1",
"description": "this is cert1",
"bits": 2048,
"deleted": false,
"issuer": "ExampeInc.",
"serial": "123450",
"chain": "-----Begin ...",
"body": "-----Begin ...",
"san": true,
"owner": "jimbob@example.com",
"active": false,
"notBefore": "2015-06-05T17:09:39",
"notAfter": "2015-06-10T17:09:39",
"cn": "example.com",
"status": "unknown"
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
self.reqparse.add_argument('active', type=bool, location='json')
self.reqparse.add_argument('owner', type=str, location='json')
args = self.reqparse.parse_args()
cert = service.get(certificate_id)
role = role_service.get_by_name(cert.owner)
permission = UpdateCertificatePermission(certificate_id, hasattr(role, 'id'))
if permission.can():
return service.update(certificate_id, args['owner'], args['active'])
return dict(message='You are not authorized to update this certificate'), 403
api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
api.add_resource(Certificates, '/certificates/<int:certificate_id>', endpoint='certificate')
api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificateStats')
api.add_resource(CertificatesUpload, '/certificates/upload', endpoint='certificateUpload')
api.add_resource(CertificatePrivateKey, '/certificates/<int:certificate_id>/key', endpoint='privateKeyCertificates')