Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored 'accounts' to be more general with 'destinations'

Browse Source
This commit is contained in:
kevgliss
2015-07-10 17:06:57 -07:00
parent b26de2b000
commit 0c7204cdb9
29 changed files with 421 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lemur/certificates/models.py
View File

@ -22,7 +22,7 @@ from lemur.database import db
from lemur.domains.models import Domain
from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE, NONSTANDARD_NAMING_TEMPLATE
from lemur.models import certificate_associations, certificate_account_associations
from lemur.models import certificate_associations, certificate_destination_associations
def create_name(issuer, not_before, not_after, subject, san):
@ -215,7 +215,7 @@ class Certificate(db.Model):
date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
user_id = Column(Integer, ForeignKey('users.id'))
authority_id = Column(Integer, ForeignKey('authorities.id'))
accounts = relationship("Account", secondary=certificate_account_associations, backref='certificate')
accounts = relationship("Destination", secondary=certificate_destination_associations, backref='certificate')
domains = relationship("Domain", secondary=certificate_associations, backref="certificate")
elb_listeners = relationship("Listener", lazy='dynamic', backref='certificate')

73
lemur/certificates/service.py
View File

@ -13,12 +13,10 @@ from sqlalchemy import func, or_
from flask import g, current_app
from lemur import database
from lemur.common.services.aws import iam
from lemur.plugins.base import plugins
from lemur.certificates.models import Certificate
from lemur.accounts.models import Account
from lemur.accounts import service as account_service
from lemur.destinations.models import Destination
from lemur.authorities.models import Authority
from lemur.roles.models import Role
@ -59,28 +57,6 @@ def delete(cert_id):
database.delete(get(cert_id))
def disassociate_aws_account(certs, account):
"""
Removes the account association from a certificate. We treat AWS as a completely
external service. Certificates are added and removed from this service but a record
of that certificate is always kept and tracked by Lemur. This allows us to migrate
certificates to different accounts with ease.
:param certs:
:param account:
"""
account_certs = Certificate.query.filter(Certificate.accounts.any(Account.id == 1)).\
filter(~Certificate.body.in_(certs)).all()
for a_cert in account_certs:
try:
a_cert.accounts.remove(account)
except Exception as e:
current_app.logger.debug("Skipping {0} account {1} is already disassociated".format(a_cert.name, account.label))
continue
database.update(a_cert)
def get_all_certs():
"""
Retrieves all certificates within Lemur.
@ -134,7 +110,7 @@ def mint(issuer_options):
issuer_options['creator'] = g.user.email
cert_body, cert_chain = issuer.create_certificate(csr, issuer_options)
cert = save_cert(cert_body, private_key, cert_chain, issuer_options.get('accounts'))
cert = save_cert(cert_body, private_key, cert_chain, issuer_options.get('destinations'))
cert.user = g.user
cert.authority = authority
database.update(cert)
@ -154,9 +130,10 @@ def import_certificate(**kwargs):
:param kwargs:
"""
from lemur.users import service as user_service
cert = Certificate(kwargs['public_certificate'])
cert.owner = kwargs.get('owner', )
cert.creator = kwargs.get('creator', 'Lemur')
cert.owner = kwargs.get('owner', current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
cert.creator = kwargs.get('creator', user_service.get_by_email('lemur@nobody'))
# NOTE existing certs may not follow our naming standard we will
# overwrite the generated name with the actual cert name
@ -166,31 +143,29 @@ def import_certificate(**kwargs):
if kwargs.get('user'):
cert.user = kwargs.get('user')
if kwargs.get('account'):
cert.accounts.append(kwargs.get('account'))
if kwargs.get('destination'):
cert.destinations.append(kwargs.get('destination'))
cert = database.create(cert)
return cert
def save_cert(cert_body, private_key, cert_chain, accounts):
def save_cert(cert_body, private_key, cert_chain, destinations):
"""
Determines if the certificate needs to be uploaded to AWS or other services.
:param cert_body:
:param private_key:
:param cert_chain:
:param challenge:
:param csr_config:
:param accounts:
:param destinations:
"""
cert = Certificate(cert_body, private_key, cert_chain)
# if we have an AWS accounts lets upload them
if accounts:
for account in accounts:
account = account_service.get(account['id'])
iam.upload_cert(account.account_number, cert, private_key, cert_chain)
cert.accounts.append(account)
# we should save them to any destination that is requested
for destination in destinations:
destination_plugin = plugins.get(destination['plugin']['slug'])
destination_plugin.upload(cert, private_key, cert_chain, destination['plugin']['pluginOptions'])
return cert
@ -198,13 +173,11 @@ def upload(**kwargs):
"""
Allows for pre-made certificates to be imported into Lemur.
"""
# save this cert the same way we save all of our certs, including uploading
# to aws if necessary
cert = save_cert(
kwargs.get('public_cert'),
kwargs.get('private_key'),
kwargs.get('intermediate_cert'),
kwargs.get('accounts')
kwargs.get('destinations')
)
cert.owner = kwargs['owner']
@ -237,7 +210,7 @@ def render(args):
query = database.session_query(Certificate)
time_range = args.pop('time_range')
account_id = args.pop('account_id')
destination_id = args.pop('destination_id')
show = args.pop('show')
owner = args.pop('owner')
creator = args.pop('creator') # TODO we should enabling filtering by owner
@ -260,8 +233,8 @@ def render(args):
)
return database.sort_and_page(query, Certificate, args)
if 'account' in terms:
query = query.filter(Certificate.accounts.any(Account.id == terms[1]))
if 'destination' in terms:
query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
elif 'active' in filt: # this is really weird but strcmp seems to not work here??
query = query.filter(Certificate.active == terms[1])
else:
@ -276,8 +249,8 @@ def render(args):
)
)
if account_id:
query = query.filter(Certificate.accounts.any(Account.id == account_id))
if destination_id:
query = query.filter(Certificate.destinations.any(Destination.id == destination_id))
if time_range:
to = arrow.now().replace(weeks=+time_range).format('YYYY-MM-DD')
@ -404,8 +377,8 @@ def stats(**kwargs):
if kwargs.get('active') == 'true':
query = query.filter(Certificate.elb_listeners.any())
if kwargs.get('account_id'):
query = query.filter(Certificate.accounts.any(Account.id == kwargs.get('account_id')))
if kwargs.get('destination_id'):
query = query.filter(Certificate.destinations.any(Destination.id == kwargs.get('destination_id')))
if kwargs.get('metric') == 'not_after':
start = arrow.utcnow()

160
lemur/certificates/sync.py
View File

@ -7,10 +7,6 @@
to 'sync' with as many different datasources as possible to try and track
any certificate that may be in use.
This include querying AWS for certificates attached to ELBs, querying our own
internal CA for certificates issued. As well as some rudimentary source code
scraping that attempts to find certificates checked into source code.
These operations are typically run on a periodic basis from either the command
line or a cron job.
@ -18,151 +14,33 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import requests
from bs4 import BeautifulSoup
from flask import current_app
from lemur.users import service as user_service
from lemur.accounts import service as account_service
from lemur.certificates import service as cert_service
from lemur.certificates.models import Certificate, get_name_from_arn
from lemur.common.services.aws.iam import get_all_server_certs
from lemur.common.services.aws.iam import get_cert_from_arn
from lemur.plugins.base import plugins
from lemur.plugins.bases.source import SourcePlugin
def aws():
"""
Attempts to retrieve all certificates located in known AWS accounts
:raise e:
"""
new = 0
updated = 0
def sync():
for plugin in plugins:
new = 0
updated = 0
if isinstance(plugin, SourcePlugin):
if plugin.is_enabled():
current_app.logger.error("Retrieving certificates from {0}".format(plugin.title))
certificates = plugin.get_certificates()
# all certificates 'discovered' by lemur are tracked by the lemur
# user
user = user_service.get_by_email('lemur@nobody')
for certificate in certificates:
exists = cert_service.find_duplicates(certificate)
# we don't need to check regions as IAM is a global service
for account in account_service.get_all():
certificate_bodies = []
try:
cert_arns = get_all_server_certs(account.account_number)
except Exception as e:
current_app.logger.error("Failed to to get Certificates from '{}/{}' reason {}".format(
account.label, account.account_number, e.message)
)
raise e
if not exists:
cert_service.import_certificate(**certificate)
new += 1
current_app.logger.info("found {} certs from '{}/{}' ... ".format(
len(cert_arns), account.account_number, account.label)
)
if len(exists) == 1:
updated += 1
for cert in cert_arns:
cert_body = get_cert_from_arn(cert.arn)[0]
certificate_bodies.append(cert_body)
existing = cert_service.find_duplicates(cert_body)
# TODO associated cert with source
# TODO update cert if found from different source
# TODO dissassociate source if missing
if not existing:
cert_service.import_certificate(
**{'owner': 'secops@netflix.com',
'creator': 'Lemur',
'name': get_name_from_arn(cert.arn),
'account': account,
'user': user,
'public_certificate': cert_body
}
)
new += 1
elif len(existing) == 1: # we check to make sure we know about the current account for this certificate
for e_account in existing[0].accounts:
if e_account.account_number == account.account_number:
break
else: # we have a new account
existing[0].accounts.append(account)
updated += 1
else:
current_app.logger.error(
"Multiple certificates with the same body found, unable to correctly determine which entry to update"
)
# make sure we remove any certs that have been removed from AWS
cert_service.disassociate_aws_account(certificate_bodies, account)
current_app.logger.info("found {} new certificates in aws {}".format(new, account.label))
def cloudca():
"""
Attempts to retrieve all certificates that are stored in CloudCA
"""
user = user_service.get_by_email('lemur@nobody')
# sync all new certificates/authorities not created through lemur
issuer = plugins.get('cloudca')
authorities = issuer.get_authorities()
total = 0
new = 1
for authority in authorities:
certs = issuer.get_cert(ca_name=authority)
for cert in certs:
total += 1
cert['user'] = user
existing = cert_service.find_duplicates(cert['public_certificate'])
if not existing:
new += 1
try:
cert_service.import_certificate(**cert)
except NameError as e:
current_app.logger.error("Cannot import certificate {0}".format(cert))
current_app.logger.debug("Found {0} total certificates in cloudca".format(total))
current_app.logger.debug("Found {0} new certificates in cloudca".format(new))
def source():
"""
Attempts to track certificates that are stored in Source Code
"""
new = 0
keywords = ['"--- Begin Certificate ---"']
endpoint = current_app.config.get('LEMUR_SOURCE_SEARCH')
maxresults = 25000
current_app.logger.info("Searching {0} for new certificates".format(endpoint))
for keyword in keywords:
current_app.logger.info("Looking for keyword: {0}".format(keyword))
url = "{}/source/s?n={}&start=1&sort=relevancy&q={}&project=github%2Cperforce%2Cstash".format(endpoint, maxresults, keyword)
current_app.logger.debug("Request url: {0}".format(url))
r = requests.get(url, timeout=20)
if r.status_code != 200:
current_app.logger.error("Unable to retrieve: {0} Status Code: {1}".format(url, r.status_code))
continue
soup = BeautifulSoup(r.text, "lxml")
results = soup.find_all(title='Download')
for result in results:
parts = result['href'].split('/')
path = "/".join(parts[:-1])
filename = parts[-1:][0]
r = requests.get("{0}{1}/{2}".format(endpoint, path, filename))
if r.status_code != 200:
current_app.logger.error("Unable to retrieve: {0} Status Code: {1}".format(url, r.status_code))
continue
try:
# validate we have a real certificate
cert = Certificate(r.content)
# do a lookup to see if we know about this certificate
existing = cert_service.find_duplicates(r.content)
if not existing:
current_app.logger.debug(cert.name)
cert_service.import_certificate()
new += 1
except Exception as e:
current_app.logger.debug("Could not parse the following 'certificate': {0} Reason: {1}".format(r.content, e))

14
lemur/certificates/views.py
View File

@ -164,7 +164,7 @@ class CertificatesList(AuthenticatedResource):
parser.add_argument('owner', type=bool, location='args')
parser.add_argument('id', type=str, location='args')
parser.add_argument('active', type=bool, location='args')
parser.add_argument('accountId', type=int, dest="account_id", location='args')
parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
parser.add_argument('creator', type=str, location='args')
parser.add_argument('show', type=str, location='args')
@ -271,7 +271,7 @@ class CertificatesList(AuthenticatedResource):
:statuscode 403: unauthenticated
"""
self.reqparse.add_argument('extensions', type=dict, location='json')
self.reqparse.add_argument('accounts', type=list, location='json')
self.reqparse.add_argument('destinations', type=list, default=[], location='json')
self.reqparse.add_argument('elbs', type=list, location='json')
self.reqparse.add_argument('owner', type=str, location='json')
self.reqparse.add_argument('validityStart', type=str, location='json') # parse date
@ -330,7 +330,7 @@ class CertificatesUpload(AuthenticatedResource):
"publicCert": "---Begin Public...",
"intermediateCert": "---Begin Public...",
"privateKey": "---Begin Private..."
"accounts": []
"destinations": []
}
**Example response**:
@ -364,19 +364,19 @@ class CertificatesUpload(AuthenticatedResource):
:arg publicCert: valid PEM public key for certificate
:arg intermediateCert valid PEM intermediate key for certificate
:arg privateKey: valid PEM private key for certificate
:arg accounts: list of aws accounts to upload the certificate to
:arg destinations: list of aws destinations to upload the certificate to
:reqheader Authorization: OAuth token to authenticate
:statuscode 403: unauthenticated
:statuscode 200: no error
"""
self.reqparse.add_argument('owner', type=str, required=True, location='json')
self.reqparse.add_argument('publicCert', type=pem_str, required=True, dest='public_cert', location='json')
self.reqparse.add_argument('accounts', type=list, dest='accounts', location='json')
self.reqparse.add_argument('destinations', type=list, default=[], dest='destinations', location='json')
self.reqparse.add_argument('intermediateCert', type=pem_str, dest='intermediate_cert', location='json')
self.reqparse.add_argument('privateKey', type=private_key_str, dest='private_key', location='json')
args = self.reqparse.parse_args()
if args.get('accounts'):
if args.get('destinations'):
if args.get('private_key'):
return service.upload(**args)
else:
@ -393,7 +393,7 @@ class CertificatesStats(AuthenticatedResource):
def get(self):
self.reqparse.add_argument('metric', type=str, location='args')
self.reqparse.add_argument('range', default=32, type=int, location='args')
self.reqparse.add_argument('accountId', dest='account_id', location='args')
self.reqparse.add_argument('destinationId', dest='destination_id', location='args')
self.reqparse.add_argument('active', type=str, default='true', location='args')
args = self.reqparse.parse_args()