Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into lemur_vault_plugin

This commit is contained in:
alwaysjolley 2019-04-18 13:55:45 -04:00
parent fb3b0e8cd7 4c7bf917f2
commit b39e2e3f66
14 changed files with 164 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -125,5 +125,9 @@ endif
@echo "--> Done installing new dependencies"
@echo ""
# Execute with make checkout-pr pr=<pr number>
checkout-pr:
git fetch upstream pull/$(pr)/head:pr-$(pr)
.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

17
lemur/certificates/schemas.py
View File

@ -112,10 +112,20 @@ class CertificateInputSchema(CertificateCreationSchema):
if data.get('replacements'):
data['replaces'] = data['replacements'] # TODO remove when field is deprecated
if data.get('csr'):
dns_names = cert_utils.get_dns_names_from_csr(data['csr'])
if not data['extensions']['subAltNames']['names']:
csr_sans = cert_utils.get_sans_from_csr(data['csr'])
if not data.get('extensions'):
data['extensions'] = {
'subAltNames': {
'names': []
}
}
elif not data['extensions'].get('subAltNames'):
data['extensions']['subAltNames'] = {
'names': []
}
elif not data['extensions']['subAltNames'].get('names'):
data['extensions']['subAltNames']['names'] = []
data['extensions']['subAltNames']['names'] += dns_names
data['extensions']['subAltNames']['names'] += csr_sans
return missing.convert_validity_years(data)
@ -255,6 +265,7 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
private_key = fields.String()
body = fields.String(required=True)
chain = fields.String(missing=None, allow_none=True)
csr = fields.String(required=False, allow_none=True, validate=validators.csr)
destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)

24
lemur/certificates/utils.py
View File

@ -14,14 +14,14 @@ from cryptography.hazmat.backends import default_backend
from marshmallow.exceptions import ValidationError
def get_dns_names_from_csr(data):
def get_sans_from_csr(data):
"""
Fetches DNSNames from CSR.
Potentially extendable to any kind of SubjectAlternativeName
Fetches SubjectAlternativeNames from CSR.
Works with any kind of SubjectAlternativeName
:param data: PEM-encoded string with CSR
:return:
:return: List of LemurAPI-compatible subAltNames
"""
dns_names = []
sub_alt_names = []
try:
request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
except Exception:
@ -29,14 +29,12 @@ def get_dns_names_from_csr(data):
try:
alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
for name in alt_names.value.get_values_for_type(x509.DNSName):
dns_name = {
'nameType': 'DNSName',
'value': name
}
dns_names.append(dns_name)
for alt_name in alt_names.value:
sub_alt_names.append({
'nameType': type(alt_name).__name__,
'value': alt_name.value
})
except x509.ExtensionNotFound:
pass
return dns_names
return sub_alt_names

1
lemur/certificates/views.py
View File

@ -306,6 +306,7 @@ class CertificatesUpload(AuthenticatedResource):
"body": "-----BEGIN CERTIFICATE-----...",
"chain": "-----BEGIN CERTIFICATE-----...",
"privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
"csr": "-----BEGIN CERTIFICATE REQUEST-----..."
"destinations": [],
"notifications": [],
"replacements": [],

31
lemur/common/celery.py
View File

@ -18,11 +18,10 @@ from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins, IPlugin
from lemur.plugins.base import plugins
from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources import service as sources_service
from lemur.sources.service import add_aws_destination_to_sources
if current_app:
flask_app = current_app
@ -266,27 +265,13 @@ def sync_source_destination():
This celery task will sync destination and source, to make sure all new destinations are also present as source.
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reviels the name of the suitable source-plugin.
The destination sync_as_source_name reveals the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
"""
current_app.logger.debug("Syncing source and destination")
# a set of all accounts numbers available as sources
src_accounts = set()
sources = validate_sources("all")
for src in sources:
src_accounts.add(IPlugin.get_option('accountNumber', src.options))
current_app.logger.debug("Syncing AWS destinations and sources")
for dst in destinations_service.get_all():
destination_plugin = plugins.get(dst.plugin_name)
account_number = IPlugin.get_option('accountNumber', dst.options)
if destination_plugin.sync_as_source and (account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
for o in src_options:
if o.get('name') == 'accountNumber':
o.update({'value': account_number})
sources_service.create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
current_app.logger.info("Source: %s added", dst.label)
if add_aws_destination_to_sources(dst):
current_app.logger.debug("Source: %s added", dst.label)
current_app.logger.debug("Completed Syncing AWS destinations and sources")

8
lemur/destinations/service.py
View File

@ -6,11 +6,13 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import func
from flask import current_app
from lemur import database
from lemur.models import certificate_destination_associations
from lemur.destinations.models import Destination
from lemur.certificates.models import Certificate
from lemur.sources.service import add_aws_destination_to_sources
def create(label, plugin_name, options, description=None):
@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None):
del option['value']['plugin_object']
destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
current_app.logger.info("Destination: %s created", label)
# add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
if add_aws_destination_to_sources(destination):
current_app.logger.info("Source: %s created", label)
return database.create(destination)

12
lemur/plugins/utils.py
View File

@ -18,4 +18,14 @@ def get_plugin_option(name, options):
"""
for o in options:
if o.get('name') == name:
return o['value']
return o.get('value', o.get('default'))
def set_plugin_option(name, value, options):
"""
Set value for option name for options dict.
:param options:
"""
for o in options:
if o.get('name') == name:
o.update({'value': value})

34
lemur/sources/service.py
View File

@ -6,6 +6,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
import copy
from flask import current_app
@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi
from lemur.common.defaults import serial
from lemur.plugins.base import plugins
from lemur.plugins.utils import get_plugin_option, set_plugin_option
def certificate_create(certificate, source):
@ -256,3 +258,35 @@ def render(args):
query = database.filter(query, Source, terms)
return database.sort_and_page(query, Source, args)
def add_aws_destination_to_sources(dst):
"""
Given a destination check, if it can be added as sources, and included it if not already a source
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reveals the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
:return: true for success and false for not adding the destination as source
"""
# a set of all accounts numbers available as sources
src_accounts = set()
sources = get_all()
for src in sources:
src_accounts.add(get_plugin_option('accountNumber', src.options))
# check
destination_plugin = plugins.get(dst.plugin_name)
account_number = get_plugin_option('accountNumber', dst.options)
if account_number is not None and \
destination_plugin.sync_as_source is not None and \
destination_plugin.sync_as_source and \
(account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
set_plugin_option('accountNumber', account_number, src_options)
create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
return True
return False

13
lemur/static/app/angular/certificates/certificate/upload.tpl.html
View File

@ -62,6 +62,19 @@
a valid certificate.</p>
</div>
</div>
<div class="form-group"
ng-class="{'has-error': uploadForm.csr.$invalid&&uploadForm.csr.$dirty, 'has-success': !uploadForm.csr.$invalid&&uploadForm.csr.$dirty}">
<label class="control-label col-sm-2">
Certificate Signing Request (CSR)
</label>
<div class="col-sm-10">
<textarea name="csr" ng-model="certificate.csr" placeholder="PEM encoded string..."
class="form-control"
ng-pattern="/^-----BEGIN CERTIFICATE REQUEST-----/"></textarea>
<p ng-show="uploadForm.csr.$invalid && !uploadForm.csr.$pristine"
class="help-block">Enter a valid certificate signing request.</p>
</div>
</div>
<div class="form-group"
ng-class="{'has-error': uploadForm.owner.$invalid&&uploadform.intermediateCert.$dirty, 'has-success': !uploadForm.intermediateCert.$invalid&&uploadForm.intermediateCert.$dirty}">
<label class="control-label col-sm-2">

28
lemur/tests/test_certificates.py
View File

@ -18,7 +18,7 @@ from lemur.domains.models import Domain
from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \
INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_CSR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
def test_get_or_increase_name(session, certificate):
@ -284,6 +284,31 @@ def test_certificate_input_with_extensions(client, authority):
assert not errors
def test_certificate_input_schema_parse_csr(authority):
from lemur.certificates.schemas import CertificateInputSchema
test_san_dns = 'foobar.com'
extensions = {'sub_alt_names': {'names': x509.SubjectAlternativeName([x509.DNSName(test_san_dns)])}}
csr, private_key = create_csr(owner='joe@example.com', common_name='ACommonName', organization='test',
organizational_unit='Meters', country='NL', state='Noord-Holland', location='Amsterdam',
key_type='RSA2048', extensions=extensions)
input_data = {
'commonName': 'test.example.com',
'owner': 'jim@example.com',
'authority': {'id': authority.id},
'description': 'testtestest',
'csr': csr,
'dnsProvider': None,
}
data, errors = CertificateInputSchema().load(input_data)
for san in data['extensions']['sub_alt_names']['names']:
assert san.value == test_san_dns
assert not errors
def test_certificate_out_of_range_date(client, authority):
from lemur.certificates.schemas import CertificateInputSchema
input_data = {
@ -456,6 +481,7 @@ def test_certificate_upload_schema_ok(client):
'body': SAN_CERT_STR,
'privateKey': SAN_CERT_KEY,
'chain': INTERMEDIATE_CERT_STR,
'csr': SAN_CERT_CSR,
'external_id': '1234',
}
data, errors = CertificateUploadInputSchema().load(data)

20
lemur/tests/vectors.py
View File

@ -137,6 +137,26 @@ eMVHHbWm1CpGO294R+vMBv4jcuhIBOx63KZE4VaoJuaazF6TE5czDw==
#: CN=san.example.org, issued by LemurTrust Unittests Class 1 CA 2018
SAN_CERT_CSR = """\
-----BEGIN CERTIFICATE REQUEST-----
MIICvTCCAaUCAQAweDELMAkGA1UEBhMCRUUxDDAKBgNVBAgMA04vQTEOMAwGA1UE
BwwFRWFydGgxGDAWBgNVBAoMD0RhbmllbCBTYW4gJiBjbzEXMBUGA1UECwwOS2Fy
YXRlIExlc3NvbnMxGDAWBgNVBAMMD3Nhbi5leGFtcGxlLm9yZzCCASIwDQYJKoZI
hvcNAQEBBQADggEPADCCAQoCggEBAMia9BcpypZUU9xJoknzdEp+AevQE93XSAyl
IlXji80ZlYS/T/mVWtu6hNwz2IJDBFh6nPaHT1Ud/AI4YanDMa+fF4KJxzlkKPbY
quWx4EOjTZ2sFBBCivwxlo1So8r5Hf4NZ9Ewu4AIma3zmk+dzxJTpnWbTIFJGsDG
LwJO9iu6uqf79VdYkGELCusq3dyF2j2DNDiGHoRcQYFMMhDKR6uYmCTYvwjf0+sf
6k1zk2EK1X+ZWUyjP+Nl2NB6bpL0TydF75fuplWROczceiO6BKO4YT2uNPdF4BAH
p/kQCkqnjw5FCX7PONRT4wTW/AjDkt5WOgY+AB90zQBPxvXWbUMCAwEAAaAAMA0G
CSqGSIb3DQEBCwUAA4IBAQAFYgEafwRmsqdK1i1xrLFYbNNLkzmAZyL+6gXUBVIJ
TbGVVWSNNIcEmHIX8O9X4lN52qDYWOsxH/OKPVxpXqoHm/ztczFlte76wOYg+VAS
yK8DwQRP/+n+j6J40o1cZwnilPWqHgee5zbIL7lpCVxuFDofWpskwP5PLbxibFq8
4TWynhjKKUw4+q4h4iCHG3PQhbV0ExWOyqX05QyDtJdkEwgJUWz1m9caHU2Jl7kX
5bWKOtXORpCYA7ed3WqktKQIxBD6vCVbQ+LuLZPYeWzGHYjfOejL6usD32KmNa2E
ZhDsC0fjqSX0FJKz6gOhP88bkbbapyHuGB71o2dwhCKV
-----END CERTIFICATE REQUEST-----
"""
SAN_CERT_STR = """\
-----BEGIN CERTIFICATE-----
MIIESjCCAzKgAwIBAgIRAK/y20+NLU2OgPo4KuJ8IzMwDQYJKoZIhvcNAQELBQAw

12
requirements-docs.txt
View File

@ -7,7 +7,7 @@
acme==0.33.1
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.8
alembic==1.0.9
amqp==2.4.2
aniso8601==6.0.0
arrow==0.13.1
@ -53,9 +53,9 @@ josepy==1.1.0
jsonlines==1.2.0
kombu==4.5.0
lockfile==0.12.2
mako==1.0.8
mako==1.0.9
markupsafe==1.1.1
marshmallow-sqlalchemy==0.16.1
marshmallow-sqlalchemy==0.16.2
marshmallow==2.19.2
mock==2.0.0
ndg-httpsclient==0.5.1
@ -63,7 +63,7 @@ packaging==19.0 # via sphinx
paramiko==2.4.2
pbr==5.1.3
pem==19.1.0
psycopg2==2.8.1
psycopg2==2.8.2
pyasn1-modules==0.2.4
pyasn1==0.4.5
pycparser==2.19
@ -89,13 +89,13 @@ sphinx-rtd-theme==0.4.3
sphinx==2.0.1
sphinxcontrib-applehelp==1.0.1 # via sphinx
sphinxcontrib-devhelp==1.0.1 # via sphinx
sphinxcontrib-htmlhelp==1.0.1 # via sphinx
sphinxcontrib-htmlhelp==1.0.2 # via sphinx
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-jsmath==1.0.1 # via sphinx
sphinxcontrib-qthelp==1.0.2 # via sphinx
sphinxcontrib-serializinghtml==1.1.3 # via sphinx
sqlalchemy-utils==0.33.11
sqlalchemy==1.3.2
sqlalchemy==1.3.3
tabulate==0.8.3
urllib3==1.24.1
vine==1.3.0

4
requirements-tests.txt
View File

@ -22,7 +22,7 @@ docker==3.7.2 # via moto
docutils==0.14 # via botocore
ecdsa==0.13 # via python-jose
factory-boy==2.11.1
faker==1.0.4
faker==1.0.5
flask==1.0.2 # via pytest-flask
freezegun==0.3.11
future==0.17.1 # via python-jose
@ -46,7 +46,7 @@ pycryptodome==3.8.1 # via python-jose
pyflakes==2.1.1
pytest-flask==0.14.0
pytest-mock==1.10.3
pytest==4.4.0
pytest==4.4.1
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2019.1 # via moto

10
requirements.txt
View File

@ -6,7 +6,7 @@
#
acme==0.33.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.8 # via flask-migrate
alembic==1.0.9 # via flask-migrate
amqp==2.4.2 # via kombu
aniso8601==6.0.0 # via flask-restful
arrow==0.13.1
@ -50,16 +50,16 @@ josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.5.0
lockfile==0.12.2
mako==1.0.8 # via alembic
mako==1.0.9 # via alembic
markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.16.1
marshmallow-sqlalchemy==0.16.2
marshmallow==2.19.2
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.1.3 # via mock
pem==19.1.0
psycopg2==2.8.1
psycopg2==2.8.2
pyasn1-modules==0.2.4 # via python-ldap
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pycparser==2.19 # via cffi
@ -80,7 +80,7 @@ retrying==1.3.3
s3transfer==0.2.0 # via boto3
six==1.12.0
sqlalchemy-utils==0.33.11
sqlalchemy==1.3.2 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlalchemy==1.3.3 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.3
urllib3==1.24.1 # via botocore, requests
vine==1.3.0 # via amqp, celery