Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' of github.com:Netflix/lemur into re-update-190412

This commit is contained in:
Hossein Shafagh 2019-04-12 14:26:35 -07:00
parent da51e7f31d 26e3982cb4
commit ad244fc83d
17 changed files with 201 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -1,6 +1,6 @@
language: python
sudo: required
dist: trusty
dist: xenial
node_js:
- "6.2.0"
@ -10,8 +10,8 @@ addons:
matrix:
include:
- python: "3.5"
env: TOXENV=py35
- python: "3.7"
env: TOXENV=py37
cache:
directories:

5
docker-compose.yml
View File

@ -13,10 +13,13 @@ services:
VIRTUAL_ENV: 'true'
postgres:
image: postgres:9.4
image: postgres
restart: always
environment:
POSTGRES_USER: lemur
POSTGRES_PASSWORD: lemur
ports:
- "5432:5432"
redis:
image: "redis:alpine"

20
lemur/common/celery.py
View File

@ -20,6 +20,8 @@ from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources.service import add_aws_destination_to_sources
if current_app:
flask_app = current_app
@ -255,3 +257,21 @@ def sync_source(source):
sync([source])
log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data)
@celery.task()
def sync_source_destination():
"""
This celery task will sync destination and source, to make sure all new destinations are also present as source.
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reveals the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
"""
current_app.logger.debug("Syncing AWS destinations and sources")
for dst in destinations_service.get_all():
if add_aws_destination_to_sources(dst):
current_app.logger.debug("Source: %s added", dst.label)
current_app.logger.debug("Completed Syncing AWS destinations and sources")

8
lemur/destinations/service.py
View File

@ -6,11 +6,13 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import func
from flask import current_app
from lemur import database
from lemur.models import certificate_destination_associations
from lemur.destinations.models import Destination
from lemur.certificates.models import Certificate
from lemur.sources.service import add_aws_destination_to_sources
def create(label, plugin_name, options, description=None):
@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None):
del option['value']['plugin_object']
destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
current_app.logger.info("Destination: %s created", label)
# add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
if add_aws_destination_to_sources(destination):
current_app.logger.info("Source: %s created", label)
return database.create(destination)

3
lemur/manage.py
View File

@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy # noqa
from lemur.pending_certificates.models import PendingCertificate # noqa
from lemur.dns_providers.models import DnsProvider # noqa
from sqlalchemy.sql import text
manager = Manager(create_app)
manager.add_option('-c', '--config', dest='config_path', required=False)
@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
@MigrateCommand.command
def create():
database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
database.db.create_all()
stamp(revision='head')

2
lemur/plugins/bases/destination.py
View File

@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
class DestinationPlugin(Plugin):
type = 'destination'
requires_key = True
sync_as_source = False
sync_as_source_name = ''
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
raise NotImplementedError

78
lemur/plugins/lemur_aws/plugin.py
View File

@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
return endpoints
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
# 'elb': {
# 'name': {'type': 'name'},
# 'region': {'type': 'str'},
# 'port': {'type': 'int'}
# }
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class AWSSourcePlugin(SourcePlugin):
title = 'AWS'
slug = 'aws-source'
@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
iam.delete_cert(certificate.name, account_number=account_number)
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
sync_as_source = True
sync_as_source_name = AWSSourcePlugin.slug
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class S3DestinationPlugin(ExportDestinationPlugin):
title = 'AWS-S3'
slug = 'aws-s3'

27
lemur/plugins/lemur_vault_dest/plugin.py
View File

@ -37,6 +37,17 @@ class VaultDestinationPlugin(DestinationPlugin):
'validation': '^https?://[a-zA-Z0-9.:-]+$',
'helpMessage': 'Valid URL to Hashi Vault instance'
},
{
'name': 'vaultKvApiVersion',
'type': 'select',
'value': '2',
'available': [
'1',
'2'
],
'required': True,
'helpMessage': 'Version of the Vault KV API to use'
},
{
'name': 'vaultAuthTokenFile',
'type': 'str',
@ -98,17 +109,20 @@ class VaultDestinationPlugin(DestinationPlugin):
path = self.get_option('vaultPath', options)
bundle = self.get_option('bundleChain', options)
obj_name = self.get_option('objectName', options)
api_version = self.get_option('vaultKvApiVersion', options)
with open(token_file, 'r') as file:
token = file.readline().rstrip('\n')
client = hvac.Client(url=url, token=token)
client.secrets.kv.default_kv_version = api_version
if obj_name:
path = '{0}/{1}'.format(path, obj_name)
else:
path = '{0}/{1}'.format(path, cname)
secret = get_secret(url, token, mount, path)
secret = get_secret(client, mount, path)
secret['data'][cname] = {}
if bundle == 'Nginx' and cert_chain:
@ -123,8 +137,9 @@ class VaultDestinationPlugin(DestinationPlugin):
if isinstance(san_list, list):
secret['data'][cname]['san'] = san_list
try:
client.secrets.kv.v1.create_or_update_secret(
path=path, mount_point=mount, secret=secret['data'])
client.secrets.kv.create_or_update_secret(
path=path, mount_point=mount, secret=secret['data']
)
except ConnectionError as err:
current_app.logger.exception(
"Exception uploading secret to vault: {0}".format(err), exc_info=True)
@ -144,12 +159,14 @@ def get_san_list(body):
return san_list
def get_secret(url, token, mount, path):
def get_secret(client, mount, path):
""" retreiive existing data from mount path and return dictionary """
result = {'data': {}}
try:
client = hvac.Client(url=url, token=token)
if client.secrets.kv.default_kv_version == '1':
result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
else:
result = client.secrets.kv.v2.read_secret_version(path=path, mount_point=mount)
except ConnectionError:
pass
finally:

12
lemur/plugins/utils.py
View File

@ -18,4 +18,14 @@ def get_plugin_option(name, options):
"""
for o in options:
if o.get('name') == name:
return o['value']
return o.get('value', o.get('default'))
def set_plugin_option(name, value, options):
"""
Set value for option name for options dict.
:param options:
"""
for o in options:
if o.get('name') == name:
o.update({'value': value})

34
lemur/sources/service.py
View File

@ -6,6 +6,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
import copy
from flask import current_app
@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi
from lemur.common.defaults import serial
from lemur.plugins.base import plugins
from lemur.plugins.utils import get_plugin_option, set_plugin_option
def certificate_create(certificate, source):
@ -256,3 +258,35 @@ def render(args):
query = database.filter(query, Source, terms)
return database.sort_and_page(query, Source, args)
def add_aws_destination_to_sources(dst):
"""
Given a destination check, if it can be added as sources, and included it if not already a source
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reveals the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
:return: true for success and false for not adding the destination as source
"""
# a set of all accounts numbers available as sources
src_accounts = set()
sources = get_all()
for src in sources:
src_accounts.add(get_plugin_option('accountNumber', src.options))
# check
destination_plugin = plugins.get(dst.plugin_name)
account_number = get_plugin_option('accountNumber', dst.options)
if account_number is not None and \
destination_plugin.sync_as_source is not None and \
destination_plugin.sync_as_source and \
(account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
set_plugin_option('accountNumber', account_number, src_options)
create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
return True
return False

2
lemur/tests/conftest.py
View File

@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from flask import current_app
from flask_principal import identity_changed, Identity
from sqlalchemy.sql import text
from lemur import create_app
from lemur.common.utils import parse_private_key
@ -55,6 +56,7 @@ def app(request):
@pytest.yield_fixture(scope="session")
def db(app, request):
_db.drop_all()
_db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
_db.create_all()
_db.app = app

8
requirements-dev.txt
View File

@ -7,18 +7,18 @@
aspy.yaml==1.2.0 # via pre-commit
bleach==3.1.0 # via readme-renderer
certifi==2019.3.9 # via requests
cfgv==1.5.0 # via pre-commit
cfgv==1.6.0 # via pre-commit
chardet==3.0.4 # via requests
docutils==0.14 # via readme-renderer
flake8==3.5.0
identify==1.4.0 # via pre-commit
identify==1.4.1 # via pre-commit
idna==2.8 # via requests
importlib-metadata==0.8 # via pre-commit
importlib-metadata==0.9 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.3
pkginfo==1.5.0.1 # via twine
pre-commit==1.14.4
pre-commit==1.15.1
pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8
pygments==2.3.1 # via readme-renderer

39
requirements-docs.txt
View File

@ -4,7 +4,7 @@
#
# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
#
acme==0.32.0
acme==0.33.1
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.8
@ -15,11 +15,11 @@ asn1crypto==0.24.0
asyncpool==1.0
babel==2.6.0 # via sphinx
bcrypt==3.1.6
billiard==3.5.0.5
billiard==3.6.0.0
blinker==1.4
boto3==1.9.120
botocore==1.12.120
celery[redis]==4.2.2
boto3==1.9.130
botocore==1.12.130
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
cffi==1.12.2
@ -42,28 +42,28 @@ flask-sqlalchemy==2.3.2
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
hvac==0.7.2
hvac==0.8.2
idna==2.8
imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==1.1.0
jinja2==2.10
jinja2==2.10.1
jmespath==0.9.4
josepy==1.1.0
jsonlines==1.2.0
kombu==4.3.0
kombu==4.5.0
lockfile==0.12.2
mako==1.0.8
markupsafe==1.1.1
marshmallow-sqlalchemy==0.16.1
marshmallow==2.19.1
marshmallow==2.19.2
mock==2.0.0
ndg-httpsclient==0.5.1
packaging==19.0 # via sphinx
paramiko==2.4.2
pbr==5.1.3
pem==19.1.0
psycopg2==2.7.7
psycopg2==2.8.1
pyasn1-modules==0.2.4
pyasn1==0.4.5
pycparser==2.19
@ -71,14 +71,14 @@ pygments==2.3.1 # via sphinx
pyjwt==1.7.1
pynacl==1.3.0
pyopenssl==19.0.0
pyparsing==2.3.1 # via packaging
pyparsing==2.4.0 # via packaging
pyrfc3339==1.1
python-dateutil==2.8.0
python-editor==1.0.4
pytz==2018.9
pytz==2019.1
pyyaml==5.1
raven[flask]==6.10.0
redis==2.10.6
redis==3.2.1
requests-toolbelt==0.9.1
requests[security]==2.21.0
retrying==1.3.3
@ -86,13 +86,18 @@ s3transfer==0.2.0
six==1.12.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.4.3
sphinx==1.8.5
sphinx==2.0.1
sphinxcontrib-applehelp==1.0.1 # via sphinx
sphinxcontrib-devhelp==1.0.1 # via sphinx
sphinxcontrib-htmlhelp==1.0.1 # via sphinx
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sphinxcontrib-jsmath==1.0.1 # via sphinx
sphinxcontrib-qthelp==1.0.2 # via sphinx
sphinxcontrib-serializinghtml==1.1.3 # via sphinx
sqlalchemy-utils==0.33.11
sqlalchemy==1.3.1
sqlalchemy==1.3.2
tabulate==0.8.3
urllib3==1.24.1
vine==1.3.0
werkzeug==0.15.1
werkzeug==0.15.2
xmltodict==0.12.0

20
requirements-tests.txt
View File

@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.3.0 # via pytest
attrs==19.1.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.9.120 # via moto
boto3==1.9.130 # via moto
boto==2.49.0 # via moto
botocore==1.12.120 # via boto3, moto, s3transfer
botocore==1.12.130 # via boto3, moto, s3transfer
certifi==2019.3.9 # via requests
cffi==1.12.2 # via cryptography
chardet==3.0.4 # via requests
@ -18,7 +18,7 @@ click==7.0 # via flask
coverage==4.5.3
cryptography==2.6.1 # via moto
docker-pycreds==0.4.0 # via docker
docker==3.7.1 # via moto
docker==3.7.2 # via moto
docutils==0.14 # via botocore
ecdsa==0.13 # via python-jose
factory-boy==2.11.1
@ -28,13 +28,13 @@ freezegun==0.3.11
future==0.17.1 # via python-jose
idna==2.8 # via requests
itsdangerous==1.1.0 # via flask
jinja2==2.10 # via flask, moto
jinja2==2.10.1 # via flask, moto
jmespath==0.9.4 # via boto3, botocore
jsondiff==1.1.1 # via moto
jsonpickle==1.1 # via aws-xray-sdk
markupsafe==1.1.1 # via jinja2
mock==2.0.0 # via moto
more-itertools==6.0.0 # via pytest
more-itertools==7.0.0 # via pytest
moto==1.3.7
nose==1.3.7
pbr==5.1.3 # via mock
@ -42,14 +42,14 @@ pluggy==0.9.0 # via pytest
py==1.8.0 # via pytest
pyaml==18.11.0 # via moto
pycparser==2.19 # via cffi
pycryptodome==3.8.0 # via python-jose
pycryptodome==3.8.1 # via python-jose
pyflakes==2.1.1
pytest-flask==0.14.0
pytest-mock==1.10.2
pytest==4.3.1
pytest-mock==1.10.3
pytest==4.4.0
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.9 # via moto
pytz==2019.1 # via moto
pyyaml==5.1
requests-mock==1.5.2
requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
@ -59,6 +59,6 @@ six==1.12.0 # via cryptography, docker, docker-pycreds, faker, fre
text-unidecode==1.2 # via faker
urllib3==1.24.1 # via botocore, requests
websocket-client==0.56.0 # via docker
werkzeug==0.15.1 # via flask, moto, pytest-flask
werkzeug==0.15.2 # via flask, moto, pytest-flask
wrapt==1.11.1 # via aws-xray-sdk
xmltodict==0.12.0 # via moto

4
requirements.in
View File

@ -27,7 +27,7 @@ gunicorn
hvac # required for the vault destination plugin
inflection
jinja2
kombu==4.3.0 # kombu 4.4.0 requires redis 3
kombu
lockfile
marshmallow-sqlalchemy
marshmallow
@ -39,7 +39,7 @@ pyjwt
pyOpenSSL
python_ldap
raven[flask]
redis<3 # redis>=3 is not compatible with celery
redis
requests
retrying
six

30
requirements.txt
View File

@ -4,7 +4,7 @@
#
# pip-compile --output-file requirements.txt requirements.in -U --no-index
#
acme==0.32.0
acme==0.33.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.8 # via flask-migrate
amqp==2.4.2 # via kombu
@ -13,11 +13,11 @@ arrow==0.13.1
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.6 # via flask-bcrypt, paramiko
billiard==3.5.0.5 # via celery
billiard==3.6.0.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.120
botocore==1.12.120
celery[redis]==4.2.2
boto3==1.9.130
botocore==1.12.130
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
cffi==1.12.2 # via bcrypt, cryptography, pynacl
@ -40,26 +40,26 @@ flask-sqlalchemy==2.3.2
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
hvac==0.7.2
hvac==0.8.2
idna==2.8 # via requests
inflection==0.3.1
itsdangerous==1.1.0 # via flask
jinja2==2.10
jinja2==2.10.1
jmespath==0.9.4 # via boto3, botocore
josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.3.0
kombu==4.5.0
lockfile==0.12.2
mako==1.0.8 # via alembic
markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.16.1
marshmallow==2.19.1
marshmallow==2.19.2
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.1.3 # via mock
pem==19.1.0
psycopg2==2.7.7
psycopg2==2.8.1
pyasn1-modules==0.2.4 # via python-ldap
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pycparser==2.19 # via cffi
@ -70,19 +70,19 @@ pyrfc3339==1.1 # via acme
python-dateutil==2.8.0 # via alembic, arrow, botocore
python-editor==1.0.4 # via alembic
python-ldap==3.2.0
pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339
pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339
pyyaml==5.1
raven[flask]==6.10.0
redis==2.10.6
redis==3.2.1
requests-toolbelt==0.9.1 # via acme
requests[security]==2.21.0
retrying==1.3.3
s3transfer==0.2.0 # via boto3
six==1.12.0
sqlalchemy-utils==0.33.11
sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlalchemy==1.3.2 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.3
urllib3==1.24.1 # via botocore, requests
vine==1.3.0 # via amqp
werkzeug==0.15.1 # via flask
vine==1.3.0 # via amqp, celery
werkzeug==0.15.2 # via flask
xmltodict==0.12.0

2
tox.ini
View File

@ -1,2 +1,2 @@
[tox]
envlist = py35
envlist = py37