Merge branch 'master' into hshafagh-src-dst-register
This commit is contained in:
Hossein Shafagh
GitHub
commit
9a210c055a
|
|
@ -1,6 +1,6 @@
|
|||
language: python
|
||||
sudo: required
|
||||
dist: trusty
|
||||
dist: xenial
|
||||
|
||||
node_js:
|
||||
- "6.2.0"
|
||||
|
|
@ -10,8 +10,8 @@ addons:
|
|||
|
||||
matrix:
|
||||
include:
|
||||
- python: "3.5"
|
||||
env: TOXENV=py35
|
||||
- python: "3.7"
|
||||
env: TOXENV=py37
|
||||
|
||||
cache:
|
||||
directories:
|
||||
|
|
|
|||
|
|
@ -13,10 +13,13 @@ services:
|
|||
VIRTUAL_ENV: 'true'
|
||||
|
||||
postgres:
|
||||
image: postgres:9.4
|
||||
image: postgres
|
||||
restart: always
|
||||
environment:
|
||||
POSTGRES_USER: lemur
|
||||
POSTGRES_PASSWORD: lemur
|
||||
ports:
|
||||
- "5432:5432"
|
||||
|
||||
redis:
|
||||
image: "redis:alpine"
|
||||
|
|
|
|||
|
|
@ -18,8 +18,11 @@ from lemur.authorities.service import get as get_authority
|
|||
from lemur.factory import create_app
|
||||
from lemur.notifications.messaging import send_pending_failure_notification
|
||||
from lemur.pending_certificates import service as pending_certificate_service
|
||||
from lemur.plugins.base import plugins
|
||||
from lemur.plugins.base import plugins, IPlugin
|
||||
from lemur.sources.cli import clean, sync, validate_sources
|
||||
from lemur.destinations import service as destinations_service
|
||||
from lemur.sources import service as sources_service
|
||||
|
||||
|
||||
if current_app:
|
||||
flask_app = current_app
|
||||
|
|
@ -255,3 +258,35 @@ def sync_source(source):
|
|||
sync([source])
|
||||
log_data["message"] = "Done syncing source"
|
||||
current_app.logger.debug(log_data)
|
||||
|
||||
|
||||
@celery.task()
|
||||
def sync_source_destination():
|
||||
"""
|
||||
This celery task will sync destination and source, to make sure all new destinations are also present as source.
|
||||
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
|
||||
We identify qualified destinations based on the sync_as_source attributed of the plugin.
|
||||
The destination sync_as_source_name reviels the name of the suitable source-plugin.
|
||||
We rely on account numbers to avoid duplicates.
|
||||
"""
|
||||
current_app.logger.debug("Syncing source and destination")
|
||||
|
||||
# a set of all accounts numbers available as sources
|
||||
src_accounts = set()
|
||||
sources = validate_sources("all")
|
||||
for src in sources:
|
||||
src_accounts.add(IPlugin.get_option('accountNumber', src.options))
|
||||
|
||||
for dst in destinations_service.get_all():
|
||||
destination_plugin = plugins.get(dst.plugin_name)
|
||||
account_number = IPlugin.get_option('accountNumber', dst.options)
|
||||
if destination_plugin.sync_as_source and (account_number not in src_accounts):
|
||||
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
|
||||
for o in src_options:
|
||||
if o.get('name') == 'accountNumber':
|
||||
o.update({'value': account_number})
|
||||
sources_service.create(label=dst.label,
|
||||
plugin_name=destination_plugin.sync_as_source_name,
|
||||
options=src_options,
|
||||
description=dst.description)
|
||||
current_app.logger.info("Source: %s added", dst.label)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy # noqa
|
|||
from lemur.pending_certificates.models import PendingCertificate # noqa
|
||||
from lemur.dns_providers.models import DnsProvider # noqa
|
||||
|
||||
from sqlalchemy.sql import text
|
||||
|
||||
manager = Manager(create_app)
|
||||
manager.add_option('-c', '--config', dest='config_path', required=False)
|
||||
|
||||
|
|
@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
|
|||
|
||||
@MigrateCommand.command
|
||||
def create():
|
||||
database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
|
||||
database.db.create_all()
|
||||
stamp(revision='head')
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
|
|||
class DestinationPlugin(Plugin):
|
||||
type = 'destination'
|
||||
requires_key = True
|
||||
sync_as_source = False
|
||||
sync_as_source_name = ''
|
||||
|
||||
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
|
|
|||
|
|
@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
|
|||
return endpoints
|
||||
|
||||
|
||||
class AWSDestinationPlugin(DestinationPlugin):
|
||||
title = 'AWS'
|
||||
slug = 'aws-destination'
|
||||
description = 'Allow the uploading of certificates to AWS IAM'
|
||||
version = aws.VERSION
|
||||
|
||||
author = 'Kevin Glisson'
|
||||
author_url = 'https://github.com/netflix/lemur'
|
||||
|
||||
options = [
|
||||
{
|
||||
'name': 'accountNumber',
|
||||
'type': 'str',
|
||||
'required': True,
|
||||
'validation': '[0-9]{12}',
|
||||
'helpMessage': 'Must be a valid AWS account number!',
|
||||
},
|
||||
{
|
||||
'name': 'path',
|
||||
'type': 'str',
|
||||
'default': '/',
|
||||
'helpMessage': 'Path to upload certificate.'
|
||||
}
|
||||
]
|
||||
|
||||
# 'elb': {
|
||||
# 'name': {'type': 'name'},
|
||||
# 'region': {'type': 'str'},
|
||||
# 'port': {'type': 'int'}
|
||||
# }
|
||||
|
||||
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
|
||||
iam.upload_cert(name, body, private_key,
|
||||
self.get_option('path', options),
|
||||
cert_chain=cert_chain,
|
||||
account_number=self.get_option('accountNumber', options))
|
||||
|
||||
def deploy(self, elb_name, account, region, certificate):
|
||||
pass
|
||||
|
||||
|
||||
class AWSSourcePlugin(SourcePlugin):
|
||||
title = 'AWS'
|
||||
slug = 'aws-source'
|
||||
|
|
@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
|
|||
iam.delete_cert(certificate.name, account_number=account_number)
|
||||
|
||||
|
||||
class AWSDestinationPlugin(DestinationPlugin):
|
||||
title = 'AWS'
|
||||
slug = 'aws-destination'
|
||||
description = 'Allow the uploading of certificates to AWS IAM'
|
||||
version = aws.VERSION
|
||||
sync_as_source = True
|
||||
sync_as_source_name = AWSSourcePlugin.slug
|
||||
|
||||
author = 'Kevin Glisson'
|
||||
author_url = 'https://github.com/netflix/lemur'
|
||||
|
||||
options = [
|
||||
{
|
||||
'name': 'accountNumber',
|
||||
'type': 'str',
|
||||
'required': True,
|
||||
'validation': '[0-9]{12}',
|
||||
'helpMessage': 'Must be a valid AWS account number!',
|
||||
},
|
||||
{
|
||||
'name': 'path',
|
||||
'type': 'str',
|
||||
'default': '/',
|
||||
'helpMessage': 'Path to upload certificate.'
|
||||
}
|
||||
]
|
||||
|
||||
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
|
||||
iam.upload_cert(name, body, private_key,
|
||||
self.get_option('path', options),
|
||||
cert_chain=cert_chain,
|
||||
account_number=self.get_option('accountNumber', options))
|
||||
|
||||
def deploy(self, elb_name, account, region, certificate):
|
||||
pass
|
||||
|
||||
|
||||
class S3DestinationPlugin(ExportDestinationPlugin):
|
||||
title = 'AWS-S3'
|
||||
slug = 'aws-s3'
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend
|
|||
from cryptography.hazmat.primitives import hashes
|
||||
from flask import current_app
|
||||
from flask_principal import identity_changed, Identity
|
||||
from sqlalchemy.sql import text
|
||||
|
||||
from lemur import create_app
|
||||
from lemur.common.utils import parse_private_key
|
||||
|
|
@ -55,6 +56,7 @@ def app(request):
|
|||
@pytest.yield_fixture(scope="session")
|
||||
def db(app, request):
|
||||
_db.drop_all()
|
||||
_db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
|
||||
_db.create_all()
|
||||
|
||||
_db.app = app
|
||||
|
|
|
|||
|
|
@ -7,18 +7,18 @@
|
|||
aspy.yaml==1.2.0 # via pre-commit
|
||||
bleach==3.1.0 # via readme-renderer
|
||||
certifi==2019.3.9 # via requests
|
||||
cfgv==1.5.0 # via pre-commit
|
||||
cfgv==1.6.0 # via pre-commit
|
||||
chardet==3.0.4 # via requests
|
||||
docutils==0.14 # via readme-renderer
|
||||
flake8==3.5.0
|
||||
identify==1.4.0 # via pre-commit
|
||||
identify==1.4.1 # via pre-commit
|
||||
idna==2.8 # via requests
|
||||
importlib-metadata==0.8 # via pre-commit
|
||||
importlib-metadata==0.9 # via pre-commit
|
||||
invoke==1.2.0
|
||||
mccabe==0.6.1 # via flake8
|
||||
nodeenv==1.3.3
|
||||
pkginfo==1.5.0.1 # via twine
|
||||
pre-commit==1.14.4
|
||||
pre-commit==1.15.1
|
||||
pycodestyle==2.3.1 # via flake8
|
||||
pyflakes==1.6.0 # via flake8
|
||||
pygments==2.3.1 # via readme-renderer
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
#
|
||||
# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
|
||||
#
|
||||
acme==0.32.0
|
||||
acme==0.33.1
|
||||
alabaster==0.7.12 # via sphinx
|
||||
alembic-autogenerate-enums==0.0.2
|
||||
alembic==1.0.8
|
||||
|
|
@ -15,11 +15,11 @@ asn1crypto==0.24.0
|
|||
asyncpool==1.0
|
||||
babel==2.6.0 # via sphinx
|
||||
bcrypt==3.1.6
|
||||
billiard==3.5.0.5
|
||||
billiard==3.6.0.0
|
||||
blinker==1.4
|
||||
boto3==1.9.120
|
||||
botocore==1.12.120
|
||||
celery[redis]==4.2.2
|
||||
boto3==1.9.130
|
||||
botocore==1.12.130
|
||||
celery[redis]==4.3.0
|
||||
certifi==2019.3.9
|
||||
certsrv==2.1.1
|
||||
cffi==1.12.2
|
||||
|
|
@ -42,28 +42,28 @@ flask-sqlalchemy==2.3.2
|
|||
flask==1.0.2
|
||||
future==0.17.1
|
||||
gunicorn==19.9.0
|
||||
hvac==0.7.2
|
||||
hvac==0.8.2
|
||||
idna==2.8
|
||||
imagesize==1.1.0 # via sphinx
|
||||
inflection==0.3.1
|
||||
itsdangerous==1.1.0
|
||||
jinja2==2.10
|
||||
jinja2==2.10.1
|
||||
jmespath==0.9.4
|
||||
josepy==1.1.0
|
||||
jsonlines==1.2.0
|
||||
kombu==4.3.0
|
||||
kombu==4.5.0
|
||||
lockfile==0.12.2
|
||||
mako==1.0.8
|
||||
markupsafe==1.1.1
|
||||
marshmallow-sqlalchemy==0.16.1
|
||||
marshmallow==2.19.1
|
||||
marshmallow==2.19.2
|
||||
mock==2.0.0
|
||||
ndg-httpsclient==0.5.1
|
||||
packaging==19.0 # via sphinx
|
||||
paramiko==2.4.2
|
||||
pbr==5.1.3
|
||||
pem==19.1.0
|
||||
psycopg2==2.7.7
|
||||
psycopg2==2.8.1
|
||||
pyasn1-modules==0.2.4
|
||||
pyasn1==0.4.5
|
||||
pycparser==2.19
|
||||
|
|
@ -71,14 +71,14 @@ pygments==2.3.1 # via sphinx
|
|||
pyjwt==1.7.1
|
||||
pynacl==1.3.0
|
||||
pyopenssl==19.0.0
|
||||
pyparsing==2.3.1 # via packaging
|
||||
pyparsing==2.4.0 # via packaging
|
||||
pyrfc3339==1.1
|
||||
python-dateutil==2.8.0
|
||||
python-editor==1.0.4
|
||||
pytz==2018.9
|
||||
pytz==2019.1
|
||||
pyyaml==5.1
|
||||
raven[flask]==6.10.0
|
||||
redis==2.10.6
|
||||
redis==3.2.1
|
||||
requests-toolbelt==0.9.1
|
||||
requests[security]==2.21.0
|
||||
retrying==1.3.3
|
||||
|
|
@ -86,13 +86,18 @@ s3transfer==0.2.0
|
|||
six==1.12.0
|
||||
snowballstemmer==1.2.1 # via sphinx
|
||||
sphinx-rtd-theme==0.4.3
|
||||
sphinx==1.8.5
|
||||
sphinx==2.0.1
|
||||
sphinxcontrib-applehelp==1.0.1 # via sphinx
|
||||
sphinxcontrib-devhelp==1.0.1 # via sphinx
|
||||
sphinxcontrib-htmlhelp==1.0.1 # via sphinx
|
||||
sphinxcontrib-httpdomain==1.7.0
|
||||
sphinxcontrib-websupport==1.1.0 # via sphinx
|
||||
sphinxcontrib-jsmath==1.0.1 # via sphinx
|
||||
sphinxcontrib-qthelp==1.0.2 # via sphinx
|
||||
sphinxcontrib-serializinghtml==1.1.3 # via sphinx
|
||||
sqlalchemy-utils==0.33.11
|
||||
sqlalchemy==1.3.1
|
||||
sqlalchemy==1.3.2
|
||||
tabulate==0.8.3
|
||||
urllib3==1.24.1
|
||||
vine==1.3.0
|
||||
werkzeug==0.15.1
|
||||
werkzeug==0.15.2
|
||||
xmltodict==0.12.0
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography
|
|||
atomicwrites==1.3.0 # via pytest
|
||||
attrs==19.1.0 # via pytest
|
||||
aws-xray-sdk==0.95 # via moto
|
||||
boto3==1.9.120 # via moto
|
||||
boto3==1.9.130 # via moto
|
||||
boto==2.49.0 # via moto
|
||||
botocore==1.12.120 # via boto3, moto, s3transfer
|
||||
botocore==1.12.130 # via boto3, moto, s3transfer
|
||||
certifi==2019.3.9 # via requests
|
||||
cffi==1.12.2 # via cryptography
|
||||
chardet==3.0.4 # via requests
|
||||
|
|
@ -18,7 +18,7 @@ click==7.0 # via flask
|
|||
coverage==4.5.3
|
||||
cryptography==2.6.1 # via moto
|
||||
docker-pycreds==0.4.0 # via docker
|
||||
docker==3.7.1 # via moto
|
||||
docker==3.7.2 # via moto
|
||||
docutils==0.14 # via botocore
|
||||
ecdsa==0.13 # via python-jose
|
||||
factory-boy==2.11.1
|
||||
|
|
@ -28,13 +28,13 @@ freezegun==0.3.11
|
|||
future==0.17.1 # via python-jose
|
||||
idna==2.8 # via requests
|
||||
itsdangerous==1.1.0 # via flask
|
||||
jinja2==2.10 # via flask, moto
|
||||
jinja2==2.10.1 # via flask, moto
|
||||
jmespath==0.9.4 # via boto3, botocore
|
||||
jsondiff==1.1.1 # via moto
|
||||
jsonpickle==1.1 # via aws-xray-sdk
|
||||
markupsafe==1.1.1 # via jinja2
|
||||
mock==2.0.0 # via moto
|
||||
more-itertools==6.0.0 # via pytest
|
||||
more-itertools==7.0.0 # via pytest
|
||||
moto==1.3.7
|
||||
nose==1.3.7
|
||||
pbr==5.1.3 # via mock
|
||||
|
|
@ -42,14 +42,14 @@ pluggy==0.9.0 # via pytest
|
|||
py==1.8.0 # via pytest
|
||||
pyaml==18.11.0 # via moto
|
||||
pycparser==2.19 # via cffi
|
||||
pycryptodome==3.8.0 # via python-jose
|
||||
pycryptodome==3.8.1 # via python-jose
|
||||
pyflakes==2.1.1
|
||||
pytest-flask==0.14.0
|
||||
pytest-mock==1.10.2
|
||||
pytest==4.3.1
|
||||
pytest-mock==1.10.3
|
||||
pytest==4.4.0
|
||||
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
|
||||
python-jose==2.0.2 # via moto
|
||||
pytz==2018.9 # via moto
|
||||
pytz==2019.1 # via moto
|
||||
pyyaml==5.1
|
||||
requests-mock==1.5.2
|
||||
requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
|
||||
|
|
@ -59,6 +59,6 @@ six==1.12.0 # via cryptography, docker, docker-pycreds, faker, fre
|
|||
text-unidecode==1.2 # via faker
|
||||
urllib3==1.24.1 # via botocore, requests
|
||||
websocket-client==0.56.0 # via docker
|
||||
werkzeug==0.15.1 # via flask, moto, pytest-flask
|
||||
werkzeug==0.15.2 # via flask, moto, pytest-flask
|
||||
wrapt==1.11.1 # via aws-xray-sdk
|
||||
xmltodict==0.12.0 # via moto
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ gunicorn
|
|||
hvac # required for the vault destination plugin
|
||||
inflection
|
||||
jinja2
|
||||
kombu==4.3.0 # kombu 4.4.0 requires redis 3
|
||||
kombu
|
||||
lockfile
|
||||
marshmallow-sqlalchemy
|
||||
marshmallow
|
||||
|
|
@ -39,7 +39,7 @@ pyjwt
|
|||
pyOpenSSL
|
||||
python_ldap
|
||||
raven[flask]
|
||||
redis<3 # redis>=3 is not compatible with celery
|
||||
redis
|
||||
requests
|
||||
retrying
|
||||
six
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
#
|
||||
# pip-compile --output-file requirements.txt requirements.in -U --no-index
|
||||
#
|
||||
acme==0.32.0
|
||||
acme==0.33.1
|
||||
alembic-autogenerate-enums==0.0.2
|
||||
alembic==1.0.8 # via flask-migrate
|
||||
amqp==2.4.2 # via kombu
|
||||
|
|
@ -13,11 +13,11 @@ arrow==0.13.1
|
|||
asn1crypto==0.24.0 # via cryptography
|
||||
asyncpool==1.0
|
||||
bcrypt==3.1.6 # via flask-bcrypt, paramiko
|
||||
billiard==3.5.0.5 # via celery
|
||||
billiard==3.6.0.0 # via celery
|
||||
blinker==1.4 # via flask-mail, flask-principal, raven
|
||||
boto3==1.9.120
|
||||
botocore==1.12.120
|
||||
celery[redis]==4.2.2
|
||||
boto3==1.9.130
|
||||
botocore==1.12.130
|
||||
celery[redis]==4.3.0
|
||||
certifi==2019.3.9
|
||||
certsrv==2.1.1
|
||||
cffi==1.12.2 # via bcrypt, cryptography, pynacl
|
||||
|
|
@ -40,26 +40,26 @@ flask-sqlalchemy==2.3.2
|
|||
flask==1.0.2
|
||||
future==0.17.1
|
||||
gunicorn==19.9.0
|
||||
hvac==0.7.2
|
||||
hvac==0.8.2
|
||||
idna==2.8 # via requests
|
||||
inflection==0.3.1
|
||||
itsdangerous==1.1.0 # via flask
|
||||
jinja2==2.10
|
||||
jinja2==2.10.1
|
||||
jmespath==0.9.4 # via boto3, botocore
|
||||
josepy==1.1.0 # via acme
|
||||
jsonlines==1.2.0 # via cloudflare
|
||||
kombu==4.3.0
|
||||
kombu==4.5.0
|
||||
lockfile==0.12.2
|
||||
mako==1.0.8 # via alembic
|
||||
markupsafe==1.1.1 # via jinja2, mako
|
||||
marshmallow-sqlalchemy==0.16.1
|
||||
marshmallow==2.19.1
|
||||
marshmallow==2.19.2
|
||||
mock==2.0.0 # via acme
|
||||
ndg-httpsclient==0.5.1
|
||||
paramiko==2.4.2
|
||||
pbr==5.1.3 # via mock
|
||||
pem==19.1.0
|
||||
psycopg2==2.7.7
|
||||
psycopg2==2.8.1
|
||||
pyasn1-modules==0.2.4 # via python-ldap
|
||||
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
|
||||
pycparser==2.19 # via cffi
|
||||
|
|
@ -70,19 +70,19 @@ pyrfc3339==1.1 # via acme
|
|||
python-dateutil==2.8.0 # via alembic, arrow, botocore
|
||||
python-editor==1.0.4 # via alembic
|
||||
python-ldap==3.2.0
|
||||
pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339
|
||||
pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339
|
||||
pyyaml==5.1
|
||||
raven[flask]==6.10.0
|
||||
redis==2.10.6
|
||||
redis==3.2.1
|
||||
requests-toolbelt==0.9.1 # via acme
|
||||
requests[security]==2.21.0
|
||||
retrying==1.3.3
|
||||
s3transfer==0.2.0 # via boto3
|
||||
six==1.12.0
|
||||
sqlalchemy-utils==0.33.11
|
||||
sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
|
||||
sqlalchemy==1.3.2 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
|
||||
tabulate==0.8.3
|
||||
urllib3==1.24.1 # via botocore, requests
|
||||
vine==1.3.0 # via amqp
|
||||
werkzeug==0.15.1 # via flask
|
||||
vine==1.3.0 # via amqp, celery
|
||||
werkzeug==0.15.2 # via flask
|
||||
xmltodict==0.12.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue