Merge branch 'master' into hackday

This commit is contained in:
Curtis 2018-04-12 12:49:52 -07:00 committed by GitHub
commit 3ef550f738
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 257 additions and 103 deletions

View File

@ -110,10 +110,10 @@ ifndef VIRTUAL_ENV
endif
@echo "--> Updating Python requirements"
pip install --upgrade pip-tools
pip-compile --output-file requirements-docs.txt requirements-docs.in -U
pip-compile --output-file requirements-dev.txt requirements-dev.in -U
pip-compile --output-file requirements-tests.txt requirements-tests.in -U
pip-compile --output-file requirements.txt requirements.in -U
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
pip-compile --output-file requirements.txt requirements.in -U --no-index
@echo "--> Done updating Python requirements"
@echo "--> Installing new dependencies"
pip install -e .

View File

@ -65,6 +65,36 @@ Basic Configuration
SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
.. data:: SQLALCHEMY_POOL_SIZE
:noindex:
The default connection pool size is 5 for sqlalchemy managed connections. Depending on the number of Lemur instances,
please specify per instance connection pool size. Below is an example to set connection pool size to 10.
::
SQLALCHEMY_POOL_SIZE = 10
.. warning::
This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
.. data:: SQLALCHEMY_MAX_OVERFLOW
:noindex:
This setting allows to create connections in addition to specified number of connections in pool size. By default, sqlalchemy
allows 10 connections to create in addition to the pool size. This is also an optional setting. If `SQLALCHEMY_POOL_SIZE` and
`SQLALCHEMY_MAX_OVERFLOW` are not speficied then each Lemur instance may create maximum of 15 connections.
::
SQLALCHECK_MAX_OVERFLOW = 0
.. note::
Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size.
.. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
:noindex:

View File

@ -48,7 +48,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi
* pip
* virtualenv (ideally virtualenvwrapper)
* node.js (for npm and building css/javascript)
* (Optional) PostgreSQL
+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
Once you've got all that, the rest is simple:
@ -77,6 +77,7 @@ Create a default Lemur configuration just as if this were a production instance:
::
lemur create_config
lemur init
You'll likely want to make some changes to the default configuration (we recommend developing against Postgres, for example). Once done, migrate your database using the following command:

View File

@ -8,7 +8,8 @@
"""
from __future__ import absolute_import, division, print_function
import time
from flask import g, request
from lemur import factory
from lemur.extensions import metrics
@ -75,17 +76,6 @@ def configure_hook(app):
"""
from flask import jsonify
from werkzeug.exceptions import HTTPException
from lemur.decorators import crossdomain
if app.config.get('CORS'):
@app.after_request
@crossdomain(origin=u"http://localhost:3000", methods=['PUT', 'HEAD', 'GET', 'POST', 'OPTIONS', 'DELETE'])
def after(response):
return response
@app.after_request
def log_status(response):
metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
return response
@app.errorhandler(Exception)
def handle_error(e):
@ -95,3 +85,29 @@ def configure_hook(app):
app.logger.exception(e)
return jsonify(error=str(e)), code
@app.before_request
def before_request():
g.request_start_time = time.time()
@app.after_request
def after_request(response):
# Return early if we don't have the start time
if not hasattr(g, 'request_start_time'):
return response
# Get elapsed time in milliseconds
elapsed = time.time() - g.request_start_time
elapsed = int(round(1000 * elapsed))
# Collect request/response tags
tags = {
'endpoint': request.endpoint,
'request_method': request.method.lower(),
'status_code': response.status_code
}
# Record our response time metric
metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags)
metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
return response

View File

@ -14,10 +14,11 @@ from sqlalchemy import and_, func
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives.asymmetric import rsa, ec
from flask_restful.reqparse import RequestParser
from lemur.constants import CERTIFICATE_KEY_TYPES
from lemur.exceptions import InvalidConfiguration
paginated_parser = RequestParser()
@ -78,17 +79,43 @@ def generate_private_key(key_type):
"""
Generates a new private key based on key_type.
Valid key types: RSA2048, RSA4096
Valid key types: RSA2048, RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1',
'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1',
'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2'
:param key_type:
:return:
"""
valid_key_types = ['RSA2048', 'RSA4096']
if key_type not in valid_key_types:
_CURVE_TYPES = {
"ECCPRIME192V1": ec.SECP192R1(),
"ECCPRIME256V1": ec.SECP256R1(),
"ECCSECP192R1": ec.SECP192R1(),
"ECCSECP224R1": ec.SECP224R1(),
"ECCSECP256R1": ec.SECP256R1(),
"ECCSECP384R1": ec.SECP384R1(),
"ECCSECP521R1": ec.SECP521R1(),
"ECCSECP256K1": ec.SECP256K1(),
"ECCSECT163K1": ec.SECT163K1(),
"ECCSECT233K1": ec.SECT233K1(),
"ECCSECT283K1": ec.SECT283K1(),
"ECCSECT409K1": ec.SECT409K1(),
"ECCSECT571K1": ec.SECT571K1(),
"ECCSECT163R2": ec.SECT163R2(),
"ECCSECT233R1": ec.SECT233R1(),
"ECCSECT283R1": ec.SECT283R1(),
"ECCSECT409R1": ec.SECT409R1(),
"ECCSECT571R2": ec.SECT571R1(),
}
if key_type not in CERTIFICATE_KEY_TYPES:
raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format(
key_type=key_type,
choices=",".join(valid_key_types)
choices=",".join(CERTIFICATE_KEY_TYPES)
))
if 'RSA' in key_type:
@ -98,6 +125,11 @@ def generate_private_key(key_type):
key_size=key_size,
backend=default_backend()
)
elif 'ECC' in key_type:
return ec.generate_private_key(
curve=_CURVE_TYPES[key_type],
backend=default_backend()
)
def is_weekend(date):

View File

@ -9,3 +9,26 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
SUCCESS_METRIC_STATUS = 'success'
FAILURE_METRIC_STATUS = 'failure'
CERTIFICATE_KEY_TYPES = [
'RSA2048',
'RSA4096',
'ECCPRIME192V1',
'ECCPRIME256V1',
'ECCSECP192R1',
'ECCSECP224R1',
'ECCSECP256R1',
'ECCSECP384R1',
'ECCSECP521R1',
'ECCSECP256K1',
'ECCSECT163K1',
'ECCSECT233K1',
'ECCSECT283K1',
'ECCSECT409K1',
'ECCSECT571K1',
'ECCSECT163R2',
'ECCSECT233R1',
'ECCSECT283R1',
'ECCSECT409R1',
'ECCSECT571R2'
]

View File

@ -1,56 +0,0 @@
"""
.. module: lemur.decorators
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
"""
from builtins import str
from datetime import timedelta
from flask import make_response, request, current_app
from functools import update_wrapper
# this is only used for dev
def crossdomain(origin=None, methods=None, headers=None,
max_age=21600, attach_to_all=True,
automatic_options=True): # pragma: no cover
if methods is not None:
methods = ', '.join(sorted(x.upper() for x in methods))
if headers is not None and not isinstance(headers, str):
headers = ', '.join(x.upper() for x in headers)
if not isinstance(origin, str):
origin = ', '.join(origin)
if isinstance(max_age, timedelta):
max_age = max_age.total_seconds()
def get_methods():
if methods is not None:
return methods
options_resp = current_app.make_default_options_response()
return options_resp.headers['allow']
def decorator(f):
def wrapped_function(*args, **kwargs):
if automatic_options and request.method == 'OPTIONS':
resp = current_app.make_default_options_response()
else:
resp = make_response(f(*args, **kwargs))
if not attach_to_all and request.method != 'OPTIONS':
return resp
h = resp.headers
h['Access-Control-Allow-Origin'] = origin
h['Access-Control-Allow-Methods'] = get_methods()
h['Access-Control-Max-Age'] = str(max_age)
h['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept, Authorization "
h['Access-Control-Allow-Credentials'] = 'true'
return resp
f.provide_automatic_options = False
return update_wrapper(wrapped_function, f)
return decorator

View File

@ -26,3 +26,6 @@ sentry = Sentry()
from blinker import Namespace
signals = Namespace()
from flask_cors import CORS
cors = CORS()

View File

@ -21,7 +21,7 @@ from flask import Flask
from lemur.certificates.hooks import activate_debug_dump
from lemur.common.health import mod as health
from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry
from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors
DEFAULT_BLUEPRINTS = (
@ -124,6 +124,8 @@ def configure_extensions(app):
smtp_mail.init_app(app)
metrics.init_app(app)
sentry.init_app(app)
app.config['CORS_HEADERS'] = 'Content-Type'
cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True)
def configure_blueprints(app, blueprints):

View File

@ -157,7 +157,7 @@ def map_cis_fields(options, csr):
"csr": csr,
"signature_hash": signature_hash(options.get('signing_algorithm')),
"validity": {
"valid_to": options['validity_end'].format('YYYY-MM-DD')
"valid_to": options['validity_end'].format('YYYY-MM-DDTHH:MM:SSZ')
},
"organization": {
"name": options['organization'],
@ -491,6 +491,11 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
self.session.headers.pop('Accept')
end_entity = pem.parse(certificate_pem)[0]
if 'ECC' in issuer_options['key_type']:
return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_ECC_CIS_INTERMEDIATE'), data['id']
# By default return RSA
return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_CIS_INTERMEDIATE'), data['id']
def revoke_certificate(self, certificate, comments):

View File

@ -103,7 +103,7 @@ def test_map_cis_fields(app):
'signature_hash': 'sha256',
'organization': {'name': 'Example, Inc.', 'units': ['Example Org']},
'validity': {
'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DD')
'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DDTHH:MM:SSZ')
},
'profile_name': None
}
@ -132,7 +132,7 @@ def test_map_cis_fields(app):
'signature_hash': 'sha256',
'organization': {'name': 'Example, Inc.', 'units': ['Example Org']},
'validity': {
'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DD')
'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DDTHH:MM:SSZ')
},
'profile_name': None
}

View File

@ -0,0 +1 @@
datadog==0.14.0

View File

@ -0,0 +1,4 @@
try:
VERSION = __import__('pkg_resources').get_distribution(__name__).version
except Exception as e:
VERSION = 'Unknown'

View File

@ -0,0 +1,45 @@
import lemur_statsd as plug
from flask import current_app
from lemur.plugins.bases.metric import MetricPlugin
from datadog import DogStatsd
class StatsdMetricPlugin(MetricPlugin):
title = 'Statsd'
slug = 'statsd-metrics'
description = 'Adds support for sending metrics to Statsd'
version = plug.VERSION
def __init__(self):
host = current_app.config.get('STATSD_HOST')
port = current_app.config.get('STATSD_PORT')
prefix = current_app.config.get('STATSD_PREFIX')
self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None):
valid_types = ['COUNTER', 'GAUGE', 'TIMER']
tags = []
if metric_type.upper() not in valid_types:
raise Exception(
"Invalid Metric Type for Statsd, '{metric}' choose from: {options}".format(
metric=metric_type, options=','.join(valid_types)
)
)
if metric_tags:
if not isinstance(metric_tags, dict):
raise Exception("Invalid Metric Tags for Statsd: Tags must be in dict format")
else:
tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items())
if metric_type.upper() == 'COUNTER':
self.statsd.increment(metric_name, metric_value, tags)
elif metric_type.upper() == 'GAUGE':
self.statsd.gauge(metric_name, metric_value, tags)
elif metric_type.upper() == 'TIMER':
self.statsd.timing(metric_name, metric_value, tags)
return

View File

@ -0,0 +1,24 @@
"""Basic package information"""
from __future__ import absolute_import
from setuptools import setup, find_packages
install_requires = [
'lemur',
'datadog'
]
setup(
name='lemur_statsd',
version='1.0.0',
author='Cloudflare Security Engineering',
author_email='',
include_package_data=True,
packages=find_packages(),
zip_safe=False,
install_requires=install_requires,
entry_points={
'lemur.plugins': [
'statsd = lemur_statsd.plugin:StatsdMetricPlugin',
]
}
)

View File

@ -20,7 +20,8 @@
Key Type
</label>
<div class="col-sm-10">
<select class="form-control" ng-model="authority.keyType" ng-options="option for option in ['RSA2048', 'RSA4096']" ng-init="authority.keyType = 'RSA2048'"></select>
<select class="form-control" ng-model="authority.keyType" ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']" ng-init="authority.keyType = 'RSA2048'"></select>
</div>
</div>
<div ng-show="authority.sensitivity == 'high'" class="form-group">

View File

@ -32,7 +32,10 @@
</label>
<div class="col-sm-10">
<select class="form-control" ng-model="certificate.keyType"
ng-options="option for option in ['RSA2048', 'RSA4096']"
ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1',
'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1',
'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']"
ng-init="certificate.keyType = 'RSA2048'"></select>
</div>
</div>

View File

@ -6,9 +6,27 @@ def test_generate_private_key():
assert generate_private_key('RSA2048')
assert generate_private_key('RSA4096')
assert generate_private_key('ECCPRIME192V1')
assert generate_private_key('ECCPRIME256V1')
assert generate_private_key('ECCSECP192R1')
assert generate_private_key('ECCSECP224R1')
assert generate_private_key('ECCSECP256R1')
assert generate_private_key('ECCSECP384R1')
assert generate_private_key('ECCSECP521R1')
assert generate_private_key('ECCSECP256K1')
assert generate_private_key('ECCSECT163K1')
assert generate_private_key('ECCSECT233K1')
assert generate_private_key('ECCSECT283K1')
assert generate_private_key('ECCSECT409K1')
assert generate_private_key('ECCSECT571K1')
assert generate_private_key('ECCSECT163R2')
assert generate_private_key('ECCSECT233R1')
assert generate_private_key('ECCSECT283R1')
assert generate_private_key('ECCSECT409R1')
assert generate_private_key('ECCSECT571R2')
with pytest.raises(Exception):
generate_private_key('ECC')
generate_private_key('LEMUR')
def test_get_authority_key():

View File

@ -2,15 +2,15 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements-dev.txt requirements-dev.in
# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
#
aspy.yaml==1.0.0 # via pre-commit
aspy.yaml==1.1.0 # via pre-commit
cached-property==1.4.2 # via pre-commit
certifi==2018.1.18 # via requests
cfgv==1.0.0 # via pre-commit
chardet==3.0.4 # via requests
flake8==3.5.0
identify==1.0.9 # via pre-commit
identify==1.0.11 # via pre-commit
idna==2.6 # via requests
invoke==0.22.1
mccabe==0.6.1 # via flake8
@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit
requests-toolbelt==0.8.0 # via twine
requests==2.18.4 # via requests-toolbelt, twine
six==1.11.0 # via cfgv, pre-commit
tqdm==4.21.0 # via twine
tqdm==4.22.0 # via twine
twine==1.11.0
urllib3==1.22 # via requests
virtualenv==15.2.0 # via pre-commit

View File

@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements-docs.txt requirements-docs.in
# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
#
alabaster==0.7.10 # via sphinx
babel==2.5.3 # via sphinx
@ -16,7 +16,7 @@ markupsafe==1.0 # via jinja2
packaging==17.1 # via sphinx
pygments==2.2.0 # via sphinx
pyparsing==2.2.0 # via packaging
pytz==2018.3 # via babel
pytz==2018.4 # via babel
requests==2.18.4 # via sphinx
six==1.11.0 # via packaging, sphinx, sphinxcontrib-httpdomain
snowballstemmer==1.2.1 # via sphinx

View File

@ -2,14 +2,14 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements-tests.txt requirements-tests.in
# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in
#
asn1crypto==0.24.0 # via cryptography
attrs==17.4.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.7.2 # via moto
boto3==1.7.4 # via moto
boto==2.48.0 # via moto
botocore==1.10.2 # via boto3, moto, s3transfer
botocore==1.10.4 # via boto3, moto, s3transfer
certifi==2018.1.18 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
@ -21,7 +21,7 @@ docker-pycreds==0.2.2 # via docker
docker==3.2.1 # via moto
docutils==0.14 # via botocore
factory-boy==2.10.0
faker==0.8.12
faker==0.8.13
flask==0.12.2 # via pytest-flask
freezegun==0.3.10
idna==2.6 # via cryptography, requests
@ -35,17 +35,17 @@ mock==2.0.0 # via moto
more-itertools==4.1.0 # via pytest
moto==1.3.1
nose==1.3.7
pbr==4.0.1 # via mock
pbr==4.0.2 # via mock
pluggy==0.6.0 # via pytest
py==1.5.3 # via pytest
pyaml==17.12.1 # via moto
pycparser==2.18 # via cffi
pyflakes==1.6.0
pytest-flask==0.10.0
pytest-mock==1.8.0
pytest-mock==1.9.0
pytest==3.5.0
python-dateutil==2.6.1 # via botocore, faker, freezegun, moto
pytz==2018.3 # via moto
pytz==2018.4 # via moto
pyyaml==3.12 # via pyaml
requests-mock==1.4.0
requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock

View File

@ -11,6 +11,7 @@ Flask-RESTful==0.3.6
Flask-Script==2.0.6
Flask-SQLAlchemy
Flask==0.12
Flask-Cors
future
gunicorn
inflection

View File

@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements.txt requirements.in
# pip-compile --no-index --output-file requirements.txt requirements.in
#
acme==0.23.0
alembic-autogenerate-enums==0.0.2
@ -12,13 +12,14 @@ arrow==0.12.1
asn1crypto==0.24.0 # via cryptography
bcrypt==3.1.4 # via flask-bcrypt, paramiko
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.7.2
botocore==1.10.2 # via boto3, s3transfer
boto3==1.7.4
botocore==1.10.4 # via boto3, s3transfer
cffi==1.11.5 # via bcrypt, cryptography, pynacl
click==6.7 # via flask
cryptography==2.2.2
docutils==0.14 # via botocore
flask-bcrypt==0.7.1
flask-cors==3.0.3
flask-mail==0.9.1
flask-migrate==2.1.1
flask-principal==0.4.0
@ -42,7 +43,7 @@ marshmallow==2.15.0
mock==2.0.0 # via acme
ndg-httpsclient==0.4.4
paramiko==2.4.1
pbr==4.0.1 # via mock
pbr==4.0.2 # via mock
pem==17.1.0
psycopg2==2.7.4
pyasn1-modules==0.2.1 # via python-ldap
@ -55,7 +56,7 @@ pyrfc3339==1.0 # via acme
python-dateutil==2.6.1 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic
python-ldap==3.0.0
pytz==2018.3 # via acme, flask-restful, pyrfc3339
pytz==2018.4 # via acme, flask-restful, pyrfc3339
raven[flask]==6.6.0
requests[security]==2.11.1
retrying==1.3.3