Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into get_by_attributes

Browse Source
This commit is contained in:
Hossein Shafagh 2019-02-01 16:48:50 -08:00 committed by GitHub
commit 8e93d007be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
63 changed files with 1232 additions and 422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -28,6 +28,7 @@ env:
before_script:
- psql -c "create database lemur;" -U postgres
- psql -c "create user lemur with password 'lemur;'" -U postgres
- psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
- npm config set registry https://registry.npmjs.org
- npm install -g bower
- pip install --upgrade setuptools
@ -45,4 +46,4 @@ after_success:
notifications:
email:
kglisson@netflix.com
ccastrapel@netflix.com

4
Makefile
View File

@ -43,6 +43,8 @@ reset-db:
dropdb lemur || true
@echo "--> Creating 'lemur' database"
createdb -E utf-8 lemur
@echo "--> Enabling pg_trgm extension"
psql lemur -c "create extension IF NOT EXISTS pg_trgm;"
@echo "--> Applying migrations"
lemur db upgrade
@ -111,10 +113,10 @@ endif
@echo "--> Updating Python requirements"
pip install --upgrade pip
pip install --upgrade pip-tools
pip-compile --output-file requirements.txt requirements.in -U --no-index
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
pip-compile --output-file requirements.txt requirements.in -U --no-index
@echo "--> Done updating Python requirements"
@echo "--> Removing python-ldap from requirements-docs.txt"
grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

64
docker/Dockerfile Normal file
View File

@ -0,0 +1,64 @@
FROM alpine:3.8
ARG VERSION
ENV VERSION master
ENV uid 1337
ENV gid 1337
ENV user lemur
ENV group lemur
COPY entrypoint /
COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
COPY supervisor.conf /
COPY nginx/default.conf /etc/nginx/conf.d/
COPY nginx/default-ssl.conf /etc/nginx/conf.d/
RUN addgroup -S ${group} -g ${gid} && \
adduser -D -S ${user} -G ${group} -u ${uid} && \
apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
apk --update add --virtual build-dependencies \
git \
tar \
curl \
python3-dev \
npm \
bash \
musl-dev \
gcc \
autoconf \
automake \
make \
nasm \
zlib-dev \
postgresql-dev \
libressl-dev \
libffi-dev \
cyrus-sasl-dev \
openldap-dev && \
mkdir -p /opt/lemur /home/lemur/.lemur/ && \
curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
pip3 install --upgrade pip && \
pip3 install --upgrade setuptools && \
chmod +x /entrypoint && \
mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
WORKDIR /opt/lemur
RUN npm install --unsafe-perm && \
pip3 install -e . && \
node_modules/.bin/gulp build && \
node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
apk del build-dependencies
WORKDIR /
HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \
CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
USER root
ENTRYPOINT ["/entrypoint"]
CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

54
docker/entrypoint Normal file
View File

@ -0,0 +1,54 @@
#!/bin/sh
if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
echo "Database vars not set"
exit 1
fi
export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
source /etc/profile
PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
echo " # Create Postgres trgm extension"
PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
echo " # Done"
if [ -z "${SKIP_SSL}" ]; then
if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
fi
mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
fi
# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
# echo "Creating config"
# https://github.com/Netflix/lemur/issues/2257
# python3 /opt/lemur/lemur/manage.py create_config
# echo "Done"
# fi
echo " # Running init"
su lemur -c "python3 /opt/lemur/lemur/manage.py init"
echo " # Done"
# echo "Creating user"
# https://github.com/Netflix/lemur/issues/
# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
# echo "Done"
cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
echo " # Populating crontab"
echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
echo " # Done"
exec "$@"

37
docker/nginx/default-ssl.conf Normal file
View File

@ -0,0 +1,37 @@
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
server {
listen 80;
server_name _;
return 301 https://$host$request_uri;
}
server {
listen 443;
server_name _;
access_log /dev/stdout;
error_log /dev/stderr;
ssl_certificate /etc/nginx/ssl/server.crt;
ssl_certificate_key /etc/nginx/ssl/server.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
location /api {
proxy_pass http://127.0.0.1:8000;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
root /opt/lemur/lemur/static/dist;
include mime.types;
index index.html;
}
}

26
docker/nginx/default.conf Normal file
View File

@ -0,0 +1,26 @@
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
server {
listen 80;
access_log /dev/stdout;
error_log /dev/stderr;
location /api {
proxy_pass http://127.0.0.1:8000;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
root /opt/lemur/lemur/static/dist;
include mime.types;
index index.html;
}
}

31
docker/src/lemur.conf.py Normal file
View File

@ -0,0 +1,31 @@
import os
_basedir = os.path.abspath(os.path.dirname(__file__))
CORS = os.environ.get("CORS") == "True"
debug = os.environ.get("DEBUG") == "True"
SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw=='))
LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ=='))
LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo='))
LEMUR_WHITELISTED_DOMAINS = []
LEMUR_EMAIL = ''
LEMUR_SECURITY_TEAM_EMAIL = []
LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
ACTIVE_PROVIDERS = []
METRIC_PROVIDERS = []
LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')

32
docker/supervisor.conf Normal file
View File

@ -0,0 +1,32 @@
[supervisord]
nodaemon=true
user=root
logfile=/dev/stdout
logfile_maxbytes=0
pidfile = /tmp/supervisord.pid
[program:lemur]
environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
user=lemur
directory=/opt/lemur/lemur
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:nginx]
command=/usr/sbin/nginx -g "daemon off;"
user=root
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:cron]
command=/usr/sbin/crond -f
user=root
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0

12
docs/administration.rst
View File

@ -324,6 +324,7 @@ Here is an example LDAP configuration stanza you can add to your config. Adjust
LDAP_CACERT_FILE = '/opt/lemur/trusted.pem'
LDAP_REQUIRED_GROUP = 'certificate-management-access'
LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'}
LDAP_IS_ACTIVE_DIRECTORY = True
The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN.
@ -406,6 +407,17 @@ The following LDAP options are not required, however TLS is always recommended.
LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'}
.. data:: LDAP_IS_ACTIVE_DIRECTORY
:noindex:
When set to True, nested group memberships are supported, by searching for groups with the member:1.2.840.113556.1.4.1941 attribute set to the user DN.
When set to False, the list of groups will be determined by the 'memberof' attribute of the LDAP user logging in.
::
LDAP_IS_ACTIVE_DIRECTORY = False
Authentication Providers
~~~~~~~~~~~~~~~~~~~~~~~~

10
docs/quickstart/index.rst
View File

@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur:
Installing Build Dependencies
-----------------------------
If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
.. code-block:: bash
@ -117,7 +117,7 @@ Simply run:
.. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment.
You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment.
Update your configuration
@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config
LEMUR_DEFAULT_ORGANIZATION
LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
Setup Postgres
Set Up Postgres
--------------
For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API. See :ref:`Creati
.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
Setup a Reverse Proxy
Set Up a Reverse Proxy
---------------------
By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
Proxying with Nginx

28
lemur/auth/ldap.py
View File

@ -41,6 +41,7 @@ class LdapPrincipal():
self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
self.ldap_is_active_directory = current_app.config.get("LDAP_IS_ACTIVE_DIRECTORY", False)
self.ldap_attrs = ['memberOf']
self.ldap_client = None
self.ldap_groups = None
@ -168,11 +169,28 @@ class LdapPrincipal():
except ldap.LDAPError as e:
raise Exception("ldap error: {0}".format(e))
lgroups = self.ldap_client.search_s(self.ldap_base_dn,
ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
# lgroups is a list of utf-8 encoded strings
# convert to a single string of groups to allow matching
self.ldap_groups = b''.join(lgroups).decode('ascii')
if self.ldap_is_active_directory:
# Lookup user DN, needed to search for group membership
userdn = self.ldap_client.search_s(self.ldap_base_dn,
ldap.SCOPE_SUBTREE, ldap_filter,
['distinguishedName'])[0][1]['distinguishedName'][0]
userdn = userdn.decode('utf-8')
# Search all groups that have the userDN as a member
groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn)
lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn'])
# Create a list of group CN's from the result
self.ldap_groups = []
for group in lgroups:
(dn, values) = group
self.ldap_groups.append(values['cn'][0].decode('ascii'))
else:
lgroups = self.ldap_client.search_s(self.ldap_base_dn,
ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
# lgroups is a list of utf-8 encoded strings
# convert to a single string of groups to allow matching
self.ldap_groups = b''.join(lgroups).decode('ascii')
self.ldap_client.unbind()
def _ldap_validate_conf(self):

8
lemur/authorities/service.py
View File

@ -15,6 +15,7 @@ from lemur import database
from lemur.common.utils import truthiness
from lemur.extensions import metrics
from lemur.authorities.models import Authority
from lemur.certificates.models import Certificate
from lemur.roles import service as role_service
from lemur.certificates.service import upload
@ -178,6 +179,13 @@ def render(args):
terms = filt.split(';')
if 'active' in filt:
query = query.filter(Authority.active == truthiness(terms[1]))
elif 'cn' in filt:
term = '%{0}%'.format(terms[1])
sub_query = database.session_query(Certificate.root_authority_id) \
.filter(Certificate.cn.ilike(term)) \
.subquery()
query = query.filter(Authority.id.in_(sub_query))
else:
query = database.filter(query, Authority, terms)

40
lemur/certificates/cli.py
View File

@ -265,30 +265,31 @@ def query(fqdns, issuer, owner, expired):
table = []
q = database.session_query(Certificate)
if issuer:
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike('%{0}%'.format(issuer))) \
.subquery()
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike('%{0}%'.format(issuer))) \
.subquery()
q = q.filter(
or_(
Certificate.issuer.ilike('%{0}%'.format(issuer)),
Certificate.authority_id.in_(sub_query)
q = q.filter(
or_(
Certificate.issuer.ilike('%{0}%'.format(issuer)),
Certificate.authority_id.in_(sub_query)
)
)
)
q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
if owner:
q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
if not expired:
q = q.filter(Certificate.expired == False) # noqa
for f in fqdns.split(','):
q = q.filter(
or_(
Certificate.cn.ilike('%{0}%'.format(f)),
Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f)))
if fqdns:
for f in fqdns.split(','):
q = q.filter(
or_(
Certificate.cn.ilike('%{0}%'.format(f)),
Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f)))
)
)
)
for c in q.all():
table.append([c.id, c.name, c.owner, c.issuer])
@ -363,10 +364,7 @@ def check_revoked():
else:
status = verify_string(cert.body, "")
if status is None:
cert.status = 'unknown'
else:
cert.status = 'valid' if status else 'revoked'
cert.status = 'valid' if status else 'revoked'
except Exception as e:
sentry.captureException()

29
lemur/certificates/models.py
View File

@ -19,7 +19,7 @@ from sqlalchemy.sql.expression import case, extract
from sqlalchemy_utils.types.arrow import ArrowType
from werkzeug.utils import cached_property
from lemur.common import defaults, utils
from lemur.common import defaults, utils, validators
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.database import db
from lemur.domains.models import Domain
@ -77,6 +77,14 @@ def get_or_increase_name(name, serial):
class Certificate(db.Model):
__tablename__ = 'certificates'
__table_args__ = (
Index('ix_certificates_cn', "cn",
postgresql_ops={"cn": "gin_trgm_ops"},
postgresql_using='gin'),
Index('ix_certificates_name', "name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
)
id = Column(Integer, primary_key=True)
ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
external_id = Column(String(128))
@ -130,7 +138,6 @@ class Certificate(db.Model):
logs = relationship('Log', backref='certificate')
endpoints = relationship('Endpoint', backref='certificate')
rotation_policy = relationship("RotationPolicy")
sensitive_fields = ('private_key',)
def __init__(self, **kwargs):
@ -179,6 +186,18 @@ class Certificate(db.Model):
for domain in defaults.domains(cert):
self.domains.append(Domain(name=domain))
# Check integrity before saving anything into the database.
# For user-facing API calls, validation should also be done in schema validators.
self.check_integrity()
def check_integrity(self):
"""
Integrity checks: Does the cert have a matching private key?
"""
if self.private_key:
validators.verify_private_key_match(utils.parse_private_key(self.private_key), self.parsed_cert,
error_class=AssertionError)
@cached_property
def parsed_cert(self):
assert self.body, "Certificate body not set"
@ -208,6 +227,10 @@ class Certificate(db.Model):
def location(self):
return defaults.location(self.parsed_cert)
@property
def distinguished_name(self):
return self.parsed_cert.subject.rfc4514_string()
@property
def key_type(self):
if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey):
@ -359,7 +382,7 @@ def update_destinations(target, value, initiator):
destination_plugin = plugins.get(value.plugin_name)
status = FAILURE_METRIC_STATUS
try:
if target.private_key:
if target.private_key or not destination_plugin.requires_key:
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
status = SUCCESS_METRIC_STATUS
except Exception as e:

27
lemur/certificates/schemas.py
View File

@ -10,7 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.common import validators, missing
from lemur.common import missing, utils, validators
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.constants import CERTIFICATE_KEY_TYPES
@ -206,6 +206,7 @@ class CertificateOutputSchema(LemurOutputSchema):
cn = fields.String()
common_name = fields.String(attribute='cn')
distinguished_name = fields.String()
not_after = fields.DateTime()
validity_end = ArrowDateTime(attribute='not_after')
@ -242,8 +243,8 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
authority = fields.Nested(AssociatedAuthoritySchema, required=False)
notify = fields.Boolean(missing=True)
external_id = fields.String(missing=None, allow_none=True)
private_key = fields.String(validate=validators.private_key)
body = fields.String(required=True, validate=validators.public_certificate)
private_key = fields.String()
body = fields.String(required=True)
chain = fields.String(validate=validators.public_certificate, missing=None,
allow_none=True) # TODO this could be multiple certificates
@ -258,6 +259,26 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
if not data.get('private_key'):
raise ValidationError('Destinations require private key.')
@validates_schema
def validate_cert_private_key(self, data):
cert = None
key = None
if data.get('body'):
try:
cert = utils.parse_certificate(data['body'])
except ValueError:
raise ValidationError("Public certificate presented is not valid.", field_names=['body'])
if data.get('private_key'):
try:
key = utils.parse_private_key(data['private_key'])
except ValueError:
raise ValidationError("Private key presented is not valid.", field_names=['private_key'])
if cert and key:
# Throws ValidationError
validators.verify_private_key_match(key, cert)
class CertificateExportInputSchema(LemurInputSchema):
plugin = fields.Nested(PluginInputSchema)

26
lemur/certificates/service.py
View File

@ -237,11 +237,6 @@ def upload(**kwargs):
else:
kwargs['roles'] = roles
if kwargs.get('private_key'):
private_key = kwargs['private_key']
if not isinstance(private_key, bytes):
kwargs['private_key'] = private_key.encode('utf-8')
cert = Certificate(**kwargs)
cert.authority = kwargs.get('authority')
cert = database.create(cert)
@ -291,6 +286,14 @@ def create(**kwargs):
certificate_issued.send(certificate=cert, authority=cert.authority)
metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
if isinstance(cert, PendingCertificate):
# We need to refresh the pending certificate to avoid "Instance is not bound to a Session; "
# "attribute refresh operation cannot proceed"
pending_cert = database.session_query(PendingCertificate).get(cert.id)
from lemur.common.celery import fetch_acme_cert
if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False):
fetch_acme_cert.apply_async((pending_cert.id,), countdown=5)
return cert
@ -314,7 +317,7 @@ def render(args):
if filt:
terms = filt.split(';')
term = '%{0}%'.format(terms[1])
term = '{0}%'.format(terms[1])
# Exact matches for quotes. Only applies to name, issuer, and cn
if terms[1].startswith('"') and terms[1].endswith('"'):
term = terms[1][1:-1]
@ -378,7 +381,8 @@ def render(args):
now = arrow.now().format('YYYY-MM-DD')
query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
return database.sort_and_page(query, Certificate, args)
result = database.sort_and_page(query, Certificate, args)
return result
def create_csr(**csr_config):
@ -439,10 +443,7 @@ def create_csr(**csr_config):
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL, # would like to use PKCS8 but AWS ELBs don't like it
encryption_algorithm=serialization.NoEncryption()
)
if isinstance(private_key, bytes):
private_key = private_key.decode('utf-8')
).decode('utf-8')
csr = request.public_bytes(
encoding=serialization.Encoding.PEM
@ -554,6 +555,9 @@ def reissue_certificate(certificate, replace=None, user=None):
"""
primitives = get_certificate_primitives(certificate)
if primitives.get("csr"):
# We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation.
del primitives["csr"]
if not user:
primitives['creator'] = certificate.user

52
lemur/common/celery.py
View File

@ -19,14 +19,17 @@ from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.sources.cli import clean, validate_sources
from lemur.sources.cli import clean, sync, validate_sources
flask_app = create_app()
if current_app:
flask_app = current_app
else:
flask_app = create_app()
def make_celery(app):
celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
broker=app.config.get('CELERY_BROKER_URL'))
celery.conf.update(app.config)
TaskBase = celery.Task
@ -53,8 +56,10 @@ def fetch_acme_cert(id):
id: an id of a PendingCertificate
"""
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
"message": "Resolving pending certificate {}".format(id)
}
current_app.logger.debug(log_data)
pending_certs = pending_certificate_service.get_pending_certs([id])
new = 0
failed = 0
@ -138,12 +143,22 @@ def fetch_all_pending_acme_certs():
"""Instantiate celery workers to resolve all pending Acme certificates"""
pending_certs = pending_certificate_service.get_unresolved_pending_certs()
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
"message": "Starting job."
}
current_app.logger.debug(log_data)
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
if cert.last_updated == cert.date_created or datetime.now(
timezone.utc) - cert.last_updated > timedelta(minutes=5):
if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
log_data["message"] = "Triggering job for cert {}".format(cert.name)
log_data["cert_name"] = cert.name
log_data["cert_id"] = cert.id
current_app.logger.debug(log_data)
fetch_acme_cert.delay(cert.id)
@ -188,3 +203,26 @@ def clean_source(source):
"""
current_app.logger.debug("Cleaning source {}".format(source))
clean([source], True)
@celery.task()
def sync_all_sources():
"""
This function will sync certificates from all sources. This function triggers one celery task per source.
"""
sources = validate_sources("all")
for source in sources:
current_app.logger.debug("Creating celery task to sync source {}".format(source.label))
sync_source.delay(source.label)
@celery.task()
def sync_source(source):
"""
This celery task will sync the specified source.
:param source:
:return:
"""
current_app.logger.debug("Syncing source {}".format(source))
sync([source])

38
lemur/common/defaults.py
View File

@ -7,18 +7,21 @@ from lemur.extensions import sentry
from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
def text_to_slug(value):
"""Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters."""
def text_to_slug(value, joiner='-'):
"""
Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.
A series of non-alphanumeric characters is replaced with the joiner character.
"""
# Strip all character accents: decompose Unicode characters and then drop combining chars.
value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c))
# Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash.
# Except, keep 'xn--' used in IDNA domain names as is.
value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', '-', value)
# Replace all remaining non-alphanumeric characters with joiner string. Multiple characters get collapsed into a
# single joiner. Except, keep 'xn--' used in IDNA domain names as is.
value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', joiner, value)
# '-' in the beginning or end of string looks ugly.
return value.strip('-')
return value.strip(joiner)
def certificate_name(common_name, issuer, not_before, not_after, san):
@ -224,25 +227,20 @@ def bitstrength(cert):
def issuer(cert):
"""
Gets a sane issuer name from a given certificate.
Gets a sane issuer slug from a given certificate, stripping non-alphanumeric characters.
:param cert:
:return: Issuer
:return: Issuer slug
"""
delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
try:
# Try organization name or fall back to CN
issuer = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME))
issuer = str(issuer[0].value)
for c in delchars:
issuer = issuer.replace(c, "")
return issuer
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get issuer! {0}".format(e))
# Try Common Name or fall back to Organization name
attrs = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME))
if not attrs:
current_app.logger.error("Unable to get issuer! Cert serial {:x}".format(cert.serial_number))
return "Unknown"
return text_to_slug(attrs[0].value, '')
def not_before(cert):
"""

1
lemur/common/fields.py
View File

@ -350,6 +350,7 @@ class SubjectAlternativeNameExtension(Field):
value = value.dotted_string
else:
current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name))
continue
general_names.append({'nameType': name_type, 'value': value})

1
lemur/common/missing.py
View File

@ -16,6 +16,7 @@ def convert_validity_years(data):
data['validity_start'] = now.isoformat()
end = now.replace(years=+int(data['validity_years']))
if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
if is_weekend(end):
end = end.replace(days=-2)

35
lemur/common/utils.py
View File

@ -12,7 +12,9 @@ import string
import sqlalchemy
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import rsa, ec
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from flask_restful.reqparse import RequestParser
from sqlalchemy import and_, func
@ -46,10 +48,22 @@ def parse_certificate(body):
:param body:
:return:
"""
if isinstance(body, str):
body = body.encode('utf-8')
assert isinstance(body, str)
return x509.load_pem_x509_certificate(body, default_backend())
return x509.load_pem_x509_certificate(body.encode('utf-8'), default_backend())
def parse_private_key(private_key):
"""
Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm).
Raises ValueError for an invalid string. Raises AssertionError when passed value is not str-type.
:param private_key: String containing PEM private key
"""
assert isinstance(private_key, str)
return load_pem_private_key(private_key.encode('utf8'), password=None, backend=default_backend())
def parse_csr(csr):
@ -59,10 +73,9 @@ def parse_csr(csr):
:param csr:
:return:
"""
if isinstance(csr, str):
csr = csr.encode('utf-8')
assert isinstance(csr, str)
return x509.load_pem_x509_csr(csr, default_backend())
return x509.load_pem_x509_csr(csr.encode('utf-8'), default_backend())
def get_authority_key(body):
@ -211,3 +224,13 @@ def truthiness(s):
"""If input string resembles something truthy then return True, else False."""
return s.lower() in ('true', 'yes', 'on', 't', '1')
def find_matching_certificates_by_hash(cert, matching_certs):
"""Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
determine if any of the certificate hashes match and return the matches."""
matching = []
for c in matching_certs:
if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()):
matching.append(c)
return matching

33
lemur/common/validators.py
View File

@ -2,14 +2,12 @@ import re
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.x509 import NameOID
from flask import current_app
from marshmallow.exceptions import ValidationError
from lemur.auth.permissions import SensitiveDomainPermission
from lemur.common.utils import parse_certificate, is_weekend
from lemur.domains import service as domain_service
def public_certificate(body):
@ -26,22 +24,6 @@ def public_certificate(body):
raise ValidationError('Public certificate presented is not valid.')
def private_key(key):
"""
User to validate that a given string is a RSA private key
:param key:
:return: :raise ValueError:
"""
try:
if isinstance(key, bytes):
serialization.load_pem_private_key(key, None, backend=default_backend())
else:
serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
except Exception:
raise ValidationError('Private key presented is not valid.')
def common_name(value):
"""If the common name could be a domain name, apply domain validation rules."""
# Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
@ -66,6 +48,9 @@ def sensitive_domain(domain):
raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
'Contact an administrator to issue the certificate.'.format(domain))
# Avoid circular import.
from lemur.domains import service as domain_service
if any(d.sensitive for d in domain_service.get_by_name(domain)):
raise ValidationError('Domain {0} has been marked as sensitive. '
'Contact an administrator to issue the certificate.'.format(domain))
@ -141,3 +126,15 @@ def dates(data):
raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
return data
def verify_private_key_match(key, cert, error_class=ValidationError):
"""
Checks that the supplied private key matches the certificate.
:param cert: Parsed certificate
:param key: Parsed private key
:param error_class: Exception class to raise on error
"""
if key.public_key().public_numbers() != cert.public_key().public_numbers():
raise error_class("Private key does not match certificate.")

32
lemur/database.py
View File

@ -10,12 +10,12 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from inflection import underscore
from sqlalchemy import exc, func
from sqlalchemy import exc, func, distinct
from sqlalchemy.orm import make_transient, lazyload
from sqlalchemy.sql import and_, or_
from sqlalchemy.orm import make_transient
from lemur.extensions import db
from lemur.exceptions import AttrNotFound, DuplicateError
from lemur.extensions import db
def filter_none(kwargs):
@ -273,7 +273,31 @@ def get_count(q):
:param q:
:return:
"""
count_q = q.statement.with_only_columns([func.count()]).order_by(None)
disable_group_by = False
if len(q._entities) > 1:
# currently support only one entity
raise Exception('only one entity is supported for get_count, got: %s' % q)
entity = q._entities[0]
if hasattr(entity, 'column'):
# _ColumnEntity has column attr - on case: query(Model.column)...
col = entity.column
if q._group_by and q._distinct:
# which query can have both?
raise NotImplementedError
if q._group_by or q._distinct:
col = distinct(col)
if q._group_by:
# need to disable group_by and enable distinct - we can do this because we have only 1 entity
disable_group_by = True
count_func = func.count(col)
else:
# _MapperEntity doesn't have column attr - on case: query(Model)...
count_func = func.count()
if q._group_by and not disable_group_by:
count_func = count_func.over(None)
count_q = q.options(lazyload('*')).statement.with_only_columns([count_func]).order_by(None)
if disable_group_by:
count_q = count_q.group_by(None)
count = q.session.execute(count_q).scalar()
return count

3
lemur/dns_providers/models.py
View File

@ -23,7 +23,8 @@ class DnsProvider(db.Model):
status = Column(String(length=128), nullable=True)
options = Column(JSON, nullable=True)
domains = Column(JSON, nullable=True)
certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id')
certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id',
lazy='dynamic')
def __init__(self, name, description, provider_type, credentials):
self.name = name

7
lemur/domains/models.py
View File

@ -7,13 +7,18 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import Column, Integer, String, Boolean
from sqlalchemy import Column, Integer, String, Boolean, Index
from lemur.database import db
class Domain(db.Model):
__tablename__ = 'domains'
__table_args__ = (
Index('ix_domains_name_gin', "name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
)
id = Column(Integer, primary_key=True)
name = Column(String(256), index=True)
sensitive = Column(Boolean, default=False)

21
lemur/manage.py
View File

@ -47,7 +47,7 @@ from lemur.logs.models import Log # noqa
from lemur.endpoints.models import Endpoint # noqa
from lemur.policies.models import RotationPolicy # noqa
from lemur.pending_certificates.models import PendingCertificate # noqa
from lemur.dns_providers.models import DnsProvider # noqa
manager = Manager(create_app)
manager.add_option('-c', '--config', dest='config')
@ -273,10 +273,11 @@ class CreateUser(Command):
Option('-u', '--username', dest='username', required=True),
Option('-e', '--email', dest='email', required=True),
Option('-a', '--active', dest='active', default=True),
Option('-r', '--roles', dest='roles', action='append', default=[])
Option('-r', '--roles', dest='roles', action='append', default=[]),
Option('-p', '--password', dest='password', default=None)
)
def run(self, username, email, active, roles):
def run(self, username, email, active, roles, password):
role_objs = []
for r in roles:
role_obj = role_service.get_by_name(r)
@ -286,14 +287,16 @@ class CreateUser(Command):
sys.stderr.write("[!] Cannot find role {0}\n".format(r))
sys.exit(1)
password1 = prompt_pass("Password")
password2 = prompt_pass("Confirm Password")
if not password:
password1 = prompt_pass("Password")
password2 = prompt_pass("Confirm Password")
password = password1
if password1 != password2:
sys.stderr.write("[!] Passwords do not match!\n")
sys.exit(1)
if password1 != password2:
sys.stderr.write("[!] Passwords do not match!\n")
sys.exit(1)
user_service.create(username, password1, email, active, None, role_objs)
user_service.create(username, password, email, active, None, role_objs)
sys.stdout.write("[+] Created new user: {0}\n".format(username))

8
lemur/migrations/versions/449c3d5c7299_.py
View File

@ -21,6 +21,14 @@ COLUMNS = ["notification_id", "certificate_id"]
def upgrade():
connection = op.get_bind()
# Delete duplicate entries
connection.execute("""\
DELETE FROM certificate_notification_associations WHERE ctid NOT IN (
-- Select the first tuple ID for each (notification_id, certificate_id) combination and keep that
SELECT min(ctid) FROM certificate_notification_associations GROUP BY notification_id, certificate_id
)
""")
op.create_unique_constraint(CONSTRAINT_NAME, TABLE, COLUMNS)

31
lemur/migrations/versions/ee827d1e1974_.py Normal file
View File

@ -0,0 +1,31 @@
"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries.
Revision ID: ee827d1e1974
Revises: 7ead443ba911
Create Date: 2018-11-05 09:49:40.226368
"""
# revision identifiers, used by Alembic.
revision = 'ee827d1e1974'
down_revision = '7ead443ba911'
from alembic import op
from sqlalchemy.exc import ProgrammingError
def upgrade():
connection = op.get_bind()
connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'},
postgresql_using='gin')
op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False,
postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin')
op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'},
postgresql_using='gin')
def downgrade():
op.drop_index('ix_domains_name', table_name='domains')
op.drop_index('ix_certificates_name', table_name='certificates')
op.drop_index('ix_certificates_cn', table_name='certificates')

35
lemur/notifications/messaging.py
View File

@ -8,24 +8,21 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from itertools import groupby
from collections import defaultdict
from datetime import timedelta
from itertools import groupby
import arrow
from datetime import timedelta
from flask import current_app
from sqlalchemy import and_
from lemur import database
from lemur.certificates.models import Certificate
from lemur.certificates.schemas import certificate_notification_output_schema
from lemur.common.utils import windowed_query
from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS
from lemur.extensions import metrics, sentry
from lemur.common.utils import windowed_query
from lemur.certificates.schemas import certificate_notification_output_schema
from lemur.certificates.models import Certificate
from lemur.pending_certificates.schemas import pending_certificate_output_schema
from lemur.plugins import plugins
from lemur.plugins.utils import get_plugin_option
@ -74,10 +71,11 @@ def get_eligible_certificates(exclude=None):
notification_groups = []
for certificate in items:
notification = needs_notification(certificate)
notifications = needs_notification(certificate)
if notification:
notification_groups.append((notification, certificate))
if notifications:
for notification in notifications:
notification_groups.append((notification, certificate))
# group by notification
for notification, items in groupby(notification_groups, lambda x: x[0].label):
@ -133,11 +131,21 @@ def send_expiration_notifications(exclude):
notification_data.append(cert_data)
security_data.append(cert_data)
notification_recipient = get_plugin_option('recipients', notification.options)
if notification_recipient:
notification_recipient = notification_recipient.split(",")
if send_notification('expiration', notification_data, [owner], notification):
success += 1
else:
failure += 1
if notification_recipient and owner != notification_recipient and security_email != notification_recipient:
if send_notification('expiration', notification_data, notification_recipient, notification):
success += 1
else:
failure += 1
if send_notification('expiration', security_data, security_email, notification):
success += 1
else:
@ -228,6 +236,8 @@ def needs_notification(certificate):
now = arrow.utcnow()
days = (certificate.not_after - now).days
notifications = []
for notification in certificate.notifications:
if not notification.active or not notification.options:
return
@ -248,4 +258,5 @@ def needs_notification(certificate):
raise Exception("Invalid base unit for expiration interval: {0}".format(unit))
if days == interval:
return notification
notifications.append(notification)
return notifications

24
lemur/pending_certificates/models.py
View File

@ -10,7 +10,7 @@ from sqlalchemy.orm import relationship
from sqlalchemy_utils import JSONType
from sqlalchemy_utils.types.arrow import ArrowType
from lemur.certificates.models import get_or_increase_name
from lemur.certificates.models import get_sequence
from lemur.common import defaults, utils
from lemur.database import db
from lemur.models import pending_cert_source_associations, \
@ -19,6 +19,28 @@ from lemur.models import pending_cert_source_associations, \
from lemur.utils import Vault
def get_or_increase_name(name, serial):
certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(name))).all()
if not certificates:
return name
serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper())
certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(serial_name))).all()
if not certificates:
return serial_name
ends = [0]
root, end = get_sequence(serial_name)
for cert in certificates:
root, end = get_sequence(cert.name)
if end:
ends.append(end)
return '{0}-{1}'.format(root, max(ends) + 1)
class PendingCertificate(db.Model):
__tablename__ = 'pending_certs'
id = Column(Integer, primary_key=True)

8
lemur/plugins/lemur_acme/dyn.py
View File

@ -5,7 +5,7 @@ import dns.exception
import dns.name
import dns.query
import dns.resolver
from dyn.tm.errors import DynectCreateError
from dyn.tm.errors import DynectCreateError, DynectGetError
from dyn.tm.session import DynectSession
from dyn.tm.zones import Node, Zone, get_all_zones
from flask import current_app
@ -119,7 +119,11 @@ def delete_txt_record(change_id, account_number, domain, token):
zone = Zone(zone_name)
node = Node(zone_name, fqdn)
all_txt_records = node.get_all_records_by_type('TXT')
try:
all_txt_records = node.get_all_records_by_type('TXT')
except DynectGetError:
# No Text Records remain or host is not in the zone anymore because all records have been deleted.
return
for txt_record in all_txt_records:
if txt_record.txtdata == ("{}".format(token)):
current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))

21
lemur/plugins/lemur_acme/plugin.py
View File

@ -44,7 +44,11 @@ class AuthorizationRecord(object):
class AcmeHandler(object):
def __init__(self):
self.dns_providers_for_domain = {}
self.all_dns_providers = dns_provider_service.get_all_dns_providers()
try:
self.all_dns_providers = dns_provider_service.get_all_dns_providers()
except Exception as e:
current_app.logger.error("Unable to fetch DNS Providers: {}".format(e))
self.all_dns_providers = []
def find_dns_challenge(self, authorizations):
dns_challenges = []
@ -211,12 +215,18 @@ class AcmeHandler(object):
:return: dns_providers: List of DNS providers that have the correct zone.
"""
self.dns_providers_for_domain[domain] = []
match_length = 0
for dns_provider in self.all_dns_providers:
if not dns_provider.domains:
continue
for name in dns_provider.domains:
if domain.endswith("." + name):
self.dns_providers_for_domain[domain].append(dns_provider)
if len(name) > match_length:
self.dns_providers_for_domain[domain] = [dns_provider]
match_length = len(name)
elif len(name) == match_length:
self.dns_providers_for_domain[domain].append(dns_provider)
return self.dns_providers_for_domain
def finalize_authorizations(self, acme_client, authorizations):
@ -329,9 +339,10 @@ class ACMEIssuerPlugin(IssuerPlugin):
def __init__(self, *args, **kwargs):
super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
self.acme = AcmeHandler()
def get_dns_provider(self, type):
self.acme = AcmeHandler()
provider_types = {
'cloudflare': cloudflare,
'dyn': dyn,
@ -343,12 +354,14 @@ class ACMEIssuerPlugin(IssuerPlugin):
return provider
def get_all_zones(self, dns_provider):
self.acme = AcmeHandler()
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
return dns_provider_plugin.get_zones(account_number=account_number)
def get_ordered_certificate(self, pending_cert):
self.acme = AcmeHandler()
acme_client, registration = self.acme.setup_acme_client(pending_cert.authority)
order_info = authorization_service.get(pending_cert.external_id)
if pending_cert.dns_provider_id:
@ -384,6 +397,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
return cert
def get_ordered_certificates(self, pending_certs):
self.acme = AcmeHandler()
pending = []
certs = []
for pending_cert in pending_certs:
@ -466,6 +480,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
:param issuer_options:
:return: :raise Exception:
"""
self.acme = AcmeHandler()
authority = issuer_options.get('authority')
create_immediately = issuer_options.get('create_immediately', False)
acme_client, registration = self.acme.setup_acme_client(authority)

18
lemur/plugins/lemur_aws/elb.py
View File

@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
"""
Get a listener ARN from an endpoint.
@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_elbs(**kwargs):
"""
Fetches one page elb objects for a given account and region.
@ -123,7 +123,7 @@ def get_elbs(**kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_elbs_v2(**kwargs):
"""
Fetches one page of elb objects for a given account and region.
@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_listeners_v2(**kwargs):
"""
Fetches one page of listener objects for a given elb arn.
@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
"""
Fetching all policies currently associated with an ELB.
@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_ssl_policies_v2(policy_names, **kwargs):
"""
Fetching all policies currently associated with an ELB.
@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def describe_load_balancer_types(policies, **kwargs):
"""
Describe the policies with policy details.
@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def attach_certificate(name, port, certificate_id, **kwargs):
"""
Attaches a certificate to a listener, throws exception
@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def attach_certificate_v2(listener_arn, port, certificates, **kwargs):
"""
Attaches a certificate to a listener, throws exception

11
lemur/plugins/lemur_aws/iam.py
View File

@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
"""
Upload a certificate to AWS
@ -64,6 +64,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
:param path:
:return:
"""
assert isinstance(private_key, str)
client = kwargs.pop('client')
if not path or path == '/':
@ -72,8 +73,6 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
name = name + '-' + path.strip('/')
try:
if isinstance(private_key, bytes):
private_key = private_key.decode("utf-8")
if cert_chain:
return client.upload_server_certificate(
Path=path,
@ -95,7 +94,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def delete_cert(cert_name, **kwargs):
"""
Delete a certificate from AWS
@ -112,7 +111,7 @@ def delete_cert(cert_name, **kwargs):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_certificate(name, **kwargs):
"""
Retrieves an SSL certificate.
@ -126,7 +125,7 @@ def get_certificate(name, **kwargs):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
def get_certificates(**kwargs):
"""
Fetches one page of certificate objects for a given account.

9
lemur/plugins/lemur_aws/plugin.py
View File

@ -35,8 +35,8 @@
from flask import current_app
from lemur.plugins import lemur_aws as aws
from lemur.plugins.lemur_aws import iam, s3, elb, ec2
from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
from lemur.plugins.lemur_aws import iam, s3, elb, ec2
def get_region_from_dns(dns):
@ -163,7 +163,7 @@ class AWSDestinationPlugin(DestinationPlugin):
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '/^[0-9]{12,12}$/',
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
@ -279,14 +279,14 @@ class S3DestinationPlugin(ExportDestinationPlugin):
'name': 'bucket',
'type': 'str',
'required': True,
'validation': '/^$|\s+/',
'validation': '[0-9a-z.-]{3,63}',
'helpMessage': 'Must be a valid S3 bucket name!',
},
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '/^[0-9]{12,12}$/',
'validation': '[0-9]{12}',
'helpMessage': 'A valid AWS account number with permission to access S3',
},
{
@ -308,7 +308,6 @@ class S3DestinationPlugin(ExportDestinationPlugin):
'name': 'prefix',
'type': 'str',
'required': False,
'validation': '/^$|\s+/',
'helpMessage': 'Must be a valid S3 object prefix!',
}
]

16
lemur/plugins/lemur_aws/sts.py
View File

@ -9,14 +9,22 @@ from functools import wraps
import boto3
from botocore.config import Config
from flask import current_app
config = Config(
retries=dict(
max_attempts=20
)
)
def sts_client(service, service_type='client'):
def decorator(f):
@wraps(f)
def decorated_function(*args, **kwargs):
sts = boto3.client('sts')
sts = boto3.client('sts', config=config)
arn = 'arn:aws:iam::{0}:role/{1}'.format(
kwargs.pop('account_number'),
current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
@ -31,7 +39,8 @@ def sts_client(service, service_type='client'):
region_name=kwargs.pop('region', 'us-east-1'),
aws_access_key_id=role['Credentials']['AccessKeyId'],
aws_secret_access_key=role['Credentials']['SecretAccessKey'],
aws_session_token=role['Credentials']['SessionToken']
aws_session_token=role['Credentials']['SessionToken'],
config=config
)
kwargs['client'] = client
elif service_type == 'resource':
@ -40,7 +49,8 @@ def sts_client(service, service_type='client'):
region_name=kwargs.pop('region', 'us-east-1'),
aws_access_key_id=role['Credentials']['AccessKeyId'],
aws_secret_access_key=role['Credentials']['SecretAccessKey'],
aws_session_token=role['Credentials']['SessionToken']
aws_session_token=role['Credentials']['SessionToken'],
config=config
)
kwargs['resource'] = resource
return f(*args, **kwargs)

2
lemur/plugins/lemur_aws/tests/test_elb.py
View File

@ -4,7 +4,7 @@ from moto import mock_sts, mock_elb
@mock_sts()
@mock_elb()
def test_get_all_elbs(app):
def test_get_all_elbs(app, aws_credentials):
from lemur.plugins.lemur_aws.elb import get_all_elbs
client = boto3.client('elb', region_name='us-east-1')

18
lemur/plugins/lemur_cfssl/plugin.py
View File

@ -10,6 +10,9 @@
import json
import requests
import base64
import hmac
import hashlib
from flask import current_app
@ -48,6 +51,21 @@ class CfsslIssuerPlugin(IssuerPlugin):
data = {'certificate_request': csr}
data = json.dumps(data)
try:
hex_key = current_app.config.get('CFSSL_KEY')
key = bytes.fromhex(hex_key)
except (ValueError, NameError):
# unable to find CFSSL_KEY in config, continue using normal sign method
pass
else:
data = data.encode()
token = base64.b64encode(hmac.new(key, data, digestmod=hashlib.sha256).digest())
data = base64.b64encode(data)
data = json.dumps({'token': token.decode('utf-8'), 'request': data.decode('utf-8')})
url = "{0}{1}".format(current_app.config.get('CFSSL_URL'), '/api/v1/cfssl/authsign')
response = self.session.post(url, data=data.encode(encoding='utf_8', errors='strict'))
if response.status_code > 399:
metrics.send('cfssl_create_certificate_failure', 'counter', 1)

11
lemur/plugins/lemur_cryptography/plugin.py
View File

@ -14,6 +14,7 @@ from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from lemur.common.utils import parse_private_key
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins import lemur_cryptography as cryptography_issuer
@ -40,7 +41,8 @@ def issue_certificate(csr, options, private_key=None):
if options.get("authority"):
# Issue certificate signed by an existing lemur_certificates authority
issuer_subject = options['authority'].authority_certificate.subject
issuer_private_key = options['authority'].authority_certificate.private_key
assert private_key is None, "Private would be ignored, authority key used instead"
private_key = options['authority'].authority_certificate.private_key
chain_cert_pem = options['authority'].authority_certificate.body
authority_key_identifier_public = options['authority'].authority_certificate.public_key
authority_key_identifier_subject = x509.SubjectKeyIdentifier.from_public_key(authority_key_identifier_public)
@ -52,7 +54,6 @@ def issue_certificate(csr, options, private_key=None):
else:
# Issue certificate that is self-signed (new lemur_certificates root authority)
issuer_subject = csr.subject
issuer_private_key = private_key
chain_cert_pem = ""
authority_key_identifier_public = csr.public_key()
authority_key_identifier_subject = None
@ -112,11 +113,7 @@ def issue_certificate(csr, options, private_key=None):
# FIXME: Not implemented in lemur/schemas.py yet https://github.com/Netflix/lemur/issues/662
pass
private_key = serialization.load_pem_private_key(
bytes(str(issuer_private_key).encode('utf-8')),
password=None,
backend=default_backend()
)
private_key = parse_private_key(private_key)
cert = builder.sign(private_key, hashes.SHA256(), default_backend())
cert_pem = cert.public_bytes(

11
lemur/plugins/lemur_csr/plugin.py
View File

@ -38,14 +38,9 @@ def create_csr(cert, chain, csr_tmp, key):
:param csr_tmp:
:param key:
"""
if isinstance(cert, bytes):
cert = cert.decode('utf-8')
if isinstance(chain, bytes):
chain = chain.decode('utf-8')
if isinstance(key, bytes):
key = key.decode('utf-8')
assert isinstance(cert, str)
assert isinstance(chain, str)
assert isinstance(key, str)
with mktempfile() as key_tmp:
with open(key_tmp, 'w') as f:

18
lemur/plugins/lemur_java/plugin.py
View File

@ -59,11 +59,8 @@ def split_chain(chain):
def create_truststore(cert, chain, jks_tmp, alias, passphrase):
if isinstance(cert, bytes):
cert = cert.decode('utf-8')
if isinstance(chain, bytes):
chain = chain.decode('utf-8')
assert isinstance(cert, str)
assert isinstance(chain, str)
with mktempfile() as cert_tmp:
with open(cert_tmp, 'w') as f:
@ -98,14 +95,9 @@ def create_truststore(cert, chain, jks_tmp, alias, passphrase):
def create_keystore(cert, chain, jks_tmp, key, alias, passphrase):
if isinstance(cert, bytes):
cert = cert.decode('utf-8')
if isinstance(chain, bytes):
chain = chain.decode('utf-8')
if isinstance(key, bytes):
key = key.decode('utf-8')
assert isinstance(cert, str)
assert isinstance(chain, str)
assert isinstance(key, str)
# Create PKCS12 keystore from private key and public certificate
with mktempfile() as cert_tmp:

233
lemur/plugins/lemur_kubernetes/plugin.py
View File

@ -11,31 +11,37 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
"""
import base64
import os
import urllib
import requests
import itertools
import os
from lemur.certificates.models import Certificate
import requests
from flask import current_app
from lemur.common.defaults import common_name
from lemur.common.utils import parse_certificate
from lemur.plugins.bases import DestinationPlugin
DEFAULT_API_VERSION = 'v1'
def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data):
# _resolve_uri(k8s_base_uri, namespace, kind, name, api_ver=DEFAULT_API_VERSION)
url = _resolve_uri(k8s_base_uri, namespace, kind)
current_app.logger.debug("K8S POST request URL: %s", url)
create_resp = k8s_api.post(url, json=data)
current_app.logger.debug("K8S POST response: %s", create_resp)
if 200 <= create_resp.status_code <= 299:
return None
elif create_resp.json()['reason'] != 'AlreadyExists':
elif create_resp.json().get('reason', '') != 'AlreadyExists':
return create_resp.content
update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data)
url = _resolve_uri(k8s_base_uri, namespace, kind, name)
current_app.logger.debug("K8S PUT request URL: %s", url)
update_resp = k8s_api.put(url, json=data)
current_app.logger.debug("K8S PUT response: %s", update_resp)
if not 200 <= update_resp.status_code <= 299:
return update_resp.content
@ -43,11 +49,12 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data):
return
def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,):
def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION):
api_group = 'api'
if '/' in api_ver:
api_group = 'apis'
return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ('/' + namespace if namespace else '')
return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + (
'/' + namespace if namespace else '')
def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION):
@ -61,6 +68,41 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V
]))
# Performs Base64 encoding of string to string using the base64.b64encode() function
# which encodes bytes to bytes.
def base64encode(string):
return base64.b64encode(string.encode()).decode()
def build_secret(secret_format, secret_name, body, private_key, cert_chain):
secret = {
'apiVersion': 'v1',
'kind': 'Secret',
'type': 'Opaque',
'metadata': {
'name': secret_name,
}
}
if secret_format == 'Full':
secret['data'] = {
'combined.pem': base64encode('%s\n%s' % (body, private_key)),
'ca.crt': base64encode(cert_chain),
'service.key': base64encode(private_key),
'service.crt': base64encode(body),
}
if secret_format == 'TLS':
secret['type'] = 'kubernetes.io/tls'
secret['data'] = {
'tls.crt': base64encode(cert_chain),
'tls.key': base64encode(private_key)
}
if secret_format == 'Certificate':
secret['data'] = {
'tls.crt': base64encode(cert_chain),
}
return secret
class KubernetesDestinationPlugin(DestinationPlugin):
title = 'Kubernetes'
slug = 'kubernetes-destination'
@ -70,35 +112,81 @@ class KubernetesDestinationPlugin(DestinationPlugin):
author_url = 'https://github.com/mik373/lemur'
options = [
{
'name': 'secretNameFormat',
'type': 'str',
'required': False,
# Validation is difficult. This regex is used by kubectl to validate secret names:
# [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*
# Allowing the insertion of "{common_name}" (or any other such placeholder}
# at any point in the string proved very challenging and had a tendency to
# cause my browser to hang. The specified expression will allow any valid string
# but will also accept many invalid strings.
'validation': '(?:[a-z0-9.-]|\\{common_name\\})+',
'helpMessage': 'Must be a valid secret name, possibly including "{common_name}"',
'default': '{common_name}'
},
{
'name': 'kubernetesURL',
'type': 'str',
'required': True,
'validation': '@(https?|http)://(-\.)?([^\s/?\.#-]+\.?)+(/[^\s]*)?$@iS',
'required': False,
'validation': 'https?://[a-zA-Z0-9.-]+(?::[0-9]+)?',
'helpMessage': 'Must be a valid Kubernetes server URL!',
'default': 'https://kubernetes.default'
},
{
'name': 'kubernetesAuthToken',
'type': 'str',
'required': True,
'validation': '/^$|\s+/',
'required': False,
'validation': '[0-9a-zA-Z-_.]+',
'helpMessage': 'Must be a valid Kubernetes server Token!',
},
{
'name': 'kubernetesServerCertificate',
'name': 'kubernetesAuthTokenFile',
'type': 'str',
'required': True,
'validation': '/^$|\s+/',
'required': False,
'validation': '(/[^/]+)+',
'helpMessage': 'Must be a valid file path!',
'default': '/var/run/secrets/kubernetes.io/serviceaccount/token'
},
{
'name': 'kubernetesServerCertificate',
'type': 'textarea',
'required': False,
'validation': '-----BEGIN CERTIFICATE-----[a-zA-Z0-9/+\\s\\r\\n]+-----END CERTIFICATE-----',
'helpMessage': 'Must be a valid Kubernetes server Certificate!',
},
{
'name': 'kubernetesServerCertificateFile',
'type': 'str',
'required': False,
'validation': '(/[^/]+)+',
'helpMessage': 'Must be a valid file path!',
'default': '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
},
{
'name': 'kubernetesNamespace',
'type': 'str',
'required': True,
'validation': '/^$|\s+/',
'required': False,
'validation': '[a-z0-9]([-a-z0-9]*[a-z0-9])?',
'helpMessage': 'Must be a valid Kubernetes Namespace!',
},
{
'name': 'kubernetesNamespaceFile',
'type': 'str',
'required': False,
'validation': '(/[^/]+)+',
'helpMessage': 'Must be a valid file path!',
'default': '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
},
{
'name': 'secretFormat',
'type': 'select',
'required': True,
'available': ['Full', 'TLS', 'Certificate'],
'helpMessage': 'The type of Secret to create.',
'default': 'Full'
}
]
def __init__(self, *args, **kwargs):
@ -106,56 +194,91 @@ class KubernetesDestinationPlugin(DestinationPlugin):
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
k8_bearer = self.get_option('kubernetesAuthToken', options)
k8_cert = self.get_option('kubernetesServerCertificate', options)
k8_namespace = self.get_option('kubernetesNamespace', options)
k8_base_uri = self.get_option('kubernetesURL', options)
try:
k8_base_uri = self.get_option('kubernetesURL', options)
secret_format = self.get_option('secretFormat', options)
k8s_api = K8sSession(
self.k8s_bearer(options),
self.k8s_cert(options)
)
cn = common_name(parse_certificate(body))
secret_name_format = self.get_option('secretNameFormat', options)
secret_name = secret_name_format.format(common_name=cn)
secret = build_secret(secret_format, secret_name, body, private_key, cert_chain)
err = ensure_resource(
k8s_api,
k8s_base_uri=k8_base_uri,
namespace=self.k8s_namespace(options),
kind="secret",
name=secret_name,
data=secret
)
k8s_api = K8sSession(k8_bearer, k8_cert)
cert = Certificate(body=body)
# in the future once runtime properties can be passed-in - use passed-in secret name
secret_name = 'certs-' + urllib.quote_plus(cert.name)
err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={
'apiVersion': 'v1',
'kind': 'Secret',
'metadata': {
'name': secret_name,
},
'data': {
'combined.pem': base64.b64encode(body + private_key),
'ca.crt': base64.b64encode(cert_chain),
'service.key': base64.b64encode(private_key),
'service.crt': base64.b64encode(body),
}
})
except Exception as e:
current_app.logger.exception("Exception in upload: {}".format(e), exc_info=True)
raise
if err is not None:
current_app.logger.error("Error deploying resource: %s", err)
raise Exception("Error uploading secret: " + err)
def k8s_bearer(self, options):
bearer = self.get_option('kubernetesAuthToken', options)
if not bearer:
bearer_file = self.get_option('kubernetesAuthTokenFile', options)
with open(bearer_file, "r") as file:
bearer = file.readline()
if bearer:
current_app.logger.debug("Using token read from %s", bearer_file)
else:
raise Exception("Unable to locate token in options or from %s", bearer_file)
else:
current_app.logger.debug("Using token from options")
return bearer
def k8s_cert(self, options):
cert_file = self.get_option('kubernetesServerCertificateFile', options)
cert = self.get_option('kubernetesServerCertificate', options)
if cert:
cert_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert')
with open(cert_file, "w") as text_file:
text_file.write(cert)
current_app.logger.debug("Using certificate from options")
else:
current_app.logger.debug("Using certificate from %s", cert_file)
return cert_file
def k8s_namespace(self, options):
namespace = self.get_option('kubernetesNamespace', options)
if not namespace:
namespace_file = self.get_option('kubernetesNamespaceFile', options)
with open(namespace_file, "r") as file:
namespace = file.readline()
if namespace:
current_app.logger.debug("Using namespace %s from %s", namespace, namespace_file)
else:
raise Exception("Unable to locate namespace in options or from %s", namespace_file)
else:
current_app.logger.debug("Using namespace %s from options", namespace)
return namespace
class K8sSession(requests.Session):
def __init__(self, bearer, cert):
def __init__(self, bearer, cert_file):
super(K8sSession, self).__init__()
self.headers.update({
'Authorization': 'Bearer %s' % bearer
})
k8_ca = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert')
self.verify = cert_file
with open(k8_ca, "w") as text_file:
text_file.write(cert)
self.verify = k8_ca
def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=30, allow_redirects=True, proxies=None,
hooks=None, stream=None, verify=None, cert=None, json=None):
def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None,
timeout=30, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None,
json=None):
"""
This method overrides the default timeout to be 10s.
"""
return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream,
verify, cert, json)
return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout,
allow_redirects, proxies, hooks, stream, verify, cert, json)

11
lemur/plugins/lemur_openssl/plugin.py
View File

@ -44,14 +44,9 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase):
:param alias:
:param passphrase:
"""
if isinstance(cert, bytes):
cert = cert.decode('utf-8')
if isinstance(chain, bytes):
chain = chain.decode('utf-8')
if isinstance(key, bytes):
key = key.decode('utf-8')
assert isinstance(cert, str)
assert isinstance(chain, str)
assert isinstance(key, str)
with mktempfile() as key_tmp:
with open(key_tmp, 'w') as f:

15
lemur/plugins/lemur_verisign/plugin.py
View File

@ -111,10 +111,19 @@ def process_options(options):
data['subject_alt_names'] = ",".join(get_additional_names(options))
if options.get('validity_end') > arrow.utcnow().replace(years=2):
raise Exception("Verisign issued certificates cannot exceed two years in validity")
if options.get('validity_end'):
period = get_default_issuance(options)
data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY")
data['validityPeriod'] = period
# VeriSign (Symantec) only accepts strictly smaller than 2 year end date
if options.get('validity_end') < arrow.utcnow().replace(years=2).replace(days=-1):
period = get_default_issuance(options)
data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY")
data['validityPeriod'] = period
else:
# allowing Symantec website setting the end date, given the validity period
data['validityPeriod'] = str(get_default_issuance(options))
options.pop('validity_end', None)
elif options.get('validity_years'):
if options['validity_years'] in [1, 2]:

1
lemur/sources/cli.py
View File

@ -93,6 +93,7 @@ def sync(source_strings):
)
sentry.captureException()
metrics.send('source_sync_fail', 'counter', 1, metric_tags={'source': source.label, 'status': status})
metrics.send('source_sync', 'counter', 1, metric_tags={'source': source.label, 'status': status})

5
lemur/sources/service.py
View File

@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service
from lemur.destinations import service as destination_service
from lemur.certificates.schemas import CertificateUploadInputSchema
from lemur.common.utils import parse_certificate
from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate
from lemur.common.defaults import serial
from lemur.plugins.base import plugins
@ -131,7 +131,8 @@ def sync_certificates(source, user):
if not exists:
cert = parse_certificate(certificate['body'])
exists = certificate_service.get_by_serial(serial(cert))
matching_serials = certificate_service.get_by_serial(serial(cert))
exists = find_matching_certificates_by_hash(cert, matching_serials)
if not certificate.get('owner'):
certificate['owner'] = user.email

2
lemur/static/app/angular/certificates/view/view.tpl.html
View File

@ -83,6 +83,8 @@
</div>
<!-- Certificate fields -->
<div class="list-group-item">
<dt>Distinguished Name</dt>
<dd>{{ certificate.distinguishedName }}</dd>
<dt>Certificate Authority</dt>
<dd>{{ certificate.authority ? certificate.authority.name : "Imported" }} <span class="text-muted">({{ certificate.issuer }})</span></dd>
<dt>Serial</dt>

6
lemur/static/app/angular/destinations/destination/destination.tpl.html
View File

@ -47,7 +47,9 @@
<select name="sub" ng-if="item.type == 'select'" class="form-control" ng-options="i for i in item.available"
ng-model="item.value"></select>
<input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox" ng-model="item.value">
<input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value"/>
<input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value" ng-pattern="item.validation"/>
<textarea name="sub" ng-if="item.type == 'textarea'" class="form-control"
ng-model="item.value" ng-pattern="item.validation"></textarea>
<div ng-if="item.type == 'export-plugin'">
<form name="exportForm" class="form-horizontal" role="form" novalidate>
<select class="form-control" ng-model="item.value"
@ -69,6 +71,8 @@
ng-model="item.value">
<input name="sub" ng-if="item.type == 'str'" type="text" class="form-control"
ng-model="item.value" ng-pattern="item.validation"/>
<textarea name="sub" ng-if="item.type == 'textarea'" class="form-control"
ng-model="item.value" ng-pattern="item.validation"></textarea>
<p ng-show="subForm.sub.$invalid && !subForm.sub.$pristine"
class="help-block">{{ item.helpMessage }}</p>
</div>

5
lemur/tests/conf.py
View File

@ -1,7 +1,7 @@
# This is just Python which means you can inherit and tweak settings
import os
_basedir = os.path.abspath(os.path.dirname(__file__))
THREADS_PER_PAGE = 8
@ -78,14 +78,12 @@ DIGICERT_API_KEY = 'api-key'
DIGICERT_ORG_ID = 111111
DIGICERT_ROOT = "ROOT"
VERISIGN_URL = 'http://example.com'
VERISIGN_PEM_PATH = '~/'
VERISIGN_FIRST_NAME = 'Jim'
VERISIGN_LAST_NAME = 'Bob'
VERSIGN_EMAIL = 'jim@example.com'
ACME_AWS_ACCOUNT_NUMBER = '11111111111'
ACME_PRIVATE_KEY = '''
@ -180,6 +178,7 @@ ACME_URL = 'https://acme-v01.api.letsencrypt.org'
ACME_EMAIL = 'jim@example.com'
ACME_TEL = '4088675309'
ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org'
ACME_DISABLE_AUTORESOLVE = True
LDAP_AUTH = True
LDAP_BIND_URI = 'ldap://localhost'

29
lemur/tests/conftest.py
View File

@ -3,19 +3,18 @@ import os
import datetime
import pytest
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from flask import current_app
from flask_principal import identity_changed, Identity
from lemur import create_app
from lemur.common.utils import parse_private_key
from lemur.database import db as _db
from lemur.auth.service import create_token
from lemur.tests.vectors import SAN_CERT_KEY
from lemur.tests.vectors import SAN_CERT_KEY, INTERMEDIATE_KEY
from .factories import ApiKeyFactory, AuthorityFactory, NotificationFactory, DestinationFactory, \
CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, \
RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory
RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory, CryptoAuthorityFactory
def pytest_runtest_setup(item):
@ -91,6 +90,13 @@ def authority(session):
return a
@pytest.fixture
def crypto_authority(session):
a = CryptoAuthorityFactory()
session.commit()
return a
@pytest.fixture
def async_authority(session):
a = AsyncAuthorityFactory()
@ -228,7 +234,12 @@ def logged_in_admin(session, app):
@pytest.fixture
def private_key():
return load_pem_private_key(SAN_CERT_KEY.encode(), password=None, backend=default_backend())
return parse_private_key(SAN_CERT_KEY)
@pytest.fixture
def issuer_private_key():
return parse_private_key(INTERMEDIATE_KEY)
@pytest.fixture
@ -240,3 +251,11 @@ def cert_builder(private_key):
.public_key(private_key.public_key())
.not_valid_before(datetime.datetime(2017, 12, 22))
.not_valid_after(datetime.datetime(2040, 1, 1)))
@pytest.fixture(scope='function')
def aws_credentials():
os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
os.environ['AWS_SECURITY_TOKEN'] = 'testing'
os.environ['AWS_SESSION_TOKEN'] = 'testing'

5
lemur/tests/factories.py
View File

@ -168,6 +168,11 @@ class AsyncAuthorityFactory(AuthorityFactory):
authority_certificate = SubFactory(CertificateFactory)
class CryptoAuthorityFactory(AuthorityFactory):
"""Authority factory based on 'cryptography' plugin."""
plugin = {'slug': 'cryptography-issuer'}
class DestinationFactory(BaseFactory):
"""Destination factory."""
plugin_name = 'test-destination'

94
lemur/tests/test_certificates.py
View File

@ -18,7 +18,7 @@ from lemur.domains.models import Domain
from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \
INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY
INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
def test_get_or_increase_name(session, certificate):
@ -448,6 +448,85 @@ def test_certificate_sensitive_name(client, authority, session, logged_in_user):
assert errors['common_name'][0].startswith("Domain sensitive.example.com has been marked as sensitive")
def test_certificate_upload_schema_ok(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'name': 'Jane',
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
'privateKey': SAN_CERT_KEY,
'chain': INTERMEDIATE_CERT_STR,
'external_id': '1234',
}
data, errors = CertificateUploadInputSchema().load(data)
assert not errors
def test_certificate_upload_schema_minimal(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
}
data, errors = CertificateUploadInputSchema().load(data)
assert not errors
def test_certificate_upload_schema_long_chain(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
'chain': INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR
}
data, errors = CertificateUploadInputSchema().load(data)
assert not errors
def test_certificate_upload_schema_invalid_body(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': 'Hereby I certify that this is a valid body',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'body': ['Public certificate presented is not valid.']}
def test_certificate_upload_schema_invalid_pkey(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'owner': 'pwner@example.com',
'body': SAN_CERT_STR,
'privateKey': 'Look at me Im a private key!!111',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'private_key': ['Private key presented is not valid.']}
def test_certificate_upload_schema_invalid_chain(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'body': SAN_CERT_STR,
'chain': 'CHAINSAW',
'owner': 'pwner@example.com',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'chain': ['Public certificate presented is not valid.']}
def test_certificate_upload_schema_wrong_pkey(client):
from lemur.certificates.schemas import CertificateUploadInputSchema
data = {
'body': SAN_CERT_STR,
'privateKey': ROOTCA_KEY,
'chain': INTERMEDIATE_CERT_STR,
'owner': 'pwner@example.com',
}
data, errors = CertificateUploadInputSchema().load(data)
assert errors == {'_schema': ['Private key does not match certificate.']}
def test_create_basic_csr(client):
csr_config = dict(
common_name='example.com',
@ -545,8 +624,11 @@ def test_create_certificate(issuer_plugin, authority, user):
assert cert.name == 'ACustomName1'
def test_reissue_certificate(issuer_plugin, authority, certificate):
def test_reissue_certificate(issuer_plugin, crypto_authority, certificate, logged_in_user):
from lemur.certificates.service import reissue_certificate
# test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead.
certificate.authority = crypto_authority
new_cert = reissue_certificate(certificate)
assert new_cert
@ -570,7 +652,7 @@ def test_import(user):
assert str(cert.not_after) == '2047-12-31T22:00:00+00:00'
assert str(cert.not_before) == '2017-12-31T22:00:00+00:00'
assert cert.issuer == 'LemurTrustUnittestsClass1CA2018'
assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2'
assert cert.name.startswith('SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231')
cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, owner='joe@example.com', name='ACustomName2', creator=user['user'])
assert cert.name == 'ACustomName2'
@ -620,6 +702,12 @@ def test_certificate_get_body(client):
response_body = client.get(api.url_for(Certificates, certificate_id=1), headers=VALID_USER_HEADER_TOKEN).json
assert response_body['serial'] == '211983098819107449768450703123665283596'
assert response_body['serialHex'] == '9F7A75B39DAE4C3F9524C68B06DA6A0C'
assert response_body['distinguishedName'] == ('CN=LemurTrust Unittests Class 1 CA 2018,'
'O=LemurTrust Enterprises Ltd,'
'OU=Unittesting Operations Center,'
'C=EE,'
'ST=N/A,'
'L=Earth')
@pytest.mark.parametrize("token,status", [

32
lemur/tests/test_defaults.py
View File

@ -1,3 +1,7 @@
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from .vectors import SAN_CERT, WILDCARD_CERT, INTERMEDIATE_CERT
@ -41,12 +45,14 @@ def test_cert_issuer(client):
def test_text_to_slug(client):
from lemur.common.defaults import text_to_slug
assert text_to_slug('test - string') == 'test-string'
assert text_to_slug('test - string', '') == 'teststring'
# Accented characters are decomposed
assert text_to_slug('föö bär') == 'foo-bar'
# Melt away the Unicode Snowman
assert text_to_slug('\u2603') == ''
assert text_to_slug('\u2603test\u2603') == 'test'
assert text_to_slug('snow\u2603man') == 'snow-man'
assert text_to_slug('snow\u2603man', '') == 'snowman'
# IDNA-encoded domain names should be kept as-is
assert text_to_slug('xn--i1b6eqas.xn--xmpl-loa9b3671b.com') == 'xn--i1b6eqas.xn--xmpl-loa9b3671b.com'
@ -75,3 +81,29 @@ def test_create_name(client):
datetime(2015, 5, 12, 0, 0, 0),
False
) == 'xn--mnchen-3ya.de-VertrauenswurdigAutoritat-20150507-20150512'
def test_issuer(client, cert_builder, issuer_private_key):
from lemur.common.defaults import issuer
assert issuer(INTERMEDIATE_CERT) == 'LemurTrustUnittestsRootCA2018'
# We need to override builder's issuer name
cert_builder._issuer_name = None
# Unicode issuer name
cert = (cert_builder
.issuer_name(x509.Name([x509.NameAttribute(x509.NameOID.COMMON_NAME, 'Vertrauenswürdig Autorität')]))
.sign(issuer_private_key, hashes.SHA256(), default_backend()))
assert issuer(cert) == 'VertrauenswurdigAutoritat'
# Fallback to 'Organization' field when issuer CN is missing
cert = (cert_builder
.issuer_name(x509.Name([x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, 'No Such Organization')]))
.sign(issuer_private_key, hashes.SHA256(), default_backend()))
assert issuer(cert) == 'NoSuchOrganization'
# Missing issuer name
cert = (cert_builder
.issuer_name(x509.Name([]))
.sign(issuer_private_key, hashes.SHA256(), default_backend()))
assert issuer(cert) == 'Unknown'

6
lemur/tests/test_pending_certificates.py
View File

@ -2,11 +2,10 @@ import json
import pytest
from lemur.pending_certificates.views import * # noqa
from .vectors import CSR_STR, INTERMEDIATE_CERT_STR, VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, \
VALID_USER_HEADER_TOKEN, WILDCARD_CERT_STR
from lemur.pending_certificates.views import * # noqa
def test_increment_attempt(pending_certificate):
from lemur.pending_certificates.service import increment_attempt
@ -17,7 +16,8 @@ def test_increment_attempt(pending_certificate):
def test_create_pending_certificate(async_issuer_plugin, async_authority, user):
from lemur.certificates.service import create
pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], common_name='ACommonName')
pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'],
common_name='ACommonName')
assert pending_cert.external_id == '12345'

22
lemur/tests/test_validators.py
View File

@ -1,16 +1,28 @@
import pytest
from datetime import datetime
from .vectors import SAN_CERT_KEY
import pytest
from marshmallow.exceptions import ValidationError
from lemur.common.utils import parse_private_key
from lemur.common.validators import verify_private_key_match
from lemur.tests.vectors import INTERMEDIATE_CERT, SAN_CERT, SAN_CERT_KEY
def test_private_key(session):
from lemur.common.validators import private_key
parse_private_key(SAN_CERT_KEY)
private_key(SAN_CERT_KEY)
with pytest.raises(ValueError):
parse_private_key('invalid_private_key')
def test_validate_private_key(session):
key = parse_private_key(SAN_CERT_KEY)
verify_private_key_match(key, SAN_CERT)
with pytest.raises(ValidationError):
private_key('invalid_private_key')
# Wrong key for certificate
verify_private_key_match(key, INTERMEDIATE_CERT)
def test_sub_alt_type(session):

2
requirements-dev.in
View File

@ -1,6 +1,6 @@
# Run `make up-reqs` to update pinned dependencies in requirement text files
flake8>=3.2,<4.0
flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors.
pre-commit
invoke
twine

42
requirements-dev.txt
View File

@ -4,36 +4,34 @@
#
# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
#
aspy.yaml==1.1.1 # via pre-commit
bleach==3.0.2 # via readme-renderer
cached-property==1.5.1 # via pre-commit
certifi==2018.10.15 # via requests
cffi==1.11.5 # via cmarkgfm
cfgv==1.1.0 # via pre-commit
aspy.yaml==1.1.2 # via pre-commit
bleach==3.1.0 # via readme-renderer
certifi==2018.11.29 # via requests
cfgv==1.4.0 # via pre-commit
chardet==3.0.4 # via requests
cmarkgfm==0.4.2 # via readme-renderer
docutils==0.14 # via readme-renderer
flake8==3.5.0
future==0.16.0 # via readme-renderer
identify==1.1.7 # via pre-commit
idna==2.7 # via requests
identify==1.2.1 # via pre-commit
idna==2.8 # via requests
importlib-metadata==0.8 # via pre-commit
importlib-resources==1.0.2 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.2
pkginfo==1.4.2 # via twine
pre-commit==1.11.2
nodeenv==1.3.3
pkginfo==1.5.0.1 # via twine
pre-commit==1.14.2
pycodestyle==2.3.1 # via flake8
pycparser==2.19 # via cffi
pyflakes==1.6.0 # via flake8
pygments==2.2.0 # via readme-renderer
pygments==2.3.1 # via readme-renderer
pyyaml==3.13 # via aspy.yaml, pre-commit
readme-renderer==22.0 # via twine
requests-toolbelt==0.8.0 # via twine
requests==2.20.0 # via requests-toolbelt, twine
six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer
readme-renderer==24.0 # via twine
requests-toolbelt==0.9.0 # via twine
requests==2.21.0 # via requests-toolbelt, twine
six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.28.1 # via twine
tqdm==4.30.0 # via twine
twine==1.12.1
urllib3==1.24 # via requests
virtualenv==16.0.0 # via pre-commit
urllib3==1.24.1 # via requests
virtualenv==16.3.0 # via pre-commit
webencodings==0.5.1 # via bleach
zipp==0.3.3 # via importlib-metadata

2
requirements-docs.in
View File

@ -4,4 +4,4 @@
-r requirements.txt
sphinx
sphinxcontrib-httpdomain
sphinx-rtd-theme
sphinx-rtd-theme

86
requirements-docs.txt
View File

@ -4,93 +4,93 @@
#
# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
#
acme==0.27.1
acme==0.30.2
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.1
amqp==2.3.2
aniso8601==3.0.2
arrow==0.12.1
alembic==1.0.7
amqp==2.4.0
aniso8601==4.1.0
arrow==0.13.0
asn1crypto==0.24.0
asyncpool==1.0
babel==2.6.0 # via sphinx
bcrypt==3.1.4
billiard==3.5.0.4
bcrypt==3.1.6
billiard==3.5.0.5
blinker==1.4
boto3==1.7.79
botocore==1.10.84
boto3==1.9.86
botocore==1.12.86
celery[redis]==4.2.1
certifi==2018.10.15
certifi==2018.11.29
cffi==1.11.5
chardet==3.0.4
click==7.0
cloudflare==2.1.0
cryptography==2.3.1
cryptography==2.5
dnspython3==1.15.0
dnspython==1.15.0
docutils==0.14
dyn==1.8.1
flask-bcrypt==0.7.1
flask-cors==3.0.6
flask-cors==3.0.7
flask-mail==0.9.1
flask-migrate==2.1.1
flask-migrate==2.3.1
flask-principal==0.4.0
flask-restful==0.3.6
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask==0.12
future==0.16.0
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
idna==2.7
idna==2.8
imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==1.0.0
itsdangerous==1.1.0
jinja2==2.10
jmespath==0.9.3
josepy==1.1.0
jsonlines==1.2.0
kombu==4.2.1
kombu==4.2.2.post1
lockfile==0.12.2
mako==1.0.7
markupsafe==1.0
marshmallow-sqlalchemy==0.14.1
marshmallow==2.16.0
markupsafe==1.1.0
marshmallow-sqlalchemy==0.15.0
marshmallow==2.18.0
mock==2.0.0
ndg-httpsclient==0.5.1
packaging==18.0 # via sphinx
packaging==19.0 # via sphinx
paramiko==2.4.2
pbr==5.0.0
pbr==5.1.1
pem==18.2.0
psycopg2==2.7.5
pyasn1-modules==0.2.2
pyasn1==0.4.4
psycopg2==2.7.7
pyasn1-modules==0.2.4
pyasn1==0.4.5
pycparser==2.19
pygments==2.2.0 # via sphinx
pyjwt==1.6.4
pygments==2.3.1 # via sphinx
pyjwt==1.7.1
pynacl==1.3.0
pyopenssl==18.0.0
pyparsing==2.2.2 # via packaging
pyopenssl==19.0.0
pyparsing==2.3.1 # via packaging
pyrfc3339==1.1
python-dateutil==2.7.3
python-dateutil==2.7.5
python-editor==1.0.3
pytz==2018.5
pytz==2018.9
pyyaml==3.13
raven[flask]==6.9.0
raven[flask]==6.10.0
redis==2.10.6
requests-toolbelt==0.8.0
requests[security]==2.20.0
requests-toolbelt==0.9.0
requests[security]==2.21.0
retrying==1.3.3
s3transfer==0.1.13
six==1.11.0
six==1.12.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.4.2
sphinx==1.8.1
sphinx==1.8.3
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sqlalchemy-utils==0.33.6
sqlalchemy==1.2.12
tabulate==0.8.2
urllib3==1.24
vine==1.1.4
sqlalchemy-utils==0.33.11
sqlalchemy==1.2.17
tabulate==0.8.3
urllib3==1.24.1
vine==1.2.0
werkzeug==0.14.1
xmltodict==0.11.0

2
requirements-tests.in
View File

@ -4,7 +4,7 @@ coverage
factory-boy
Faker
freezegun
moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813
moto
nose
pyflakes
pytest

60
requirements-tests.txt
View File

@ -8,59 +8,57 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto
biscuits==0.1.1 # via responses
boto3==1.9.28 # via moto
boto3==1.9.86 # via moto
boto==2.49.0 # via moto
botocore==1.12.28 # via boto3, moto, s3transfer
certifi==2018.10.15 # via requests
botocore==1.12.86 # via boto3, moto, s3transfer
certifi==2018.11.29 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
click==7.0 # via flask
cookies==2.2.1 # via moto
coverage==4.5.1
cryptography==2.3.1 # via moto
docker-pycreds==0.3.0 # via docker
docker==3.5.1 # via moto
coverage==4.5.2
cryptography==2.5 # via moto
docker-pycreds==0.4.0 # via docker
docker==3.7.0 # via moto
docutils==0.14 # via botocore
ecdsa==0.13 # via python-jose
factory-boy==2.11.1
faker==0.9.2
faker==1.0.2
flask==1.0.2 # via pytest-flask
freezegun==0.3.11
future==0.16.0 # via python-jose
idna==2.7 # via cryptography, requests
itsdangerous==1.0.0 # via flask
future==0.17.1 # via python-jose
idna==2.8 # via requests
itsdangerous==1.1.0 # via flask
jinja2==2.10 # via flask, moto
jmespath==0.9.3 # via boto3, botocore
jsondiff==1.1.1 # via moto
jsonpickle==1.0 # via aws-xray-sdk
markupsafe==1.0 # via jinja2
jsonpickle==1.1 # via aws-xray-sdk
markupsafe==1.1.0 # via jinja2
mock==2.0.0 # via moto
more-itertools==4.3.0 # via pytest
moto==1.3.4
more-itertools==5.0.0 # via pytest
moto==1.3.7
nose==1.3.7
pbr==5.0.0 # via mock
pluggy==0.8.0 # via pytest
pbr==5.1.1 # via mock
pluggy==0.8.1 # via pytest
py==1.7.0 # via pytest
pyaml==17.12.1 # via moto
pyaml==18.11.0 # via moto
pycparser==2.19 # via cffi
pycryptodome==3.6.6 # via python-jose
pyflakes==2.0.0
pycryptodome==3.7.3 # via python-jose
pyflakes==2.1.0
pytest-flask==0.14.0
pytest-mock==1.10.0
pytest==3.9.1
python-dateutil==2.7.3 # via botocore, faker, freezegun, moto
pytest==4.1.1
python-dateutil==2.7.5 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.5 # via moto
pytz==2018.9 # via moto
pyyaml==3.13 # via pyaml
requests-mock==1.5.2
requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
responses==0.10.1 # via moto
requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
responses==0.10.5 # via moto
s3transfer==0.1.13 # via boto3
six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.23 # via botocore, requests
websocket-client==0.53.0 # via docker
urllib3==1.24.1 # via botocore, requests
websocket-client==0.54.0 # via docker
werkzeug==0.14.1 # via flask, moto, pytest-flask
wrapt==1.10.11 # via aws-xray-sdk
wrapt==1.11.1 # via aws-xray-sdk
xmltodict==0.11.0 # via moto

19
requirements.in
View File

@ -4,22 +4,22 @@ acme
alembic-autogenerate-enums
arrow
asyncpool
boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813
botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813
boto3
botocore
celery[redis]
certifi
CloudFlare
cryptography
dnspython3
dyn
Flask-Bcrypt==0.7.1
Flask-Mail==0.9.1
Flask-Migrate==2.1.1
Flask-Principal==0.4.0
Flask-RESTful==0.3.6
Flask-Script==2.0.6
Flask-Bcrypt
Flask-Mail
Flask-Migrate
Flask-Principal
Flask-RESTful
Flask-Script
Flask-SQLAlchemy
Flask==0.12
Flask
Flask-Cors
future
gunicorn
@ -36,6 +36,7 @@ pyjwt
pyOpenSSL
python_ldap
raven[flask]
redis<3 # redis>=3 is not compatible with celery
requests
retrying
six

80
requirements.txt
View File

@ -4,83 +4,83 @@
#
# pip-compile --no-index --output-file requirements.txt requirements.in
#
acme==0.27.1
acme==0.30.2
alembic-autogenerate-enums==0.0.2
alembic==1.0.1 # via flask-migrate
amqp==2.3.2 # via kombu
aniso8601==3.0.2 # via flask-restful
arrow==0.12.1
alembic==1.0.7 # via flask-migrate
amqp==2.4.0 # via kombu
aniso8601==4.1.0 # via flask-restful
arrow==0.13.0
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.4 # via flask-bcrypt, paramiko
billiard==3.5.0.4 # via celery
bcrypt==3.1.6 # via flask-bcrypt, paramiko
billiard==3.5.0.5 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.7.79
botocore==1.10.84
boto3==1.9.86
botocore==1.12.86
celery[redis]==4.2.1
certifi==2018.10.15
certifi==2018.11.29
cffi==1.11.5 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
click==7.0 # via flask
cloudflare==2.1.0
cryptography==2.3.1
cryptography==2.5
dnspython3==1.15.0
dnspython==1.15.0 # via dnspython3
docutils==0.14 # via botocore
dyn==1.8.1
flask-bcrypt==0.7.1
flask-cors==3.0.6
flask-cors==3.0.7
flask-mail==0.9.1
flask-migrate==2.1.1
flask-migrate==2.3.1
flask-principal==0.4.0
flask-restful==0.3.6
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask==0.12
future==0.16.0
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
idna==2.7 # via cryptography, requests
idna==2.8 # via requests
inflection==0.3.1
itsdangerous==1.0.0 # via flask
itsdangerous==1.1.0 # via flask
jinja2==2.10
jmespath==0.9.3 # via boto3, botocore
josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.2.1 # via celery
kombu==4.2.2.post1 # via celery
lockfile==0.12.2
mako==1.0.7 # via alembic
markupsafe==1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.14.1
marshmallow==2.16.0
markupsafe==1.1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.15.0
marshmallow==2.18.0
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.0.0 # via mock
pbr==5.1.1 # via mock
pem==18.2.0
psycopg2==2.7.5
pyasn1-modules==0.2.2 # via python-ldap
pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
psycopg2==2.7.7
pyasn1-modules==0.2.4 # via python-ldap
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pycparser==2.19 # via cffi
pyjwt==1.6.4
pyjwt==1.7.1
pynacl==1.3.0 # via paramiko
pyopenssl==18.0.0
pyopenssl==19.0.0
pyrfc3339==1.1 # via acme
python-dateutil==2.7.3 # via alembic, arrow, botocore
python-dateutil==2.7.5 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic
python-ldap==3.1.0
pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339
pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339
pyyaml==3.13 # via cloudflare
raven[flask]==6.9.0
redis==2.10.6 # via celery
requests-toolbelt==0.8.0 # via acme
requests[security]==2.20.0
raven[flask]==6.10.0
redis==2.10.6
requests-toolbelt==0.9.0 # via acme
requests[security]==2.21.0
retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.11.0
sqlalchemy-utils==0.33.6
sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
urllib3==1.24 # via requests
vine==1.1.4 # via amqp
six==1.12.0
sqlalchemy-utils==0.33.11
sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.3
urllib3==1.24.1 # via botocore, requests
vine==1.2.0 # via amqp
werkzeug==0.14.1 # via flask
xmltodict==0.11.0