diff --git a/.travis.yml b/.travis.yml index 37ec1434..b540937d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres + - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools @@ -45,4 +46,4 @@ after_success: notifications: email: - kglisson@netflix.com + ccastrapel@netflix.com diff --git a/Makefile b/Makefile index f740faab..f859f554 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ reset-db: dropdb lemur || true @echo "--> Creating 'lemur' database" createdb -E utf-8 lemur + @echo "--> Enabling pg_trgm extension" + psql lemur -c "create extension IF NOT EXISTS pg_trgm;" @echo "--> Applying migrations" lemur db upgrade @@ -111,10 +113,10 @@ endif @echo "--> Updating Python requirements" pip install --upgrade pip pip install --upgrade pip-tools + pip-compile --output-file requirements.txt requirements.in -U --no-index pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index - pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" @echo "--> Removing python-ldap from requirements-docs.txt" grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..f7d1caf7 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,64 @@ +FROM alpine:3.8 + +ARG VERSION +ENV VERSION master + +ENV uid 1337 +ENV gid 1337 +ENV user lemur +ENV group lemur + +COPY entrypoint / +COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py +COPY supervisor.conf / +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/default-ssl.conf /etc/nginx/conf.d/ + +RUN addgroup -S ${group} -g ${gid} && \ + adduser -D -S ${user} -G ${group} -u ${uid} && \ + apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \ + apk --update add --virtual build-dependencies \ + git \ + tar \ + curl \ + python3-dev \ + npm \ + bash \ + musl-dev \ + gcc \ + autoconf \ + automake \ + make \ + nasm \ + zlib-dev \ + postgresql-dev \ + libressl-dev \ + libffi-dev \ + cyrus-sasl-dev \ + openldap-dev && \ + mkdir -p /opt/lemur /home/lemur/.lemur/ && \ + curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ + pip3 install --upgrade pip && \ + pip3 install --upgrade setuptools && \ + chmod +x /entrypoint && \ + mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ + chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ + +WORKDIR /opt/lemur + +RUN npm install --unsafe-perm && \ + pip3 install -e . && \ + node_modules/.bin/gulp build && \ + node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \ + apk del build-dependencies + +WORKDIR / + +HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ + CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1 + +USER root + +ENTRYPOINT ["/entrypoint"] + +CMD ["/usr/bin/supervisord","-c","supervisor.conf"] diff --git a/docker/entrypoint b/docker/entrypoint new file mode 100644 index 00000000..6077167a --- /dev/null +++ b/docker/entrypoint @@ -0,0 +1,54 @@ +#!/bin/sh + +if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then + echo "Database vars not set" + exit 1 +fi + +export POSTGRES_PORT="${POSTGRES_PORT:-5432}" + +echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile + +source /etc/profile + +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' + +echo " # Create Postgres trgm extension" +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' +echo " # Done" + +if [ -z "${SKIP_SSL}" ]; then + if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" + fi + mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf + mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a +fi + +# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then +# echo "Creating config" +# https://github.com/Netflix/lemur/issues/2257 +# python3 /opt/lemur/lemur/manage.py create_config +# echo "Done" +# fi + +echo " # Running init" +su lemur -c "python3 /opt/lemur/lemur/manage.py init" +echo " # Done" + +# echo "Creating user" +# https://github.com/Netflix/lemur/issues/ +# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell +# echo "Done" + +cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" +cron_sync="${CRON_SYNC:-"*/15 * * * *"}" +cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" + +echo " # Populating crontab" +echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify +echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync +echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked +echo " # Done" + +exec "$@" diff --git a/docker/nginx/default-ssl.conf b/docker/nginx/default-ssl.conf new file mode 100644 index 00000000..86c770df --- /dev/null +++ b/docker/nginx/default-ssl.conf @@ -0,0 +1,37 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 80; + server_name _; + return 301 https://$host$request_uri; +} + +server { + listen 443; + server_name _; + access_log /dev/stdout; + error_log /dev/stderr; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf new file mode 100644 index 00000000..d71a93d3 --- /dev/null +++ b/docker/nginx/default.conf @@ -0,0 +1,26 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 80; + access_log /dev/stdout; + error_log /dev/stderr; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} diff --git a/docker/src/lemur.conf.py b/docker/src/lemur.conf.py new file mode 100644 index 00000000..a5f7e8b6 --- /dev/null +++ b/docker/src/lemur.conf.py @@ -0,0 +1,31 @@ +import os +_basedir = os.path.abspath(os.path.dirname(__file__)) + +CORS = os.environ.get("CORS") == "True" +debug = os.environ.get("DEBUG") == "True" + +SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw==')) + +LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ==')) +LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo=')) + +LEMUR_WHITELISTED_DOMAINS = [] + +LEMUR_EMAIL = '' +LEMUR_SECURITY_TEAM_EMAIL = [] + + +LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY','')) +LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE','')) +LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION','')) +LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION','')) +LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT','')) + +ACTIVE_PROVIDERS = [] + +METRIC_PROVIDERS = [] + +LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) +LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log')) + +SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') diff --git a/docker/supervisor.conf b/docker/supervisor.conf new file mode 100644 index 00000000..fed01581 --- /dev/null +++ b/docker/supervisor.conf @@ -0,0 +1,32 @@ +[supervisord] +nodaemon=true +user=root +logfile=/dev/stdout +logfile_maxbytes=0 +pidfile = /tmp/supervisord.pid + +[program:lemur] +environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py +command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000 +user=lemur +directory=/opt/lemur/lemur +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:cron] +command=/usr/sbin/crond -f +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/docs/administration.rst b/docs/administration.rst index eec01cc5..9d6c8d12 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -324,6 +324,7 @@ Here is an example LDAP configuration stanza you can add to your config. Adjust LDAP_CACERT_FILE = '/opt/lemur/trusted.pem' LDAP_REQUIRED_GROUP = 'certificate-management-access' LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'} + LDAP_IS_ACTIVE_DIRECTORY = True The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN. @@ -406,6 +407,17 @@ The following LDAP options are not required, however TLS is always recommended. LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'} +.. data:: LDAP_IS_ACTIVE_DIRECTORY + :noindex: + + When set to True, nested group memberships are supported, by searching for groups with the member:1.2.840.113556.1.4.1941 attribute set to the user DN. + When set to False, the list of groups will be determined by the 'memberof' attribute of the LDAP user logging in. + + :: + + LDAP_IS_ACTIVE_DIRECTORY = False + + Authentication Providers ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 70ca1312..adeadd7c 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur: Installing Build Dependencies ----------------------------- -If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies: +If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies: .. code-block:: bash @@ -117,7 +117,7 @@ Simply run: .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command. -You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment. +You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment. Update your configuration @@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config LEMUR_DEFAULT_ORGANIZATION LEMUR_DEFAULT_ORGANIZATIONAL_UNIT -Setup Postgres +Set Up Postgres -------------- For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on. @@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. -Setup a Reverse Proxy +Set Up a Reverse Proxy --------------------- -By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. +By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. Proxying with Nginx diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index 398a5830..7eded060 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -41,6 +41,7 @@ class LdapPrincipal(): self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None) self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None) self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None) + self.ldap_is_active_directory = current_app.config.get("LDAP_IS_ACTIVE_DIRECTORY", False) self.ldap_attrs = ['memberOf'] self.ldap_client = None self.ldap_groups = None @@ -168,11 +169,28 @@ class LdapPrincipal(): except ldap.LDAPError as e: raise Exception("ldap error: {0}".format(e)) - lgroups = self.ldap_client.search_s(self.ldap_base_dn, - ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf'] - # lgroups is a list of utf-8 encoded strings - # convert to a single string of groups to allow matching - self.ldap_groups = b''.join(lgroups).decode('ascii') + if self.ldap_is_active_directory: + # Lookup user DN, needed to search for group membership + userdn = self.ldap_client.search_s(self.ldap_base_dn, + ldap.SCOPE_SUBTREE, ldap_filter, + ['distinguishedName'])[0][1]['distinguishedName'][0] + userdn = userdn.decode('utf-8') + # Search all groups that have the userDN as a member + groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn) + lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn']) + + # Create a list of group CN's from the result + self.ldap_groups = [] + for group in lgroups: + (dn, values) = group + self.ldap_groups.append(values['cn'][0].decode('ascii')) + else: + lgroups = self.ldap_client.search_s(self.ldap_base_dn, + ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf'] + # lgroups is a list of utf-8 encoded strings + # convert to a single string of groups to allow matching + self.ldap_groups = b''.join(lgroups).decode('ascii') + self.ldap_client.unbind() def _ldap_validate_conf(self): diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 1d35ad49..41c381e3 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -15,6 +15,7 @@ from lemur import database from lemur.common.utils import truthiness from lemur.extensions import metrics from lemur.authorities.models import Authority +from lemur.certificates.models import Certificate from lemur.roles import service as role_service from lemur.certificates.service import upload @@ -178,6 +179,13 @@ def render(args): terms = filt.split(';') if 'active' in filt: query = query.filter(Authority.active == truthiness(terms[1])) + elif 'cn' in filt: + term = '%{0}%'.format(terms[1]) + sub_query = database.session_query(Certificate.root_authority_id) \ + .filter(Certificate.cn.ilike(term)) \ + .subquery() + + query = query.filter(Authority.id.in_(sub_query)) else: query = database.filter(query, Authority, terms) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 013a4cb1..c4a95187 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -265,30 +265,31 @@ def query(fqdns, issuer, owner, expired): table = [] q = database.session_query(Certificate) + if issuer: + sub_query = database.session_query(Authority.id) \ + .filter(Authority.name.ilike('%{0}%'.format(issuer))) \ + .subquery() - sub_query = database.session_query(Authority.id) \ - .filter(Authority.name.ilike('%{0}%'.format(issuer))) \ - .subquery() - - q = q.filter( - or_( - Certificate.issuer.ilike('%{0}%'.format(issuer)), - Certificate.authority_id.in_(sub_query) + q = q.filter( + or_( + Certificate.issuer.ilike('%{0}%'.format(issuer)), + Certificate.authority_id.in_(sub_query) + ) ) - ) - - q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner))) + if owner: + q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner))) if not expired: q = q.filter(Certificate.expired == False) # noqa - for f in fqdns.split(','): - q = q.filter( - or_( - Certificate.cn.ilike('%{0}%'.format(f)), - Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f))) + if fqdns: + for f in fqdns.split(','): + q = q.filter( + or_( + Certificate.cn.ilike('%{0}%'.format(f)), + Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f))) + ) ) - ) for c in q.all(): table.append([c.id, c.name, c.owner, c.issuer]) @@ -363,10 +364,7 @@ def check_revoked(): else: status = verify_string(cert.body, "") - if status is None: - cert.status = 'unknown' - else: - cert.status = 'valid' if status else 'revoked' + cert.status = 'valid' if status else 'revoked' except Exception as e: sentry.captureException() diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 6b438c06..34305cc2 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -19,7 +19,7 @@ from sqlalchemy.sql.expression import case, extract from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property -from lemur.common import defaults, utils +from lemur.common import defaults, utils, validators from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS from lemur.database import db from lemur.domains.models import Domain @@ -77,6 +77,14 @@ def get_or_increase_name(name, serial): class Certificate(db.Model): __tablename__ = 'certificates' + __table_args__ = ( + Index('ix_certificates_cn', "cn", + postgresql_ops={"cn": "gin_trgm_ops"}, + postgresql_using='gin'), + Index('ix_certificates_name', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True) external_id = Column(String(128)) @@ -130,7 +138,6 @@ class Certificate(db.Model): logs = relationship('Log', backref='certificate') endpoints = relationship('Endpoint', backref='certificate') rotation_policy = relationship("RotationPolicy") - sensitive_fields = ('private_key',) def __init__(self, **kwargs): @@ -179,6 +186,18 @@ class Certificate(db.Model): for domain in defaults.domains(cert): self.domains.append(Domain(name=domain)) + # Check integrity before saving anything into the database. + # For user-facing API calls, validation should also be done in schema validators. + self.check_integrity() + + def check_integrity(self): + """ + Integrity checks: Does the cert have a matching private key? + """ + if self.private_key: + validators.verify_private_key_match(utils.parse_private_key(self.private_key), self.parsed_cert, + error_class=AssertionError) + @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" @@ -208,6 +227,10 @@ class Certificate(db.Model): def location(self): return defaults.location(self.parsed_cert) + @property + def distinguished_name(self): + return self.parsed_cert.subject.rfc4514_string() + @property def key_type(self): if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey): @@ -359,7 +382,7 @@ def update_destinations(target, value, initiator): destination_plugin = plugins.get(value.plugin_name) status = FAILURE_METRIC_STATUS try: - if target.private_key: + if target.private_key or not destination_plugin.requires_key: destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options) status = SUCCESS_METRIC_STATUS except Exception as e: diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index bf18eac9..946bd541 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -10,7 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError from lemur.authorities.schemas import AuthorityNestedOutputSchema -from lemur.common import validators, missing +from lemur.common import missing, utils, validators from lemur.common.fields import ArrowDateTime, Hex from lemur.common.schema import LemurInputSchema, LemurOutputSchema from lemur.constants import CERTIFICATE_KEY_TYPES @@ -206,6 +206,7 @@ class CertificateOutputSchema(LemurOutputSchema): cn = fields.String() common_name = fields.String(attribute='cn') + distinguished_name = fields.String() not_after = fields.DateTime() validity_end = ArrowDateTime(attribute='not_after') @@ -242,8 +243,8 @@ class CertificateUploadInputSchema(CertificateCreationSchema): authority = fields.Nested(AssociatedAuthoritySchema, required=False) notify = fields.Boolean(missing=True) external_id = fields.String(missing=None, allow_none=True) - private_key = fields.String(validate=validators.private_key) - body = fields.String(required=True, validate=validators.public_certificate) + private_key = fields.String() + body = fields.String(required=True) chain = fields.String(validate=validators.public_certificate, missing=None, allow_none=True) # TODO this could be multiple certificates @@ -258,6 +259,26 @@ class CertificateUploadInputSchema(CertificateCreationSchema): if not data.get('private_key'): raise ValidationError('Destinations require private key.') + @validates_schema + def validate_cert_private_key(self, data): + cert = None + key = None + if data.get('body'): + try: + cert = utils.parse_certificate(data['body']) + except ValueError: + raise ValidationError("Public certificate presented is not valid.", field_names=['body']) + + if data.get('private_key'): + try: + key = utils.parse_private_key(data['private_key']) + except ValueError: + raise ValidationError("Private key presented is not valid.", field_names=['private_key']) + + if cert and key: + # Throws ValidationError + validators.verify_private_key_match(key, cert) + class CertificateExportInputSchema(LemurInputSchema): plugin = fields.Nested(PluginInputSchema) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b8016dd4..d5012012 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -237,11 +237,6 @@ def upload(**kwargs): else: kwargs['roles'] = roles - if kwargs.get('private_key'): - private_key = kwargs['private_key'] - if not isinstance(private_key, bytes): - kwargs['private_key'] = private_key.encode('utf-8') - cert = Certificate(**kwargs) cert.authority = kwargs.get('authority') cert = database.create(cert) @@ -291,6 +286,14 @@ def create(**kwargs): certificate_issued.send(certificate=cert, authority=cert.authority) metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer)) + if isinstance(cert, PendingCertificate): + # We need to refresh the pending certificate to avoid "Instance is not bound to a Session; " + # "attribute refresh operation cannot proceed" + pending_cert = database.session_query(PendingCertificate).get(cert.id) + from lemur.common.celery import fetch_acme_cert + if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False): + fetch_acme_cert.apply_async((pending_cert.id,), countdown=5) + return cert @@ -314,7 +317,7 @@ def render(args): if filt: terms = filt.split(';') - term = '%{0}%'.format(terms[1]) + term = '{0}%'.format(terms[1]) # Exact matches for quotes. Only applies to name, issuer, and cn if terms[1].startswith('"') and terms[1].endswith('"'): term = terms[1][1:-1] @@ -378,7 +381,8 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now) - return database.sort_and_page(query, Certificate, args) + result = database.sort_and_page(query, Certificate, args) + return result def create_csr(**csr_config): @@ -439,10 +443,7 @@ def create_csr(**csr_config): encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, # would like to use PKCS8 but AWS ELBs don't like it encryption_algorithm=serialization.NoEncryption() - ) - - if isinstance(private_key, bytes): - private_key = private_key.decode('utf-8') + ).decode('utf-8') csr = request.public_bytes( encoding=serialization.Encoding.PEM @@ -554,6 +555,9 @@ def reissue_certificate(certificate, replace=None, user=None): """ primitives = get_certificate_primitives(certificate) + if primitives.get("csr"): + # We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation. + del primitives["csr"] if not user: primitives['creator'] = certificate.user diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 8dbb6c29..f2a2f826 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -19,14 +19,17 @@ from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.sources.cli import clean, validate_sources +from lemur.sources.cli import clean, sync, validate_sources -flask_app = create_app() +if current_app: + flask_app = current_app +else: + flask_app = create_app() def make_celery(app): - celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'], - broker=app.config['CELERY_BROKER_URL']) + celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'), + broker=app.config.get('CELERY_BROKER_URL')) celery.conf.update(app.config) TaskBase = celery.Task @@ -53,8 +56,10 @@ def fetch_acme_cert(id): id: an id of a PendingCertificate """ log_data = { - "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "message": "Resolving pending certificate {}".format(id) } + current_app.logger.debug(log_data) pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 @@ -138,12 +143,22 @@ def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" pending_certs = pending_certificate_service.get_unresolved_pending_certs() + log_data = { + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "message": "Starting job." + } + + current_app.logger.debug(log_data) + # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': - if cert.last_updated == cert.date_created or datetime.now( - timezone.utc) - cert.last_updated > timedelta(minutes=5): + if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5): + log_data["message"] = "Triggering job for cert {}".format(cert.name) + log_data["cert_name"] = cert.name + log_data["cert_id"] = cert.id + current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) @@ -188,3 +203,26 @@ def clean_source(source): """ current_app.logger.debug("Cleaning source {}".format(source)) clean([source], True) + + +@celery.task() +def sync_all_sources(): + """ + This function will sync certificates from all sources. This function triggers one celery task per source. + """ + sources = validate_sources("all") + for source in sources: + current_app.logger.debug("Creating celery task to sync source {}".format(source.label)) + sync_source.delay(source.label) + + +@celery.task() +def sync_source(source): + """ + This celery task will sync the specified source. + + :param source: + :return: + """ + current_app.logger.debug("Syncing source {}".format(source)) + sync([source]) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index e9bbc6e6..72e863c1 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -7,18 +7,21 @@ from lemur.extensions import sentry from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE -def text_to_slug(value): - """Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.""" +def text_to_slug(value, joiner='-'): + """ + Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters. + A series of non-alphanumeric characters is replaced with the joiner character. + """ # Strip all character accents: decompose Unicode characters and then drop combining chars. value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c)) - # Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash. - # Except, keep 'xn--' used in IDNA domain names as is. - value = re.sub(r'[^A-Za-z0-9.]+(? """ from inflection import underscore -from sqlalchemy import exc, func +from sqlalchemy import exc, func, distinct +from sqlalchemy.orm import make_transient, lazyload from sqlalchemy.sql import and_, or_ -from sqlalchemy.orm import make_transient -from lemur.extensions import db from lemur.exceptions import AttrNotFound, DuplicateError +from lemur.extensions import db def filter_none(kwargs): @@ -273,7 +273,31 @@ def get_count(q): :param q: :return: """ - count_q = q.statement.with_only_columns([func.count()]).order_by(None) + disable_group_by = False + if len(q._entities) > 1: + # currently support only one entity + raise Exception('only one entity is supported for get_count, got: %s' % q) + entity = q._entities[0] + if hasattr(entity, 'column'): + # _ColumnEntity has column attr - on case: query(Model.column)... + col = entity.column + if q._group_by and q._distinct: + # which query can have both? + raise NotImplementedError + if q._group_by or q._distinct: + col = distinct(col) + if q._group_by: + # need to disable group_by and enable distinct - we can do this because we have only 1 entity + disable_group_by = True + count_func = func.count(col) + else: + # _MapperEntity doesn't have column attr - on case: query(Model)... + count_func = func.count() + if q._group_by and not disable_group_by: + count_func = count_func.over(None) + count_q = q.options(lazyload('*')).statement.with_only_columns([count_func]).order_by(None) + if disable_group_by: + count_q = count_q.group_by(None) count = q.session.execute(count_q).scalar() return count diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index d48cd0d1..435a2398 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -23,7 +23,8 @@ class DnsProvider(db.Model): status = Column(String(length=128), nullable=True) options = Column(JSON, nullable=True) domains = Column(JSON, nullable=True) - certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id') + certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id', + lazy='dynamic') def __init__(self, name, description, provider_type, credentials): self.name = name diff --git a/lemur/domains/models.py b/lemur/domains/models.py index d0c7fef4..05fccd9c 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -7,13 +7,18 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, String, Boolean +from sqlalchemy import Column, Integer, String, Boolean, Index from lemur.database import db class Domain(db.Model): __tablename__ = 'domains' + __table_args__ = ( + Index('ix_domains_name_gin', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) name = Column(String(256), index=True) sensitive = Column(Boolean, default=False) diff --git a/lemur/manage.py b/lemur/manage.py index 6b1e1013..184b9aa6 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -47,7 +47,7 @@ from lemur.logs.models import Log # noqa from lemur.endpoints.models import Endpoint # noqa from lemur.policies.models import RotationPolicy # noqa from lemur.pending_certificates.models import PendingCertificate # noqa - +from lemur.dns_providers.models import DnsProvider # noqa manager = Manager(create_app) manager.add_option('-c', '--config', dest='config') @@ -273,10 +273,11 @@ class CreateUser(Command): Option('-u', '--username', dest='username', required=True), Option('-e', '--email', dest='email', required=True), Option('-a', '--active', dest='active', default=True), - Option('-r', '--roles', dest='roles', action='append', default=[]) + Option('-r', '--roles', dest='roles', action='append', default=[]), + Option('-p', '--password', dest='password', default=None) ) - def run(self, username, email, active, roles): + def run(self, username, email, active, roles, password): role_objs = [] for r in roles: role_obj = role_service.get_by_name(r) @@ -286,14 +287,16 @@ class CreateUser(Command): sys.stderr.write("[!] Cannot find role {0}\n".format(r)) sys.exit(1) - password1 = prompt_pass("Password") - password2 = prompt_pass("Confirm Password") + if not password: + password1 = prompt_pass("Password") + password2 = prompt_pass("Confirm Password") + password = password1 - if password1 != password2: - sys.stderr.write("[!] Passwords do not match!\n") - sys.exit(1) + if password1 != password2: + sys.stderr.write("[!] Passwords do not match!\n") + sys.exit(1) - user_service.create(username, password1, email, active, None, role_objs) + user_service.create(username, password, email, active, None, role_objs) sys.stdout.write("[+] Created new user: {0}\n".format(username)) diff --git a/lemur/migrations/versions/449c3d5c7299_.py b/lemur/migrations/versions/449c3d5c7299_.py index 1dcb7ab5..0bc30db1 100644 --- a/lemur/migrations/versions/449c3d5c7299_.py +++ b/lemur/migrations/versions/449c3d5c7299_.py @@ -21,6 +21,14 @@ COLUMNS = ["notification_id", "certificate_id"] def upgrade(): + connection = op.get_bind() + # Delete duplicate entries + connection.execute("""\ + DELETE FROM certificate_notification_associations WHERE ctid NOT IN ( + -- Select the first tuple ID for each (notification_id, certificate_id) combination and keep that + SELECT min(ctid) FROM certificate_notification_associations GROUP BY notification_id, certificate_id + ) + """) op.create_unique_constraint(CONSTRAINT_NAME, TABLE, COLUMNS) diff --git a/lemur/migrations/versions/ee827d1e1974_.py b/lemur/migrations/versions/ee827d1e1974_.py new file mode 100644 index 00000000..62ac6222 --- /dev/null +++ b/lemur/migrations/versions/ee827d1e1974_.py @@ -0,0 +1,31 @@ +"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries. + +Revision ID: ee827d1e1974 +Revises: 7ead443ba911 +Create Date: 2018-11-05 09:49:40.226368 + +""" + +# revision identifiers, used by Alembic. +revision = 'ee827d1e1974' +down_revision = '7ead443ba911' + +from alembic import op +from sqlalchemy.exc import ProgrammingError + +def upgrade(): + connection = op.get_bind() + connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") + + op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'}, + postgresql_using='gin') + op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False, + postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin') + op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'}, + postgresql_using='gin') + + +def downgrade(): + op.drop_index('ix_domains_name', table_name='domains') + op.drop_index('ix_certificates_name', table_name='certificates') + op.drop_index('ix_certificates_cn', table_name='certificates') diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index ddf8d875..cd88ebc8 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -8,24 +8,21 @@ .. moduleauthor:: Kevin Glisson """ -from itertools import groupby from collections import defaultdict +from datetime import timedelta +from itertools import groupby import arrow -from datetime import timedelta from flask import current_app - from sqlalchemy import and_ from lemur import database +from lemur.certificates.models import Certificate +from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.common.utils import windowed_query from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS from lemur.extensions import metrics, sentry -from lemur.common.utils import windowed_query - -from lemur.certificates.schemas import certificate_notification_output_schema -from lemur.certificates.models import Certificate from lemur.pending_certificates.schemas import pending_certificate_output_schema - from lemur.plugins import plugins from lemur.plugins.utils import get_plugin_option @@ -74,10 +71,11 @@ def get_eligible_certificates(exclude=None): notification_groups = [] for certificate in items: - notification = needs_notification(certificate) + notifications = needs_notification(certificate) - if notification: - notification_groups.append((notification, certificate)) + if notifications: + for notification in notifications: + notification_groups.append((notification, certificate)) # group by notification for notification, items in groupby(notification_groups, lambda x: x[0].label): @@ -133,11 +131,21 @@ def send_expiration_notifications(exclude): notification_data.append(cert_data) security_data.append(cert_data) + notification_recipient = get_plugin_option('recipients', notification.options) + if notification_recipient: + notification_recipient = notification_recipient.split(",") + if send_notification('expiration', notification_data, [owner], notification): success += 1 else: failure += 1 + if notification_recipient and owner != notification_recipient and security_email != notification_recipient: + if send_notification('expiration', notification_data, notification_recipient, notification): + success += 1 + else: + failure += 1 + if send_notification('expiration', security_data, security_email, notification): success += 1 else: @@ -228,6 +236,8 @@ def needs_notification(certificate): now = arrow.utcnow() days = (certificate.not_after - now).days + notifications = [] + for notification in certificate.notifications: if not notification.active or not notification.options: return @@ -248,4 +258,5 @@ def needs_notification(certificate): raise Exception("Invalid base unit for expiration interval: {0}".format(unit)) if days == interval: - return notification + notifications.append(notification) + return notifications diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index 1261177d..7dc8e602 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import relationship from sqlalchemy_utils import JSONType from sqlalchemy_utils.types.arrow import ArrowType -from lemur.certificates.models import get_or_increase_name +from lemur.certificates.models import get_sequence from lemur.common import defaults, utils from lemur.database import db from lemur.models import pending_cert_source_associations, \ @@ -19,6 +19,28 @@ from lemur.models import pending_cert_source_associations, \ from lemur.utils import Vault +def get_or_increase_name(name, serial): + certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(name))).all() + + if not certificates: + return name + + serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper()) + certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(serial_name))).all() + + if not certificates: + return serial_name + + ends = [0] + root, end = get_sequence(serial_name) + for cert in certificates: + root, end = get_sequence(cert.name) + if end: + ends.append(end) + + return '{0}-{1}'.format(root, max(ends) + 1) + + class PendingCertificate(db.Model): __tablename__ = 'pending_certs' id = Column(Integer, primary_key=True) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 9bab3a65..5d419f7f 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -5,7 +5,7 @@ import dns.exception import dns.name import dns.query import dns.resolver -from dyn.tm.errors import DynectCreateError +from dyn.tm.errors import DynectCreateError, DynectGetError from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app @@ -119,7 +119,11 @@ def delete_txt_record(change_id, account_number, domain, token): zone = Zone(zone_name) node = Node(zone_name, fqdn) - all_txt_records = node.get_all_records_by_type('TXT') + try: + all_txt_records = node.get_all_records_by_type('TXT') + except DynectGetError: + # No Text Records remain or host is not in the zone anymore because all records have been deleted. + return for txt_record in all_txt_records: if txt_record.txtdata == ("{}".format(token)): current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn)) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 3f0e8314..66295ed2 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -44,7 +44,11 @@ class AuthorizationRecord(object): class AcmeHandler(object): def __init__(self): self.dns_providers_for_domain = {} - self.all_dns_providers = dns_provider_service.get_all_dns_providers() + try: + self.all_dns_providers = dns_provider_service.get_all_dns_providers() + except Exception as e: + current_app.logger.error("Unable to fetch DNS Providers: {}".format(e)) + self.all_dns_providers = [] def find_dns_challenge(self, authorizations): dns_challenges = [] @@ -211,12 +215,18 @@ class AcmeHandler(object): :return: dns_providers: List of DNS providers that have the correct zone. """ self.dns_providers_for_domain[domain] = [] + match_length = 0 for dns_provider in self.all_dns_providers: if not dns_provider.domains: continue for name in dns_provider.domains: if domain.endswith("." + name): - self.dns_providers_for_domain[domain].append(dns_provider) + if len(name) > match_length: + self.dns_providers_for_domain[domain] = [dns_provider] + match_length = len(name) + elif len(name) == match_length: + self.dns_providers_for_domain[domain].append(dns_provider) + return self.dns_providers_for_domain def finalize_authorizations(self, acme_client, authorizations): @@ -329,9 +339,10 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) - self.acme = AcmeHandler() def get_dns_provider(self, type): + self.acme = AcmeHandler() + provider_types = { 'cloudflare': cloudflare, 'dyn': dyn, @@ -343,12 +354,14 @@ class ACMEIssuerPlugin(IssuerPlugin): return provider def get_all_zones(self, dns_provider): + self.acme = AcmeHandler() dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) return dns_provider_plugin.get_zones(account_number=account_number) def get_ordered_certificate(self, pending_cert): + self.acme = AcmeHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -384,6 +397,7 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): + self.acme = AcmeHandler() pending = [] certs = [] for pending_cert in pending_certs: @@ -466,6 +480,7 @@ class ACMEIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ + self.acme = AcmeHandler() authority = issuer_options.get('authority') create_immediately = issuer_options.get('create_immediately', False) acme_client, registration = self.acme.setup_acme_client(authority) diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 4c4ce97f..b4391dd8 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): """ Get a listener ARN from an endpoint. @@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_elbs(**kwargs): """ Fetches one page elb objects for a given account and region. @@ -123,7 +123,7 @@ def get_elbs(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_elbs_v2(**kwargs): """ Fetches one page of elb objects for a given account and region. @@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_listeners_v2(**kwargs): """ Fetches one page of listener objects for a given elb arn. @@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs): """ Fetching all policies currently associated with an ELB. @@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_ssl_policies_v2(policy_names, **kwargs): """ Fetching all policies currently associated with an ELB. @@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_load_balancer_types(policies, **kwargs): """ Describe the policies with policy details. @@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def attach_certificate(name, port, certificate_id, **kwargs): """ Attaches a certificate to a listener, throws exception @@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def attach_certificate_v2(listener_arn, port, certificates, **kwargs): """ Attaches a certificate to a listener, throws exception diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index b2a07798..49816c2b 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): """ Upload a certificate to AWS @@ -64,6 +64,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): :param path: :return: """ + assert isinstance(private_key, str) client = kwargs.pop('client') if not path or path == '/': @@ -72,8 +73,6 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): name = name + '-' + path.strip('/') try: - if isinstance(private_key, bytes): - private_key = private_key.decode("utf-8") if cert_chain: return client.upload_server_certificate( Path=path, @@ -95,7 +94,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def delete_cert(cert_name, **kwargs): """ Delete a certificate from AWS @@ -112,7 +111,7 @@ def delete_cert(cert_name, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_certificate(name, **kwargs): """ Retrieves an SSL certificate. @@ -126,7 +125,7 @@ def get_certificate(name, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_certificates(**kwargs): """ Fetches one page of certificate objects for a given account. diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index d959cfdc..1c2607a5 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -35,8 +35,8 @@ from flask import current_app from lemur.plugins import lemur_aws as aws -from lemur.plugins.lemur_aws import iam, s3, elb, ec2 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin +from lemur.plugins.lemur_aws import iam, s3, elb, ec2 def get_region_from_dns(dns): @@ -163,7 +163,7 @@ class AWSDestinationPlugin(DestinationPlugin): 'name': 'accountNumber', 'type': 'str', 'required': True, - 'validation': '/^[0-9]{12,12}$/', + 'validation': '[0-9]{12}', 'helpMessage': 'Must be a valid AWS account number!', }, { @@ -279,14 +279,14 @@ class S3DestinationPlugin(ExportDestinationPlugin): 'name': 'bucket', 'type': 'str', 'required': True, - 'validation': '/^$|\s+/', + 'validation': '[0-9a-z.-]{3,63}', 'helpMessage': 'Must be a valid S3 bucket name!', }, { 'name': 'accountNumber', 'type': 'str', 'required': True, - 'validation': '/^[0-9]{12,12}$/', + 'validation': '[0-9]{12}', 'helpMessage': 'A valid AWS account number with permission to access S3', }, { @@ -308,7 +308,6 @@ class S3DestinationPlugin(ExportDestinationPlugin): 'name': 'prefix', 'type': 'str', 'required': False, - 'validation': '/^$|\s+/', 'helpMessage': 'Must be a valid S3 object prefix!', } ] diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index 001ea2c8..6253ad7a 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -9,14 +9,22 @@ from functools import wraps import boto3 +from botocore.config import Config from flask import current_app +config = Config( + retries=dict( + max_attempts=20 + ) +) + + def sts_client(service, service_type='client'): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): - sts = boto3.client('sts') + sts = boto3.client('sts', config=config) arn = 'arn:aws:iam::{0}:role/{1}'.format( kwargs.pop('account_number'), current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur') @@ -31,7 +39,8 @@ def sts_client(service, service_type='client'): region_name=kwargs.pop('region', 'us-east-1'), aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] + aws_session_token=role['Credentials']['SessionToken'], + config=config ) kwargs['client'] = client elif service_type == 'resource': @@ -40,7 +49,8 @@ def sts_client(service, service_type='client'): region_name=kwargs.pop('region', 'us-east-1'), aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] + aws_session_token=role['Credentials']['SessionToken'], + config=config ) kwargs['resource'] = resource return f(*args, **kwargs) diff --git a/lemur/plugins/lemur_aws/tests/test_elb.py b/lemur/plugins/lemur_aws/tests/test_elb.py index e34b66de..7facc4dd 100644 --- a/lemur/plugins/lemur_aws/tests/test_elb.py +++ b/lemur/plugins/lemur_aws/tests/test_elb.py @@ -4,7 +4,7 @@ from moto import mock_sts, mock_elb @mock_sts() @mock_elb() -def test_get_all_elbs(app): +def test_get_all_elbs(app, aws_credentials): from lemur.plugins.lemur_aws.elb import get_all_elbs client = boto3.client('elb', region_name='us-east-1') diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index 030f290a..4bfefc85 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -10,6 +10,9 @@ import json import requests +import base64 +import hmac +import hashlib from flask import current_app @@ -48,6 +51,21 @@ class CfsslIssuerPlugin(IssuerPlugin): data = {'certificate_request': csr} data = json.dumps(data) + try: + hex_key = current_app.config.get('CFSSL_KEY') + key = bytes.fromhex(hex_key) + except (ValueError, NameError): + # unable to find CFSSL_KEY in config, continue using normal sign method + pass + else: + data = data.encode() + + token = base64.b64encode(hmac.new(key, data, digestmod=hashlib.sha256).digest()) + data = base64.b64encode(data) + + data = json.dumps({'token': token.decode('utf-8'), 'request': data.decode('utf-8')}) + + url = "{0}{1}".format(current_app.config.get('CFSSL_URL'), '/api/v1/cfssl/authsign') response = self.session.post(url, data=data.encode(encoding='utf_8', errors='strict')) if response.status_code > 399: metrics.send('cfssl_create_certificate_failure', 'counter', 1) diff --git a/lemur/plugins/lemur_cryptography/plugin.py b/lemur/plugins/lemur_cryptography/plugin.py index fe9d7bb3..97060391 100644 --- a/lemur/plugins/lemur_cryptography/plugin.py +++ b/lemur/plugins/lemur_cryptography/plugin.py @@ -14,6 +14,7 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization +from lemur.common.utils import parse_private_key from lemur.plugins.bases import IssuerPlugin from lemur.plugins import lemur_cryptography as cryptography_issuer @@ -40,7 +41,8 @@ def issue_certificate(csr, options, private_key=None): if options.get("authority"): # Issue certificate signed by an existing lemur_certificates authority issuer_subject = options['authority'].authority_certificate.subject - issuer_private_key = options['authority'].authority_certificate.private_key + assert private_key is None, "Private would be ignored, authority key used instead" + private_key = options['authority'].authority_certificate.private_key chain_cert_pem = options['authority'].authority_certificate.body authority_key_identifier_public = options['authority'].authority_certificate.public_key authority_key_identifier_subject = x509.SubjectKeyIdentifier.from_public_key(authority_key_identifier_public) @@ -52,7 +54,6 @@ def issue_certificate(csr, options, private_key=None): else: # Issue certificate that is self-signed (new lemur_certificates root authority) issuer_subject = csr.subject - issuer_private_key = private_key chain_cert_pem = "" authority_key_identifier_public = csr.public_key() authority_key_identifier_subject = None @@ -112,11 +113,7 @@ def issue_certificate(csr, options, private_key=None): # FIXME: Not implemented in lemur/schemas.py yet https://github.com/Netflix/lemur/issues/662 pass - private_key = serialization.load_pem_private_key( - bytes(str(issuer_private_key).encode('utf-8')), - password=None, - backend=default_backend() - ) + private_key = parse_private_key(private_key) cert = builder.sign(private_key, hashes.SHA256(), default_backend()) cert_pem = cert.public_bytes( diff --git a/lemur/plugins/lemur_csr/plugin.py b/lemur/plugins/lemur_csr/plugin.py index e06035d1..13f42084 100644 --- a/lemur/plugins/lemur_csr/plugin.py +++ b/lemur/plugins/lemur_csr/plugin.py @@ -38,14 +38,9 @@ def create_csr(cert, chain, csr_tmp, key): :param csr_tmp: :param key: """ - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') - - if isinstance(key, bytes): - key = key.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) + assert isinstance(key, str) with mktempfile() as key_tmp: with open(key_tmp, 'w') as f: diff --git a/lemur/plugins/lemur_java/plugin.py b/lemur/plugins/lemur_java/plugin.py index 151794da..5aab5342 100644 --- a/lemur/plugins/lemur_java/plugin.py +++ b/lemur/plugins/lemur_java/plugin.py @@ -59,11 +59,8 @@ def split_chain(chain): def create_truststore(cert, chain, jks_tmp, alias, passphrase): - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) with mktempfile() as cert_tmp: with open(cert_tmp, 'w') as f: @@ -98,14 +95,9 @@ def create_truststore(cert, chain, jks_tmp, alias, passphrase): def create_keystore(cert, chain, jks_tmp, key, alias, passphrase): - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') - - if isinstance(key, bytes): - key = key.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) + assert isinstance(key, str) # Create PKCS12 keystore from private key and public certificate with mktempfile() as cert_tmp: diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index ee466596..30b864eb 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -11,31 +11,37 @@ .. moduleauthor:: Mikhail Khodorovskiy """ import base64 -import os -import urllib -import requests import itertools +import os -from lemur.certificates.models import Certificate +import requests +from flask import current_app + +from lemur.common.defaults import common_name +from lemur.common.utils import parse_certificate from lemur.plugins.bases import DestinationPlugin DEFAULT_API_VERSION = 'v1' def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): - # _resolve_uri(k8s_base_uri, namespace, kind, name, api_ver=DEFAULT_API_VERSION) url = _resolve_uri(k8s_base_uri, namespace, kind) + current_app.logger.debug("K8S POST request URL: %s", url) create_resp = k8s_api.post(url, json=data) + current_app.logger.debug("K8S POST response: %s", create_resp) if 200 <= create_resp.status_code <= 299: return None - - elif create_resp.json()['reason'] != 'AlreadyExists': + elif create_resp.json().get('reason', '') != 'AlreadyExists': return create_resp.content - update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data) + url = _resolve_uri(k8s_base_uri, namespace, kind, name) + current_app.logger.debug("K8S PUT request URL: %s", url) + + update_resp = k8s_api.put(url, json=data) + current_app.logger.debug("K8S PUT response: %s", update_resp) if not 200 <= update_resp.status_code <= 299: return update_resp.content @@ -43,11 +49,12 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): return -def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,): +def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION): api_group = 'api' if '/' in api_ver: api_group = 'apis' - return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ('/' + namespace if namespace else '') + return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ( + '/' + namespace if namespace else '') def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION): @@ -61,6 +68,41 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V ])) +# Performs Base64 encoding of string to string using the base64.b64encode() function +# which encodes bytes to bytes. +def base64encode(string): + return base64.b64encode(string.encode()).decode() + + +def build_secret(secret_format, secret_name, body, private_key, cert_chain): + secret = { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'type': 'Opaque', + 'metadata': { + 'name': secret_name, + } + } + if secret_format == 'Full': + secret['data'] = { + 'combined.pem': base64encode('%s\n%s' % (body, private_key)), + 'ca.crt': base64encode(cert_chain), + 'service.key': base64encode(private_key), + 'service.crt': base64encode(body), + } + if secret_format == 'TLS': + secret['type'] = 'kubernetes.io/tls' + secret['data'] = { + 'tls.crt': base64encode(cert_chain), + 'tls.key': base64encode(private_key) + } + if secret_format == 'Certificate': + secret['data'] = { + 'tls.crt': base64encode(cert_chain), + } + return secret + + class KubernetesDestinationPlugin(DestinationPlugin): title = 'Kubernetes' slug = 'kubernetes-destination' @@ -70,35 +112,81 @@ class KubernetesDestinationPlugin(DestinationPlugin): author_url = 'https://github.com/mik373/lemur' options = [ + { + 'name': 'secretNameFormat', + 'type': 'str', + 'required': False, + # Validation is difficult. This regex is used by kubectl to validate secret names: + # [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)* + # Allowing the insertion of "{common_name}" (or any other such placeholder} + # at any point in the string proved very challenging and had a tendency to + # cause my browser to hang. The specified expression will allow any valid string + # but will also accept many invalid strings. + 'validation': '(?:[a-z0-9.-]|\\{common_name\\})+', + 'helpMessage': 'Must be a valid secret name, possibly including "{common_name}"', + 'default': '{common_name}' + }, { 'name': 'kubernetesURL', 'type': 'str', - 'required': True, - 'validation': '@(https?|http)://(-\.)?([^\s/?\.#-]+\.?)+(/[^\s]*)?$@iS', + 'required': False, + 'validation': 'https?://[a-zA-Z0-9.-]+(?::[0-9]+)?', 'helpMessage': 'Must be a valid Kubernetes server URL!', + 'default': 'https://kubernetes.default' }, { 'name': 'kubernetesAuthToken', 'type': 'str', - 'required': True, - 'validation': '/^$|\s+/', + 'required': False, + 'validation': '[0-9a-zA-Z-_.]+', 'helpMessage': 'Must be a valid Kubernetes server Token!', }, { - 'name': 'kubernetesServerCertificate', + 'name': 'kubernetesAuthTokenFile', 'type': 'str', - 'required': True, - 'validation': '/^$|\s+/', + 'required': False, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/var/run/secrets/kubernetes.io/serviceaccount/token' + }, + { + 'name': 'kubernetesServerCertificate', + 'type': 'textarea', + 'required': False, + 'validation': '-----BEGIN CERTIFICATE-----[a-zA-Z0-9/+\\s\\r\\n]+-----END CERTIFICATE-----', 'helpMessage': 'Must be a valid Kubernetes server Certificate!', }, + { + 'name': 'kubernetesServerCertificateFile', + 'type': 'str', + 'required': False, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' + }, { 'name': 'kubernetesNamespace', 'type': 'str', - 'required': True, - 'validation': '/^$|\s+/', + 'required': False, + 'validation': '[a-z0-9]([-a-z0-9]*[a-z0-9])?', 'helpMessage': 'Must be a valid Kubernetes Namespace!', }, - + { + 'name': 'kubernetesNamespaceFile', + 'type': 'str', + 'required': False, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/var/run/secrets/kubernetes.io/serviceaccount/namespace' + }, + { + 'name': 'secretFormat', + 'type': 'select', + 'required': True, + 'available': ['Full', 'TLS', 'Certificate'], + 'helpMessage': 'The type of Secret to create.', + 'default': 'Full' + } ] def __init__(self, *args, **kwargs): @@ -106,56 +194,91 @@ class KubernetesDestinationPlugin(DestinationPlugin): def upload(self, name, body, private_key, cert_chain, options, **kwargs): - k8_bearer = self.get_option('kubernetesAuthToken', options) - k8_cert = self.get_option('kubernetesServerCertificate', options) - k8_namespace = self.get_option('kubernetesNamespace', options) - k8_base_uri = self.get_option('kubernetesURL', options) + try: + k8_base_uri = self.get_option('kubernetesURL', options) + secret_format = self.get_option('secretFormat', options) + k8s_api = K8sSession( + self.k8s_bearer(options), + self.k8s_cert(options) + ) + cn = common_name(parse_certificate(body)) + secret_name_format = self.get_option('secretNameFormat', options) + secret_name = secret_name_format.format(common_name=cn) + secret = build_secret(secret_format, secret_name, body, private_key, cert_chain) + err = ensure_resource( + k8s_api, + k8s_base_uri=k8_base_uri, + namespace=self.k8s_namespace(options), + kind="secret", + name=secret_name, + data=secret + ) - k8s_api = K8sSession(k8_bearer, k8_cert) - - cert = Certificate(body=body) - - # in the future once runtime properties can be passed-in - use passed-in secret name - secret_name = 'certs-' + urllib.quote_plus(cert.name) - - err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={ - 'apiVersion': 'v1', - 'kind': 'Secret', - 'metadata': { - 'name': secret_name, - }, - 'data': { - 'combined.pem': base64.b64encode(body + private_key), - 'ca.crt': base64.b64encode(cert_chain), - 'service.key': base64.b64encode(private_key), - 'service.crt': base64.b64encode(body), - } - }) + except Exception as e: + current_app.logger.exception("Exception in upload: {}".format(e), exc_info=True) + raise if err is not None: + current_app.logger.error("Error deploying resource: %s", err) raise Exception("Error uploading secret: " + err) + def k8s_bearer(self, options): + bearer = self.get_option('kubernetesAuthToken', options) + if not bearer: + bearer_file = self.get_option('kubernetesAuthTokenFile', options) + with open(bearer_file, "r") as file: + bearer = file.readline() + if bearer: + current_app.logger.debug("Using token read from %s", bearer_file) + else: + raise Exception("Unable to locate token in options or from %s", bearer_file) + else: + current_app.logger.debug("Using token from options") + return bearer + + def k8s_cert(self, options): + cert_file = self.get_option('kubernetesServerCertificateFile', options) + cert = self.get_option('kubernetesServerCertificate', options) + if cert: + cert_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert') + with open(cert_file, "w") as text_file: + text_file.write(cert) + current_app.logger.debug("Using certificate from options") + else: + current_app.logger.debug("Using certificate from %s", cert_file) + return cert_file + + def k8s_namespace(self, options): + namespace = self.get_option('kubernetesNamespace', options) + if not namespace: + namespace_file = self.get_option('kubernetesNamespaceFile', options) + with open(namespace_file, "r") as file: + namespace = file.readline() + if namespace: + current_app.logger.debug("Using namespace %s from %s", namespace, namespace_file) + else: + raise Exception("Unable to locate namespace in options or from %s", namespace_file) + else: + current_app.logger.debug("Using namespace %s from options", namespace) + return namespace + class K8sSession(requests.Session): - def __init__(self, bearer, cert): + def __init__(self, bearer, cert_file): super(K8sSession, self).__init__() self.headers.update({ 'Authorization': 'Bearer %s' % bearer }) - k8_ca = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert') + self.verify = cert_file - with open(k8_ca, "w") as text_file: - text_file.write(cert) - - self.verify = k8_ca - - def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=30, allow_redirects=True, proxies=None, - hooks=None, stream=None, verify=None, cert=None, json=None): + def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, + timeout=30, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, + json=None): """ This method overrides the default timeout to be 10s. """ - return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, - verify, cert, json) + return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, + allow_redirects, proxies, hooks, stream, verify, cert, json) diff --git a/lemur/plugins/lemur_openssl/plugin.py b/lemur/plugins/lemur_openssl/plugin.py index d50b4e43..9ddce925 100644 --- a/lemur/plugins/lemur_openssl/plugin.py +++ b/lemur/plugins/lemur_openssl/plugin.py @@ -44,14 +44,9 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase): :param alias: :param passphrase: """ - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') - - if isinstance(key, bytes): - key = key.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) + assert isinstance(key, str) with mktempfile() as key_tmp: with open(key_tmp, 'w') as f: diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 3e672a43..3f16f997 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -111,10 +111,19 @@ def process_options(options): data['subject_alt_names'] = ",".join(get_additional_names(options)) + if options.get('validity_end') > arrow.utcnow().replace(years=2): + raise Exception("Verisign issued certificates cannot exceed two years in validity") + if options.get('validity_end'): - period = get_default_issuance(options) - data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY") - data['validityPeriod'] = period + # VeriSign (Symantec) only accepts strictly smaller than 2 year end date + if options.get('validity_end') < arrow.utcnow().replace(years=2).replace(days=-1): + period = get_default_issuance(options) + data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY") + data['validityPeriod'] = period + else: + # allowing Symantec website setting the end date, given the validity period + data['validityPeriod'] = str(get_default_issuance(options)) + options.pop('validity_end', None) elif options.get('validity_years'): if options['validity_years'] in [1, 2]: diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 1f2fd9b0..0ab8c9f8 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -93,6 +93,7 @@ def sync(source_strings): ) sentry.captureException() + metrics.send('source_sync_fail', 'counter', 1, metric_tags={'source': source.label, 'status': status}) metrics.send('source_sync', 'counter', 1, metric_tags={'source': source.label, 'status': status}) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 5002041c..47b7f02c 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import parse_certificate +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate from lemur.common.defaults import serial from lemur.plugins.base import plugins @@ -131,7 +131,8 @@ def sync_certificates(source, user): if not exists: cert = parse_certificate(certificate['body']) - exists = certificate_service.get_by_serial(serial(cert)) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) if not certificate.get('owner'): certificate['owner'] = user.email diff --git a/lemur/static/app/angular/certificates/view/view.tpl.html b/lemur/static/app/angular/certificates/view/view.tpl.html index ba17ffa6..28b4e08e 100644 --- a/lemur/static/app/angular/certificates/view/view.tpl.html +++ b/lemur/static/app/angular/certificates/view/view.tpl.html @@ -83,6 +83,8 @@
+
Distinguished Name
+
{{ certificate.distinguishedName }}
Certificate Authority
{{ certificate.authority ? certificate.authority.name : "Imported" }} ({{ certificate.issuer }})
Serial
diff --git a/lemur/static/app/angular/destinations/destination/destination.tpl.html b/lemur/static/app/angular/destinations/destination/destination.tpl.html index 1d240dbb..f2771b49 100644 --- a/lemur/static/app/angular/destinations/destination/destination.tpl.html +++ b/lemur/static/app/angular/destinations/destination/destination.tpl.html @@ -47,7 +47,9 @@ - + +
+

{{ item.helpMessage }}

diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index c2b5d83d..bbe155cd 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -1,7 +1,7 @@ - # This is just Python which means you can inherit and tweak settings import os + _basedir = os.path.abspath(os.path.dirname(__file__)) THREADS_PER_PAGE = 8 @@ -78,14 +78,12 @@ DIGICERT_API_KEY = 'api-key' DIGICERT_ORG_ID = 111111 DIGICERT_ROOT = "ROOT" - VERISIGN_URL = 'http://example.com' VERISIGN_PEM_PATH = '~/' VERISIGN_FIRST_NAME = 'Jim' VERISIGN_LAST_NAME = 'Bob' VERSIGN_EMAIL = 'jim@example.com' - ACME_AWS_ACCOUNT_NUMBER = '11111111111' ACME_PRIVATE_KEY = ''' @@ -180,6 +178,7 @@ ACME_URL = 'https://acme-v01.api.letsencrypt.org' ACME_EMAIL = 'jim@example.com' ACME_TEL = '4088675309' ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org' +ACME_DISABLE_AUTORESOLVE = True LDAP_AUTH = True LDAP_BIND_URI = 'ldap://localhost' diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index fcd3005d..3790358e 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -3,19 +3,18 @@ import os import datetime import pytest from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask import current_app from flask_principal import identity_changed, Identity from lemur import create_app +from lemur.common.utils import parse_private_key from lemur.database import db as _db from lemur.auth.service import create_token -from lemur.tests.vectors import SAN_CERT_KEY +from lemur.tests.vectors import SAN_CERT_KEY, INTERMEDIATE_KEY from .factories import ApiKeyFactory, AuthorityFactory, NotificationFactory, DestinationFactory, \ CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, \ - RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory + RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory, CryptoAuthorityFactory def pytest_runtest_setup(item): @@ -91,6 +90,13 @@ def authority(session): return a +@pytest.fixture +def crypto_authority(session): + a = CryptoAuthorityFactory() + session.commit() + return a + + @pytest.fixture def async_authority(session): a = AsyncAuthorityFactory() @@ -228,7 +234,12 @@ def logged_in_admin(session, app): @pytest.fixture def private_key(): - return load_pem_private_key(SAN_CERT_KEY.encode(), password=None, backend=default_backend()) + return parse_private_key(SAN_CERT_KEY) + + +@pytest.fixture +def issuer_private_key(): + return parse_private_key(INTERMEDIATE_KEY) @pytest.fixture @@ -240,3 +251,11 @@ def cert_builder(private_key): .public_key(private_key.public_key()) .not_valid_before(datetime.datetime(2017, 12, 22)) .not_valid_after(datetime.datetime(2040, 1, 1))) + + +@pytest.fixture(scope='function') +def aws_credentials(): + os.environ['AWS_ACCESS_KEY_ID'] = 'testing' + os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' + os.environ['AWS_SECURITY_TOKEN'] = 'testing' + os.environ['AWS_SESSION_TOKEN'] = 'testing' diff --git a/lemur/tests/factories.py b/lemur/tests/factories.py index cae2c354..3717c64d 100644 --- a/lemur/tests/factories.py +++ b/lemur/tests/factories.py @@ -168,6 +168,11 @@ class AsyncAuthorityFactory(AuthorityFactory): authority_certificate = SubFactory(CertificateFactory) +class CryptoAuthorityFactory(AuthorityFactory): + """Authority factory based on 'cryptography' plugin.""" + plugin = {'slug': 'cryptography-issuer'} + + class DestinationFactory(BaseFactory): """Destination factory.""" plugin_name = 'test-destination' diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 5c60342e..83bf49e5 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -18,7 +18,7 @@ from lemur.domains.models import Domain from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \ - INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY + INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR def test_get_or_increase_name(session, certificate): @@ -448,6 +448,85 @@ def test_certificate_sensitive_name(client, authority, session, logged_in_user): assert errors['common_name'][0].startswith("Domain sensitive.example.com has been marked as sensitive") +def test_certificate_upload_schema_ok(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'name': 'Jane', + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'privateKey': SAN_CERT_KEY, + 'chain': INTERMEDIATE_CERT_STR, + 'external_id': '1234', + } + data, errors = CertificateUploadInputSchema().load(data) + assert not errors + + +def test_certificate_upload_schema_minimal(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + } + data, errors = CertificateUploadInputSchema().load(data) + assert not errors + + +def test_certificate_upload_schema_long_chain(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'chain': INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR + } + data, errors = CertificateUploadInputSchema().load(data) + assert not errors + + +def test_certificate_upload_schema_invalid_body(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': 'Hereby I certify that this is a valid body', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'body': ['Public certificate presented is not valid.']} + + +def test_certificate_upload_schema_invalid_pkey(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'privateKey': 'Look at me Im a private key!!111', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'private_key': ['Private key presented is not valid.']} + + +def test_certificate_upload_schema_invalid_chain(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'body': SAN_CERT_STR, + 'chain': 'CHAINSAW', + 'owner': 'pwner@example.com', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'chain': ['Public certificate presented is not valid.']} + + +def test_certificate_upload_schema_wrong_pkey(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'body': SAN_CERT_STR, + 'privateKey': ROOTCA_KEY, + 'chain': INTERMEDIATE_CERT_STR, + 'owner': 'pwner@example.com', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'_schema': ['Private key does not match certificate.']} + + def test_create_basic_csr(client): csr_config = dict( common_name='example.com', @@ -545,8 +624,11 @@ def test_create_certificate(issuer_plugin, authority, user): assert cert.name == 'ACustomName1' -def test_reissue_certificate(issuer_plugin, authority, certificate): +def test_reissue_certificate(issuer_plugin, crypto_authority, certificate, logged_in_user): from lemur.certificates.service import reissue_certificate + + # test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead. + certificate.authority = crypto_authority new_cert = reissue_certificate(certificate) assert new_cert @@ -570,7 +652,7 @@ def test_import(user): assert str(cert.not_after) == '2047-12-31T22:00:00+00:00' assert str(cert.not_before) == '2017-12-31T22:00:00+00:00' assert cert.issuer == 'LemurTrustUnittestsClass1CA2018' - assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2' + assert cert.name.startswith('SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231') cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, owner='joe@example.com', name='ACustomName2', creator=user['user']) assert cert.name == 'ACustomName2' @@ -620,6 +702,12 @@ def test_certificate_get_body(client): response_body = client.get(api.url_for(Certificates, certificate_id=1), headers=VALID_USER_HEADER_TOKEN).json assert response_body['serial'] == '211983098819107449768450703123665283596' assert response_body['serialHex'] == '9F7A75B39DAE4C3F9524C68B06DA6A0C' + assert response_body['distinguishedName'] == ('CN=LemurTrust Unittests Class 1 CA 2018,' + 'O=LemurTrust Enterprises Ltd,' + 'OU=Unittesting Operations Center,' + 'C=EE,' + 'ST=N/A,' + 'L=Earth') @pytest.mark.parametrize("token,status", [ diff --git a/lemur/tests/test_defaults.py b/lemur/tests/test_defaults.py index 918e1ab8..ffa19727 100644 --- a/lemur/tests/test_defaults.py +++ b/lemur/tests/test_defaults.py @@ -1,3 +1,7 @@ +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes + from .vectors import SAN_CERT, WILDCARD_CERT, INTERMEDIATE_CERT @@ -41,12 +45,14 @@ def test_cert_issuer(client): def test_text_to_slug(client): from lemur.common.defaults import text_to_slug assert text_to_slug('test - string') == 'test-string' + assert text_to_slug('test - string', '') == 'teststring' # Accented characters are decomposed assert text_to_slug('föö bär') == 'foo-bar' # Melt away the Unicode Snowman assert text_to_slug('\u2603') == '' assert text_to_slug('\u2603test\u2603') == 'test' assert text_to_slug('snow\u2603man') == 'snow-man' + assert text_to_slug('snow\u2603man', '') == 'snowman' # IDNA-encoded domain names should be kept as-is assert text_to_slug('xn--i1b6eqas.xn--xmpl-loa9b3671b.com') == 'xn--i1b6eqas.xn--xmpl-loa9b3671b.com' @@ -75,3 +81,29 @@ def test_create_name(client): datetime(2015, 5, 12, 0, 0, 0), False ) == 'xn--mnchen-3ya.de-VertrauenswurdigAutoritat-20150507-20150512' + + +def test_issuer(client, cert_builder, issuer_private_key): + from lemur.common.defaults import issuer + + assert issuer(INTERMEDIATE_CERT) == 'LemurTrustUnittestsRootCA2018' + + # We need to override builder's issuer name + cert_builder._issuer_name = None + # Unicode issuer name + cert = (cert_builder + .issuer_name(x509.Name([x509.NameAttribute(x509.NameOID.COMMON_NAME, 'Vertrauenswürdig Autorität')])) + .sign(issuer_private_key, hashes.SHA256(), default_backend())) + assert issuer(cert) == 'VertrauenswurdigAutoritat' + + # Fallback to 'Organization' field when issuer CN is missing + cert = (cert_builder + .issuer_name(x509.Name([x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, 'No Such Organization')])) + .sign(issuer_private_key, hashes.SHA256(), default_backend())) + assert issuer(cert) == 'NoSuchOrganization' + + # Missing issuer name + cert = (cert_builder + .issuer_name(x509.Name([])) + .sign(issuer_private_key, hashes.SHA256(), default_backend())) + assert issuer(cert) == 'Unknown' diff --git a/lemur/tests/test_pending_certificates.py b/lemur/tests/test_pending_certificates.py index 567159e1..7accf7d9 100644 --- a/lemur/tests/test_pending_certificates.py +++ b/lemur/tests/test_pending_certificates.py @@ -2,11 +2,10 @@ import json import pytest +from lemur.pending_certificates.views import * # noqa from .vectors import CSR_STR, INTERMEDIATE_CERT_STR, VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, \ VALID_USER_HEADER_TOKEN, WILDCARD_CERT_STR -from lemur.pending_certificates.views import * # noqa - def test_increment_attempt(pending_certificate): from lemur.pending_certificates.service import increment_attempt @@ -17,7 +16,8 @@ def test_increment_attempt(pending_certificate): def test_create_pending_certificate(async_issuer_plugin, async_authority, user): from lemur.certificates.service import create - pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], common_name='ACommonName') + pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], + common_name='ACommonName') assert pending_cert.external_id == '12345' diff --git a/lemur/tests/test_validators.py b/lemur/tests/test_validators.py index 815b7c9d..c3d5357d 100644 --- a/lemur/tests/test_validators.py +++ b/lemur/tests/test_validators.py @@ -1,16 +1,28 @@ -import pytest from datetime import datetime -from .vectors import SAN_CERT_KEY + +import pytest from marshmallow.exceptions import ValidationError +from lemur.common.utils import parse_private_key +from lemur.common.validators import verify_private_key_match +from lemur.tests.vectors import INTERMEDIATE_CERT, SAN_CERT, SAN_CERT_KEY + def test_private_key(session): - from lemur.common.validators import private_key + parse_private_key(SAN_CERT_KEY) - private_key(SAN_CERT_KEY) + with pytest.raises(ValueError): + parse_private_key('invalid_private_key') + + +def test_validate_private_key(session): + key = parse_private_key(SAN_CERT_KEY) + + verify_private_key_match(key, SAN_CERT) with pytest.raises(ValidationError): - private_key('invalid_private_key') + # Wrong key for certificate + verify_private_key_match(key, INTERMEDIATE_CERT) def test_sub_alt_type(session): diff --git a/requirements-dev.in b/requirements-dev.in index de8b60d3..84104679 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ # Run `make up-reqs` to update pinned dependencies in requirement text files -flake8>=3.2,<4.0 +flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors. pre-commit invoke twine diff --git a/requirements-dev.txt b/requirements-dev.txt index c473aa56..ac35f3e9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,36 +4,34 @@ # # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # -aspy.yaml==1.1.1 # via pre-commit -bleach==3.0.2 # via readme-renderer -cached-property==1.5.1 # via pre-commit -certifi==2018.10.15 # via requests -cffi==1.11.5 # via cmarkgfm -cfgv==1.1.0 # via pre-commit +aspy.yaml==1.1.2 # via pre-commit +bleach==3.1.0 # via readme-renderer +certifi==2018.11.29 # via requests +cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests -cmarkgfm==0.4.2 # via readme-renderer docutils==0.14 # via readme-renderer flake8==3.5.0 -future==0.16.0 # via readme-renderer -identify==1.1.7 # via pre-commit -idna==2.7 # via requests +identify==1.2.1 # via pre-commit +idna==2.8 # via requests +importlib-metadata==0.8 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.2 -pkginfo==1.4.2 # via twine -pre-commit==1.11.2 +nodeenv==1.3.3 +pkginfo==1.5.0.1 # via twine +pre-commit==1.14.2 pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 -pygments==2.2.0 # via readme-renderer +pygments==2.3.1 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit -readme-renderer==22.0 # via twine -requests-toolbelt==0.8.0 # via twine -requests==2.20.0 # via requests-toolbelt, twine -six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer +readme-renderer==24.0 # via twine +requests-toolbelt==0.9.0 # via twine +requests==2.21.0 # via requests-toolbelt, twine +six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.28.1 # via twine +tqdm==4.30.0 # via twine twine==1.12.1 -urllib3==1.24 # via requests -virtualenv==16.0.0 # via pre-commit +urllib3==1.24.1 # via requests +virtualenv==16.3.0 # via pre-commit webencodings==0.5.1 # via bleach +zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.in b/requirements-docs.in index cf598240..d04d510b 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -4,4 +4,4 @@ -r requirements.txt sphinx sphinxcontrib-httpdomain -sphinx-rtd-theme \ No newline at end of file +sphinx-rtd-theme diff --git a/requirements-docs.txt b/requirements-docs.txt index 6b49b64d..15085766 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,93 +4,93 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.27.1 +acme==0.30.2 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 -amqp==2.3.2 -aniso8601==3.0.2 -arrow==0.12.1 +alembic==1.0.7 +amqp==2.4.0 +aniso8601==4.1.0 +arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx -bcrypt==3.1.4 -billiard==3.5.0.4 +bcrypt==3.1.6 +billiard==3.5.0.5 blinker==1.4 -boto3==1.7.79 -botocore==1.10.84 +boto3==1.9.86 +botocore==1.12.86 celery[redis]==4.2.1 -certifi==2018.10.15 +certifi==2018.11.29 cffi==1.11.5 chardet==3.0.4 click==7.0 cloudflare==2.1.0 -cryptography==2.3.1 +cryptography==2.5 dnspython3==1.15.0 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.6 +flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.1.1 +flask-migrate==2.3.1 flask-principal==0.4.0 -flask-restful==0.3.6 +flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.3.2 -flask==0.12 -future==0.16.0 +flask==1.0.2 +future==0.17.1 gunicorn==19.9.0 -idna==2.7 +idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 -itsdangerous==1.0.0 +itsdangerous==1.1.0 jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.1 +kombu==4.2.2.post1 lockfile==0.12.2 mako==1.0.7 -markupsafe==1.0 -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.18.0 mock==2.0.0 ndg-httpsclient==0.5.1 -packaging==18.0 # via sphinx +packaging==19.0 # via sphinx paramiko==2.4.2 -pbr==5.0.0 +pbr==5.1.1 pem==18.2.0 -psycopg2==2.7.5 -pyasn1-modules==0.2.2 -pyasn1==0.4.4 +psycopg2==2.7.7 +pyasn1-modules==0.2.4 +pyasn1==0.4.5 pycparser==2.19 -pygments==2.2.0 # via sphinx -pyjwt==1.6.4 +pygments==2.3.1 # via sphinx +pyjwt==1.7.1 pynacl==1.3.0 -pyopenssl==18.0.0 -pyparsing==2.2.2 # via packaging +pyopenssl==19.0.0 +pyparsing==2.3.1 # via packaging pyrfc3339==1.1 -python-dateutil==2.7.3 +python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.5 +pytz==2018.9 pyyaml==3.13 -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.8.0 -requests[security]==2.20.0 +requests-toolbelt==0.9.0 +requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.2 -sphinx==1.8.1 +sphinx==1.8.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 -tabulate==0.8.2 -urllib3==1.24 -vine==1.1.4 +sqlalchemy-utils==0.33.11 +sqlalchemy==1.2.17 +tabulate==0.8.3 +urllib3==1.24.1 +vine==1.2.0 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.in b/requirements-tests.in index efb4570a..02a2b0ae 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -4,7 +4,7 @@ coverage factory-boy Faker freezegun -moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813 +moto nose pyflakes pytest diff --git a/requirements-tests.txt b/requirements-tests.txt index a851e620..c326e951 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,59 +8,57 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -biscuits==0.1.1 # via responses -boto3==1.9.28 # via moto +boto3==1.9.86 # via moto boto==2.49.0 # via moto -botocore==1.12.28 # via boto3, moto, s3transfer -certifi==2018.10.15 # via requests +botocore==1.12.86 # via boto3, moto, s3transfer +certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask -cookies==2.2.1 # via moto -coverage==4.5.1 -cryptography==2.3.1 # via moto -docker-pycreds==0.3.0 # via docker -docker==3.5.1 # via moto +coverage==4.5.2 +cryptography==2.5 # via moto +docker-pycreds==0.4.0 # via docker +docker==3.7.0 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==0.9.2 +faker==1.0.2 flask==1.0.2 # via pytest-flask freezegun==0.3.11 -future==0.16.0 # via python-jose -idna==2.7 # via cryptography, requests -itsdangerous==1.0.0 # via flask +future==0.17.1 # via python-jose +idna==2.8 # via requests +itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto -jsonpickle==1.0 # via aws-xray-sdk -markupsafe==1.0 # via jinja2 +jsonpickle==1.1 # via aws-xray-sdk +markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==4.3.0 # via pytest -moto==1.3.4 +more-itertools==5.0.0 # via pytest +moto==1.3.7 nose==1.3.7 -pbr==5.0.0 # via mock -pluggy==0.8.0 # via pytest +pbr==5.1.1 # via mock +pluggy==0.8.1 # via pytest py==1.7.0 # via pytest -pyaml==17.12.1 # via moto +pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.6.6 # via python-jose -pyflakes==2.0.0 +pycryptodome==3.7.3 # via python-jose +pyflakes==2.1.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==3.9.1 -python-dateutil==2.7.3 # via botocore, faker, freezegun, moto +pytest==4.1.1 +python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.5 # via moto +pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 -requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.1 # via moto +requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses +responses==0.10.5 # via moto s3transfer==0.1.13 # via boto3 -six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client +six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.23 # via botocore, requests -websocket-client==0.53.0 # via docker +urllib3==1.24.1 # via botocore, requests +websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask -wrapt==1.10.11 # via aws-xray-sdk +wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.in b/requirements.in index a2b920c3..9824650b 100644 --- a/requirements.in +++ b/requirements.in @@ -4,22 +4,22 @@ acme alembic-autogenerate-enums arrow asyncpool -boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813 -botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813 +boto3 +botocore celery[redis] certifi CloudFlare cryptography dnspython3 dyn -Flask-Bcrypt==0.7.1 -Flask-Mail==0.9.1 -Flask-Migrate==2.1.1 -Flask-Principal==0.4.0 -Flask-RESTful==0.3.6 -Flask-Script==2.0.6 +Flask-Bcrypt +Flask-Mail +Flask-Migrate +Flask-Principal +Flask-RESTful +Flask-Script Flask-SQLAlchemy -Flask==0.12 +Flask Flask-Cors future gunicorn @@ -36,6 +36,7 @@ pyjwt pyOpenSSL python_ldap raven[flask] +redis<3 # redis>=3 is not compatible with celery requests retrying six diff --git a/requirements.txt b/requirements.txt index c8cb2e2e..c595e509 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,83 +4,83 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.27.1 +acme==0.30.2 alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 # via flask-migrate -amqp==2.3.2 # via kombu -aniso8601==3.0.2 # via flask-restful -arrow==0.12.1 +alembic==1.0.7 # via flask-migrate +amqp==2.4.0 # via kombu +aniso8601==4.1.0 # via flask-restful +arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 -bcrypt==3.1.4 # via flask-bcrypt, paramiko -billiard==3.5.0.4 # via celery +bcrypt==3.1.6 # via flask-bcrypt, paramiko +billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.79 -botocore==1.10.84 +boto3==1.9.86 +botocore==1.12.86 celery[redis]==4.2.1 -certifi==2018.10.15 +certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 -cryptography==2.3.1 +cryptography==2.5 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.6 +flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.1.1 +flask-migrate==2.3.1 flask-principal==0.4.0 -flask-restful==0.3.6 +flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.3.2 -flask==0.12 -future==0.16.0 +flask==1.0.2 +future==0.17.1 gunicorn==19.9.0 -idna==2.7 # via cryptography, requests +idna==2.8 # via requests inflection==0.3.1 -itsdangerous==1.0.0 # via flask +itsdangerous==1.1.0 # via flask jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.1 # via celery +kombu==4.2.2.post1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 # via jinja2, mako +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.18.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.0.0 # via mock +pbr==5.1.1 # via mock pem==18.2.0 -psycopg2==2.7.5 -pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap +psycopg2==2.7.7 +pyasn1-modules==0.2.4 # via python-ldap +pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi -pyjwt==1.6.4 +pyjwt==1.7.1 pynacl==1.3.0 # via paramiko -pyopenssl==18.0.0 +pyopenssl==19.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.3 # via alembic, arrow, botocore +python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare -raven[flask]==6.9.0 -redis==2.10.6 # via celery -requests-toolbelt==0.8.0 # via acme -requests[security]==2.20.0 +raven[flask]==6.10.0 +redis==2.10.6 +requests-toolbelt==0.9.0 # via acme +requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 -six==1.11.0 -sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.2 -urllib3==1.24 # via requests -vine==1.1.4 # via amqp +six==1.12.0 +sqlalchemy-utils==0.33.11 +sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.3 +urllib3==1.24.1 # via botocore, requests +vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0