Merge branch 'master' into get_by_attributes

.travis.yml

@@ -28,6 +28,7 @@ env:
 before_script:
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
+  - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
   - npm config set registry https://registry.npmjs.org
   - npm install -g bower
   - pip install --upgrade setuptools
@@ -45,4 +46,4 @@ after_success:
 
 notifications:
   email:
-    kglisson@netflix.com
+    ccastrapel@netflix.com

Makefile

@@ -43,6 +43,8 @@ reset-db:
 	dropdb lemur || true
 	@echo "--> Creating 'lemur' database"
 	createdb -E utf-8 lemur
+	@echo "--> Enabling pg_trgm extension"
+	psql lemur -c "create extension IF NOT EXISTS pg_trgm;"
 	@echo "--> Applying migrations"
 	lemur db upgrade
 
@@ -111,10 +113,10 @@ endif
 	@echo "--> Updating Python requirements"
 	pip install --upgrade pip
 	pip install --upgrade pip-tools
+	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
 	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
 	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
-	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	@echo "--> Done updating Python requirements"
 	@echo "--> Removing python-ldap from requirements-docs.txt"
 	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

docker/Dockerfile

@@ -0,0 +1,64 @@
+FROM alpine:3.8
+
+ARG VERSION
+ENV VERSION master
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    mkdir -p /opt/lemur /home/lemur/.lemur/ && \
+    curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    chmod +x /entrypoint && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
+    chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
+    
+WORKDIR /opt/lemur
+
+RUN npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
+    apk del build-dependencies
+
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/entrypoint

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
+  echo "Database vars not set"
+  exit 1
+fi
+
+export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+
+echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
+
+source /etc/profile
+
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
+
+echo " # Create Postgres trgm extension"
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
+echo " # Done"
+
+if [ -z "${SKIP_SSL}" ]; then
+  if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
+    openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
+  fi
+  mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+fi
+
+# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
+# echo "Creating config"
+# https://github.com/Netflix/lemur/issues/2257
+# python3 /opt/lemur/lemur/manage.py create_config
+# echo "Done"
+# fi
+
+echo " # Running init"
+su lemur -c "python3 /opt/lemur/lemur/manage.py init"
+echo " # Done"
+
+# echo "Creating user"
+# https://github.com/Netflix/lemur/issues/
+# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
+# echo "Done"
+
+cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
+cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
+cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
+
+echo " # Populating crontab"
+echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
+echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
+echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
+echo " # Done"
+
+exec "$@"

docker/nginx/default-ssl.conf

@@ -0,0 +1,37 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+    listen 80;
+    server_name  _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+   listen 443;
+   server_name  _;
+   access_log /dev/stdout;
+   error_log /dev/stderr;
+   ssl_certificate /etc/nginx/ssl/server.crt;
+   ssl_certificate_key /etc/nginx/ssl/server.key;
+   ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+   ssl_ciphers HIGH:!aNULL:!MD5;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/nginx/default.conf

@@ -0,0 +1,26 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+   listen      80;
+   access_log  /dev/stdout;
+   error_log   /dev/stderr;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/src/lemur.conf.py

@@ -0,0 +1,31 @@
+import os
+_basedir = os.path.abspath(os.path.dirname(__file__))
+
+CORS = os.environ.get("CORS") == "True"
+debug = os.environ.get("DEBUG") == "True"
+
+SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw=='))
+
+LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ=='))
+LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo='))
+
+LEMUR_WHITELISTED_DOMAINS = []
+
+LEMUR_EMAIL = ''
+LEMUR_SECURITY_TEAM_EMAIL = []
+
+
+LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
+LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
+LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
+LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+
+ACTIVE_PROVIDERS = []
+
+METRIC_PROVIDERS = []
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
+
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')

docker/supervisor.conf

@@ -0,0 +1,32 @@
+[supervisord]
+nodaemon=true
+user=root
+logfile=/dev/stdout
+logfile_maxbytes=0
+pidfile = /tmp/supervisord.pid
+
+[program:lemur]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
+user=lemur
+directory=/opt/lemur/lemur
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:cron]
+command=/usr/sbin/crond -f
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docs/administration.rst

@@ -324,6 +324,7 @@ Here is an example LDAP configuration stanza you can add to your config. Adjust
         LDAP_CACERT_FILE = '/opt/lemur/trusted.pem'
         LDAP_REQUIRED_GROUP = 'certificate-management-access'
         LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'}
+        LDAP_IS_ACTIVE_DIRECTORY = True
 
 
 The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN.
@@ -406,6 +407,17 @@ The following LDAP options are not required, however TLS is always recommended.
             LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'}
 
 
+.. data:: LDAP_IS_ACTIVE_DIRECTORY
+    :noindex:
+
+        When set to True, nested group memberships are supported, by searching for groups with the member:1.2.840.113556.1.4.1941 attribute set to the user DN.
+        When set to False, the list of groups will be determined by the 'memberof' attribute of the LDAP user logging in.
+
+        ::
+
+            LDAP_IS_ACTIVE_DIRECTORY = False
+
+
 Authentication Providers
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/quickstart/index.rst

@@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur:
 Installing Build Dependencies
 -----------------------------
 
-If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
+If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
 
 .. code-block:: bash
 
@@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config
     LEMUR_DEFAULT_ORGANIZATION
     LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
 
-Setup Postgres
+Set Up Postgres
 --------------
 
 For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
@@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
 .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 
-Setup a Reverse Proxy
+Set Up a Reverse Proxy
 ---------------------
 
-By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
 
 
 Proxying with Nginx

lemur/auth/ldap.py

@@ -41,6 +41,7 @@ class LdapPrincipal():
         self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
         self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
         self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
+        self.ldap_is_active_directory = current_app.config.get("LDAP_IS_ACTIVE_DIRECTORY", False)
         self.ldap_attrs = ['memberOf']
         self.ldap_client = None
         self.ldap_groups = None
@@ -168,11 +169,28 @@ class LdapPrincipal():
         except ldap.LDAPError as e:
             raise Exception("ldap error: {0}".format(e))
 
+        if self.ldap_is_active_directory:
+            # Lookup user DN, needed to search for group membership
+            userdn = self.ldap_client.search_s(self.ldap_base_dn,
+                                               ldap.SCOPE_SUBTREE, ldap_filter,
+                                               ['distinguishedName'])[0][1]['distinguishedName'][0]
+            userdn = userdn.decode('utf-8')
+            # Search all groups that have the userDN as a member
+            groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn)
+            lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn'])
+
+            # Create a list of group CN's from the result
+            self.ldap_groups = []
+            for group in lgroups:
+                (dn, values) = group
+                self.ldap_groups.append(values['cn'][0].decode('ascii'))
+        else:
             lgroups = self.ldap_client.search_s(self.ldap_base_dn,
                                                 ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
             # lgroups is a list of utf-8 encoded strings
             # convert to a single string of groups to allow matching
             self.ldap_groups = b''.join(lgroups).decode('ascii')
+
         self.ldap_client.unbind()
 
     def _ldap_validate_conf(self):

lemur/authorities/service.py

@@ -15,6 +15,7 @@ from lemur import database
 from lemur.common.utils import truthiness
 from lemur.extensions import metrics
 from lemur.authorities.models import Authority
+from lemur.certificates.models import Certificate
 from lemur.roles import service as role_service
 
 from lemur.certificates.service import upload
@@ -178,6 +179,13 @@ def render(args):
         terms = filt.split(';')
         if 'active' in filt:
             query = query.filter(Authority.active == truthiness(terms[1]))
+        elif 'cn' in filt:
+            term = '%{0}%'.format(terms[1])
+            sub_query = database.session_query(Certificate.root_authority_id) \
+                .filter(Certificate.cn.ilike(term)) \
+                .subquery()
+
+            query = query.filter(Authority.id.in_(sub_query))
         else:
             query = database.filter(query, Authority, terms)
 

lemur/certificates/cli.py

@@ -265,7 +265,7 @@ def query(fqdns, issuer, owner, expired):
     table = []
 
     q = database.session_query(Certificate)
-
+    if issuer:
         sub_query = database.session_query(Authority.id) \
             .filter(Authority.name.ilike('%{0}%'.format(issuer))) \
             .subquery()
@@ -276,12 +276,13 @@ def query(fqdns, issuer, owner, expired):
                 Certificate.authority_id.in_(sub_query)
             )
         )
-
+    if owner:
         q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
 
     if not expired:
         q = q.filter(Certificate.expired == False)  # noqa
 
+    if fqdns:
         for f in fqdns.split(','):
             q = q.filter(
                 or_(
@@ -363,9 +364,6 @@ def check_revoked():
             else:
                 status = verify_string(cert.body, "")
 
-            if status is None:
-                cert.status = 'unknown'
-            else:
             cert.status = 'valid' if status else 'revoked'
 
         except Exception as e:

lemur/certificates/models.py

@@ -19,7 +19,7 @@ from sqlalchemy.sql.expression import case, extract
 from sqlalchemy_utils.types.arrow import ArrowType
 from werkzeug.utils import cached_property
 
-from lemur.common import defaults, utils
+from lemur.common import defaults, utils, validators
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
 from lemur.database import db
 from lemur.domains.models import Domain
@@ -77,6 +77,14 @@ def get_or_increase_name(name, serial):
 
 class Certificate(db.Model):
     __tablename__ = 'certificates'
+    __table_args__ = (
+        Index('ix_certificates_cn', "cn",
+              postgresql_ops={"cn": "gin_trgm_ops"},
+              postgresql_using='gin'),
+        Index('ix_certificates_name', "name",
+              postgresql_ops={"name": "gin_trgm_ops"},
+              postgresql_using='gin'),
+    )
     id = Column(Integer, primary_key=True)
     ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
     external_id = Column(String(128))
@@ -130,7 +138,6 @@ class Certificate(db.Model):
     logs = relationship('Log', backref='certificate')
     endpoints = relationship('Endpoint', backref='certificate')
     rotation_policy = relationship("RotationPolicy")
-
     sensitive_fields = ('private_key',)
 
     def __init__(self, **kwargs):
@@ -179,6 +186,18 @@ class Certificate(db.Model):
         for domain in defaults.domains(cert):
             self.domains.append(Domain(name=domain))
 
+        # Check integrity before saving anything into the database.
+        # For user-facing API calls, validation should also be done in schema validators.
+        self.check_integrity()
+
+    def check_integrity(self):
+        """
+        Integrity checks: Does the cert have a matching private key?
+        """
+        if self.private_key:
+            validators.verify_private_key_match(utils.parse_private_key(self.private_key), self.parsed_cert,
+                                                error_class=AssertionError)
+
     @cached_property
     def parsed_cert(self):
         assert self.body, "Certificate body not set"
@@ -208,6 +227,10 @@ class Certificate(db.Model):
     def location(self):
         return defaults.location(self.parsed_cert)
 
+    @property
+    def distinguished_name(self):
+        return self.parsed_cert.subject.rfc4514_string()
+
     @property
     def key_type(self):
         if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey):
@@ -359,7 +382,7 @@ def update_destinations(target, value, initiator):
     destination_plugin = plugins.get(value.plugin_name)
     status = FAILURE_METRIC_STATUS
     try:
-        if target.private_key:
+        if target.private_key or not destination_plugin.requires_key:
             destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
             status = SUCCESS_METRIC_STATUS
     except Exception as e:

lemur/certificates/schemas.py

@@ -10,7 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load
 from marshmallow.exceptions import ValidationError
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
-from lemur.common import validators, missing
+from lemur.common import missing, utils, validators
 from lemur.common.fields import ArrowDateTime, Hex
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.constants import CERTIFICATE_KEY_TYPES
@@ -206,6 +206,7 @@ class CertificateOutputSchema(LemurOutputSchema):
 
     cn = fields.String()
     common_name = fields.String(attribute='cn')
+    distinguished_name = fields.String()
 
     not_after = fields.DateTime()
     validity_end = ArrowDateTime(attribute='not_after')
@@ -242,8 +243,8 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
     authority = fields.Nested(AssociatedAuthoritySchema, required=False)
     notify = fields.Boolean(missing=True)
     external_id = fields.String(missing=None, allow_none=True)
-    private_key = fields.String(validate=validators.private_key)
+    private_key = fields.String()
-    body = fields.String(required=True, validate=validators.public_certificate)
+    body = fields.String(required=True)
     chain = fields.String(validate=validators.public_certificate, missing=None,
                           allow_none=True)  # TODO this could be multiple certificates
 
@@ -258,6 +259,26 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
             if not data.get('private_key'):
                 raise ValidationError('Destinations require private key.')
 
+    @validates_schema
+    def validate_cert_private_key(self, data):
+        cert = None
+        key = None
+        if data.get('body'):
+            try:
+                cert = utils.parse_certificate(data['body'])
+            except ValueError:
+                raise ValidationError("Public certificate presented is not valid.", field_names=['body'])
+
+        if data.get('private_key'):
+            try:
+                key = utils.parse_private_key(data['private_key'])
+            except ValueError:
+                raise ValidationError("Private key presented is not valid.", field_names=['private_key'])
+
+        if cert and key:
+            # Throws ValidationError
+            validators.verify_private_key_match(key, cert)
+
 
 class CertificateExportInputSchema(LemurInputSchema):
     plugin = fields.Nested(PluginInputSchema)

lemur/certificates/service.py

@@ -237,11 +237,6 @@ def upload(**kwargs):
     else:
         kwargs['roles'] = roles
 
-    if kwargs.get('private_key'):
-        private_key = kwargs['private_key']
-        if not isinstance(private_key, bytes):
-            kwargs['private_key'] = private_key.encode('utf-8')
-
     cert = Certificate(**kwargs)
     cert.authority = kwargs.get('authority')
     cert = database.create(cert)
@@ -291,6 +286,14 @@ def create(**kwargs):
         certificate_issued.send(certificate=cert, authority=cert.authority)
         metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
 
+    if isinstance(cert, PendingCertificate):
+        # We need to refresh the pending certificate to avoid "Instance is not bound to a Session; "
+        # "attribute refresh operation cannot proceed"
+        pending_cert = database.session_query(PendingCertificate).get(cert.id)
+        from lemur.common.celery import fetch_acme_cert
+        if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False):
+            fetch_acme_cert.apply_async((pending_cert.id,), countdown=5)
+
     return cert
 
 
@@ -314,7 +317,7 @@ def render(args):
 
     if filt:
         terms = filt.split(';')
-        term = '%{0}%'.format(terms[1])
+        term = '{0}%'.format(terms[1])
         # Exact matches for quotes. Only applies to name, issuer, and cn
         if terms[1].startswith('"') and terms[1].endswith('"'):
             term = terms[1][1:-1]
@@ -378,7 +381,8 @@ def render(args):
         now = arrow.now().format('YYYY-MM-DD')
         query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
 
-    return database.sort_and_page(query, Certificate, args)
+    result = database.sort_and_page(query, Certificate, args)
+    return result
 
 
 def create_csr(**csr_config):
@@ -439,10 +443,7 @@ def create_csr(**csr_config):
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.TraditionalOpenSSL,  # would like to use PKCS8 but AWS ELBs don't like it
         encryption_algorithm=serialization.NoEncryption()
-    )
+    ).decode('utf-8')
-
-    if isinstance(private_key, bytes):
-        private_key = private_key.decode('utf-8')
 
     csr = request.public_bytes(
         encoding=serialization.Encoding.PEM
@@ -554,6 +555,9 @@ def reissue_certificate(certificate, replace=None, user=None):
     """
     primitives = get_certificate_primitives(certificate)
 
+    if primitives.get("csr"):
+        #  We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation.
+        del primitives["csr"]
     if not user:
         primitives['creator'] = certificate.user
 

lemur/common/celery.py

@@ -19,14 +19,17 @@ from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
-from lemur.sources.cli import clean, validate_sources
+from lemur.sources.cli import clean, sync, validate_sources
 
+if current_app:
+    flask_app = current_app
+else:
     flask_app = create_app()
 
 
 def make_celery(app):
-    celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
+    celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
-                    broker=app.config['CELERY_BROKER_URL'])
+                    broker=app.config.get('CELERY_BROKER_URL'))
     celery.conf.update(app.config)
     TaskBase = celery.Task
 
@@ -53,8 +56,10 @@ def fetch_acme_cert(id):
         id: an id of a PendingCertificate
     """
     log_data = {
-        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
+        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
+        "message": "Resolving pending certificate {}".format(id)
     }
+    current_app.logger.debug(log_data)
     pending_certs = pending_certificate_service.get_pending_certs([id])
     new = 0
     failed = 0
@@ -138,12 +143,22 @@ def fetch_all_pending_acme_certs():
     """Instantiate celery workers to resolve all pending Acme certificates"""
     pending_certs = pending_certificate_service.get_unresolved_pending_certs()
 
+    log_data = {
+        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
+        "message": "Starting job."
+    }
+
+    current_app.logger.debug(log_data)
+
     # We only care about certs using the acme-issuer plugin
     for cert in pending_certs:
         cert_authority = get_authority(cert.authority_id)
         if cert_authority.plugin_name == 'acme-issuer':
-            if cert.last_updated == cert.date_created or datetime.now(
+            if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
-                    timezone.utc) - cert.last_updated > timedelta(minutes=5):
+                log_data["message"] = "Triggering job for cert {}".format(cert.name)
+                log_data["cert_name"] = cert.name
+                log_data["cert_id"] = cert.id
+                current_app.logger.debug(log_data)
                 fetch_acme_cert.delay(cert.id)
 
 
@@ -188,3 +203,26 @@ def clean_source(source):
     """
     current_app.logger.debug("Cleaning source {}".format(source))
     clean([source], True)
+
+
+@celery.task()
+def sync_all_sources():
+    """
+    This function will sync certificates from all sources. This function triggers one celery task per source.
+    """
+    sources = validate_sources("all")
+    for source in sources:
+        current_app.logger.debug("Creating celery task to sync source {}".format(source.label))
+        sync_source.delay(source.label)
+
+
+@celery.task()
+def sync_source(source):
+    """
+    This celery task will sync the specified source.
+
+    :param source:
+    :return:
+    """
+    current_app.logger.debug("Syncing source {}".format(source))
+    sync([source])

lemur/common/defaults.py

@@ -7,18 +7,21 @@ from lemur.extensions import sentry
 from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
 
 
-def text_to_slug(value):
+def text_to_slug(value, joiner='-'):
-    """Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters."""
+    """
+    Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.
+    A series of non-alphanumeric characters is replaced with the joiner character.
+    """
 
     # Strip all character accents: decompose Unicode characters and then drop combining chars.
     value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c))
 
-    # Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash.
+    # Replace all remaining non-alphanumeric characters with joiner string. Multiple characters get collapsed into a
-    # Except, keep 'xn--' used in IDNA domain names as is.
+    # single joiner. Except, keep 'xn--' used in IDNA domain names as is.
-    value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', '-', value)
+    value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', joiner, value)
 
     # '-' in the beginning or end of string looks ugly.
-    return value.strip('-')
+    return value.strip(joiner)
 
 
 def certificate_name(common_name, issuer, not_before, not_after, san):
@@ -224,25 +227,20 @@ def bitstrength(cert):
 
 def issuer(cert):
     """
-    Gets a sane issuer name from a given certificate.
+    Gets a sane issuer slug from a given certificate, stripping non-alphanumeric characters.
 
     :param cert:
-    :return: Issuer
+    :return: Issuer slug
     """
-    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
+    # Try Common Name or fall back to Organization name
-    try:
+    attrs = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
-        # Try organization name or fall back to CN
-        issuer = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
              cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME))
-        issuer = str(issuer[0].value)
+    if not attrs:
-        for c in delchars:
+        current_app.logger.error("Unable to get issuer! Cert serial {:x}".format(cert.serial_number))
-            issuer = issuer.replace(c, "")
-        return issuer
-    except Exception as e:
-        sentry.captureException()
-        current_app.logger.error("Unable to get issuer! {0}".format(e))
         return "Unknown"
 
+    return text_to_slug(attrs[0].value, '')
+
 
 def not_before(cert):
     """

lemur/common/fields.py

@@ -350,6 +350,7 @@ class SubjectAlternativeNameExtension(Field):
                     value = value.dotted_string
                 else:
                     current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name))
+                    continue
 
                 general_names.append({'nameType': name_type, 'value': value})
 

lemur/common/missing.py

@@ -16,6 +16,7 @@ def convert_validity_years(data):
         data['validity_start'] = now.isoformat()
 
         end = now.replace(years=+int(data['validity_years']))
+
         if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
             if is_weekend(end):
                 end = end.replace(days=-2)

lemur/common/utils.py

@@ -12,7 +12,9 @@ import string
 import sqlalchemy
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import rsa, ec
+from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from flask_restful.reqparse import RequestParser
 from sqlalchemy import and_, func
 
@@ -46,10 +48,22 @@ def parse_certificate(body):
     :param body:
     :return:
     """
-    if isinstance(body, str):
+    assert isinstance(body, str)
-        body = body.encode('utf-8')
 
-    return x509.load_pem_x509_certificate(body, default_backend())
+    return x509.load_pem_x509_certificate(body.encode('utf-8'), default_backend())
+
+
+def parse_private_key(private_key):
+    """
+    Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm).
+
+    Raises ValueError for an invalid string. Raises AssertionError when passed value is not str-type.
+
+    :param private_key: String containing PEM private key
+    """
+    assert isinstance(private_key, str)
+
+    return load_pem_private_key(private_key.encode('utf8'), password=None, backend=default_backend())
 
 
 def parse_csr(csr):
@@ -59,10 +73,9 @@ def parse_csr(csr):
     :param csr:
     :return:
     """
-    if isinstance(csr, str):
+    assert isinstance(csr, str)
-        csr = csr.encode('utf-8')
 
-    return x509.load_pem_x509_csr(csr, default_backend())
+    return x509.load_pem_x509_csr(csr.encode('utf-8'), default_backend())
 
 
 def get_authority_key(body):
@@ -211,3 +224,13 @@ def truthiness(s):
     """If input string resembles something truthy then return True, else False."""
 
     return s.lower() in ('true', 'yes', 'on', 't', '1')
+
+
+def find_matching_certificates_by_hash(cert, matching_certs):
+    """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
+    determine if any of the certificate hashes match and return the matches."""
+    matching = []
+    for c in matching_certs:
+        if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()):
+            matching.append(c)
+    return matching

lemur/common/validators.py

@@ -2,14 +2,12 @@ import re
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
 from cryptography.x509 import NameOID
 from flask import current_app
 from marshmallow.exceptions import ValidationError
 
 from lemur.auth.permissions import SensitiveDomainPermission
 from lemur.common.utils import parse_certificate, is_weekend
-from lemur.domains import service as domain_service
 
 
 def public_certificate(body):
@@ -26,22 +24,6 @@ def public_certificate(body):
         raise ValidationError('Public certificate presented is not valid.')
 
 
-def private_key(key):
-    """
-    User to validate that a given string is a RSA private key
-
-    :param key:
-    :return: :raise ValueError:
-    """
-    try:
-        if isinstance(key, bytes):
-            serialization.load_pem_private_key(key, None, backend=default_backend())
-        else:
-            serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
-    except Exception:
-        raise ValidationError('Private key presented is not valid.')
-
-
 def common_name(value):
     """If the common name could be a domain name, apply domain validation rules."""
     # Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
@@ -66,6 +48,9 @@ def sensitive_domain(domain):
         raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
                               'Contact an administrator to issue the certificate.'.format(domain))
 
+    # Avoid circular import.
+    from lemur.domains import service as domain_service
+
     if any(d.sensitive for d in domain_service.get_by_name(domain)):
         raise ValidationError('Domain {0} has been marked as sensitive. '
                               'Contact an administrator to issue the certificate.'.format(domain))
@@ -141,3 +126,15 @@ def dates(data):
                 raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
 
     return data
+
+
+def verify_private_key_match(key, cert, error_class=ValidationError):
+    """
+    Checks that the supplied private key matches the certificate.
+
+    :param cert: Parsed certificate
+    :param key: Parsed private key
+    :param error_class: Exception class to raise on error
+    """
+    if key.public_key().public_numbers() != cert.public_key().public_numbers():
+        raise error_class("Private key does not match certificate.")

lemur/database.py

@@ -10,12 +10,12 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from inflection import underscore
-from sqlalchemy import exc, func
+from sqlalchemy import exc, func, distinct
+from sqlalchemy.orm import make_transient, lazyload
 from sqlalchemy.sql import and_, or_
-from sqlalchemy.orm import make_transient
 
-from lemur.extensions import db
 from lemur.exceptions import AttrNotFound, DuplicateError
+from lemur.extensions import db
 
 
 def filter_none(kwargs):
@@ -273,7 +273,31 @@ def get_count(q):
     :param q:
     :return:
     """
-    count_q = q.statement.with_only_columns([func.count()]).order_by(None)
+    disable_group_by = False
+    if len(q._entities) > 1:
+        # currently support only one entity
+        raise Exception('only one entity is supported for get_count, got: %s' % q)
+    entity = q._entities[0]
+    if hasattr(entity, 'column'):
+        # _ColumnEntity has column attr - on case: query(Model.column)...
+        col = entity.column
+        if q._group_by and q._distinct:
+            # which query can have both?
+            raise NotImplementedError
+        if q._group_by or q._distinct:
+            col = distinct(col)
+        if q._group_by:
+            # need to disable group_by and enable distinct - we can do this because we have only 1 entity
+            disable_group_by = True
+        count_func = func.count(col)
+    else:
+        # _MapperEntity doesn't have column attr - on case: query(Model)...
+        count_func = func.count()
+    if q._group_by and not disable_group_by:
+        count_func = count_func.over(None)
+    count_q = q.options(lazyload('*')).statement.with_only_columns([count_func]).order_by(None)
+    if disable_group_by:
+        count_q = count_q.group_by(None)
     count = q.session.execute(count_q).scalar()
     return count
 

lemur/dns_providers/models.py

@@ -23,7 +23,8 @@ class DnsProvider(db.Model):
     status = Column(String(length=128), nullable=True)
     options = Column(JSON, nullable=True)
     domains = Column(JSON, nullable=True)
-    certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id')
+    certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id',
+                                lazy='dynamic')
 
     def __init__(self, name, description, provider_type, credentials):
         self.name = name

lemur/domains/models.py

@@ -7,13 +7,18 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy import Column, Integer, String, Boolean, Index
 
 from lemur.database import db
 
 
 class Domain(db.Model):
     __tablename__ = 'domains'
+    __table_args__ = (
+        Index('ix_domains_name_gin', "name",
+              postgresql_ops={"name": "gin_trgm_ops"},
+              postgresql_using='gin'),
+    )
     id = Column(Integer, primary_key=True)
     name = Column(String(256), index=True)
     sensitive = Column(Boolean, default=False)

lemur/manage.py

@@ -47,7 +47,7 @@ from lemur.logs.models import Log  # noqa
 from lemur.endpoints.models import Endpoint  # noqa
 from lemur.policies.models import RotationPolicy  # noqa
 from lemur.pending_certificates.models import PendingCertificate  # noqa
-
+from lemur.dns_providers.models import DnsProvider  # noqa
 
 manager = Manager(create_app)
 manager.add_option('-c', '--config', dest='config')
@@ -273,10 +273,11 @@ class CreateUser(Command):
         Option('-u', '--username', dest='username', required=True),
         Option('-e', '--email', dest='email', required=True),
         Option('-a', '--active', dest='active', default=True),
-        Option('-r', '--roles', dest='roles', action='append', default=[])
+        Option('-r', '--roles', dest='roles', action='append', default=[]),
+        Option('-p', '--password', dest='password', default=None)
     )
 
-    def run(self, username, email, active, roles):
+    def run(self, username, email, active, roles, password):
         role_objs = []
         for r in roles:
             role_obj = role_service.get_by_name(r)
@@ -286,14 +287,16 @@ class CreateUser(Command):
                 sys.stderr.write("[!] Cannot find role {0}\n".format(r))
                 sys.exit(1)
 
+        if not password:
             password1 = prompt_pass("Password")
             password2 = prompt_pass("Confirm Password")
+            password = password1
 
             if password1 != password2:
                 sys.stderr.write("[!] Passwords do not match!\n")
                 sys.exit(1)
 
-        user_service.create(username, password1, email, active, None, role_objs)
+        user_service.create(username, password, email, active, None, role_objs)
         sys.stdout.write("[+] Created new user: {0}\n".format(username))
 
 

lemur/migrations/versions/449c3d5c7299_.py

@@ -21,6 +21,14 @@ COLUMNS = ["notification_id", "certificate_id"]
 
 
 def upgrade():
+    connection = op.get_bind()
+    # Delete duplicate entries
+    connection.execute("""\
+        DELETE FROM certificate_notification_associations WHERE ctid NOT IN (
+            -- Select the first tuple ID for each (notification_id, certificate_id) combination and keep that
+            SELECT min(ctid) FROM certificate_notification_associations GROUP BY notification_id, certificate_id
+        )
+    """)
     op.create_unique_constraint(CONSTRAINT_NAME, TABLE, COLUMNS)
 
 

lemur/migrations/versions/ee827d1e1974_.py

@@ -0,0 +1,31 @@
+"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries.
+
+Revision ID: ee827d1e1974
+Revises: 7ead443ba911
+Create Date: 2018-11-05 09:49:40.226368
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ee827d1e1974'
+down_revision = '7ead443ba911'
+
+from alembic import op
+from sqlalchemy.exc import ProgrammingError
+
+def upgrade():
+    connection = op.get_bind()
+    connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+    op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'},
+                    postgresql_using='gin')
+    op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False,
+                    postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin')
+    op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'},
+                    postgresql_using='gin')
+
+
+def downgrade():
+    op.drop_index('ix_domains_name', table_name='domains')
+    op.drop_index('ix_certificates_name', table_name='certificates')
+    op.drop_index('ix_certificates_cn', table_name='certificates')

lemur/notifications/messaging.py

@@ -8,24 +8,21 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from itertools import groupby
 from collections import defaultdict
+from datetime import timedelta
+from itertools import groupby
 
 import arrow
-from datetime import timedelta
 from flask import current_app
-
 from sqlalchemy import and_
 
 from lemur import database
+from lemur.certificates.models import Certificate
+from lemur.certificates.schemas import certificate_notification_output_schema
+from lemur.common.utils import windowed_query
 from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS
 from lemur.extensions import metrics, sentry
-from lemur.common.utils import windowed_query
-
-from lemur.certificates.schemas import certificate_notification_output_schema
-from lemur.certificates.models import Certificate
 from lemur.pending_certificates.schemas import pending_certificate_output_schema
-
 from lemur.plugins import plugins
 from lemur.plugins.utils import get_plugin_option
 
@@ -74,9 +71,10 @@ def get_eligible_certificates(exclude=None):
         notification_groups = []
 
         for certificate in items:
-            notification = needs_notification(certificate)
+            notifications = needs_notification(certificate)
 
-            if notification:
+            if notifications:
+                for notification in notifications:
                     notification_groups.append((notification, certificate))
 
         # group by notification
@@ -133,11 +131,21 @@ def send_expiration_notifications(exclude):
                 notification_data.append(cert_data)
                 security_data.append(cert_data)
 
+            notification_recipient = get_plugin_option('recipients', notification.options)
+            if notification_recipient:
+                notification_recipient = notification_recipient.split(",")
+
             if send_notification('expiration', notification_data, [owner], notification):
                 success += 1
             else:
                 failure += 1
 
+            if notification_recipient and owner != notification_recipient and security_email != notification_recipient:
+                if send_notification('expiration', notification_data, notification_recipient, notification):
+                    success += 1
+                else:
+                    failure += 1
+
             if send_notification('expiration', security_data, security_email, notification):
                 success += 1
             else:
@@ -228,6 +236,8 @@ def needs_notification(certificate):
     now = arrow.utcnow()
     days = (certificate.not_after - now).days
 
+    notifications = []
+
     for notification in certificate.notifications:
         if not notification.active or not notification.options:
             return
@@ -248,4 +258,5 @@ def needs_notification(certificate):
             raise Exception("Invalid base unit for expiration interval: {0}".format(unit))
 
         if days == interval:
-            return notification
+            notifications.append(notification)
+    return notifications

lemur/pending_certificates/models.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm import relationship
 from sqlalchemy_utils import JSONType
 from sqlalchemy_utils.types.arrow import ArrowType
 
-from lemur.certificates.models import get_or_increase_name
+from lemur.certificates.models import get_sequence
 from lemur.common import defaults, utils
 from lemur.database import db
 from lemur.models import pending_cert_source_associations, \
@@ -19,6 +19,28 @@ from lemur.models import pending_cert_source_associations, \
 from lemur.utils import Vault
 
 
+def get_or_increase_name(name, serial):
+    certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(name))).all()
+
+    if not certificates:
+        return name
+
+    serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper())
+    certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(serial_name))).all()
+
+    if not certificates:
+        return serial_name
+
+    ends = [0]
+    root, end = get_sequence(serial_name)
+    for cert in certificates:
+        root, end = get_sequence(cert.name)
+        if end:
+            ends.append(end)
+
+    return '{0}-{1}'.format(root, max(ends) + 1)
+
+
 class PendingCertificate(db.Model):
     __tablename__ = 'pending_certs'
     id = Column(Integer, primary_key=True)

lemur/plugins/lemur_acme/dyn.py

@@ -5,7 +5,7 @@ import dns.exception
 import dns.name
 import dns.query
 import dns.resolver
-from dyn.tm.errors import DynectCreateError
+from dyn.tm.errors import DynectCreateError, DynectGetError
 from dyn.tm.session import DynectSession
 from dyn.tm.zones import Node, Zone, get_all_zones
 from flask import current_app
@@ -119,7 +119,11 @@ def delete_txt_record(change_id, account_number, domain, token):
     zone = Zone(zone_name)
     node = Node(zone_name, fqdn)
 
+    try:
         all_txt_records = node.get_all_records_by_type('TXT')
+    except DynectGetError:
+        # No Text Records remain or host is not in the zone anymore because all records have been deleted.
+        return
     for txt_record in all_txt_records:
         if txt_record.txtdata == ("{}".format(token)):
             current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))

lemur/plugins/lemur_acme/plugin.py

@@ -44,7 +44,11 @@ class AuthorizationRecord(object):
 class AcmeHandler(object):
     def __init__(self):
         self.dns_providers_for_domain = {}
+        try:
             self.all_dns_providers = dns_provider_service.get_all_dns_providers()
+        except Exception as e:
+            current_app.logger.error("Unable to fetch DNS Providers: {}".format(e))
+            self.all_dns_providers = []
 
     def find_dns_challenge(self, authorizations):
         dns_challenges = []
@@ -211,12 +215,18 @@ class AcmeHandler(object):
         :return: dns_providers: List of DNS providers that have the correct zone.
         """
         self.dns_providers_for_domain[domain] = []
+        match_length = 0
         for dns_provider in self.all_dns_providers:
             if not dns_provider.domains:
                 continue
             for name in dns_provider.domains:
                 if domain.endswith("." + name):
+                    if len(name) > match_length:
+                        self.dns_providers_for_domain[domain] = [dns_provider]
+                        match_length = len(name)
+                    elif len(name) == match_length:
                         self.dns_providers_for_domain[domain].append(dns_provider)
+
         return self.dns_providers_for_domain
 
     def finalize_authorizations(self, acme_client, authorizations):
@@ -329,9 +339,10 @@ class ACMEIssuerPlugin(IssuerPlugin):
 
     def __init__(self, *args, **kwargs):
         super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
-        self.acme = AcmeHandler()
 
     def get_dns_provider(self, type):
+        self.acme = AcmeHandler()
+
         provider_types = {
             'cloudflare': cloudflare,
             'dyn': dyn,
@@ -343,12 +354,14 @@ class ACMEIssuerPlugin(IssuerPlugin):
         return provider
 
     def get_all_zones(self, dns_provider):
+        self.acme = AcmeHandler()
         dns_provider_options = json.loads(dns_provider.credentials)
         account_number = dns_provider_options.get("account_id")
         dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
         return dns_provider_plugin.get_zones(account_number=account_number)
 
     def get_ordered_certificate(self, pending_cert):
+        self.acme = AcmeHandler()
         acme_client, registration = self.acme.setup_acme_client(pending_cert.authority)
         order_info = authorization_service.get(pending_cert.external_id)
         if pending_cert.dns_provider_id:
@@ -384,6 +397,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
         return cert
 
     def get_ordered_certificates(self, pending_certs):
+        self.acme = AcmeHandler()
         pending = []
         certs = []
         for pending_cert in pending_certs:
@@ -466,6 +480,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
         :param issuer_options:
         :return: :raise Exception:
         """
+        self.acme = AcmeHandler()
         authority = issuer_options.get('authority')
         create_immediately = issuer_options.get('create_immediately', False)
         acme_client, registration = self.acme.setup_acme_client(authority)

lemur/plugins/lemur_aws/elb.py

@@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
     """
     Get a listener ARN from an endpoint.
@@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_elbs(**kwargs):
     """
     Fetches one page elb objects for a given account and region.
@@ -123,7 +123,7 @@ def get_elbs(**kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_elbs_v2(**kwargs):
     """
     Fetches one page of elb objects for a given account and region.
@@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_listeners_v2(**kwargs):
     """
     Fetches one page of listener objects for a given elb arn.
@@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
     """
     Fetching all policies currently associated with an ELB.
@@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_ssl_policies_v2(policy_names, **kwargs):
     """
     Fetching all policies currently associated with an ELB.
@@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_load_balancer_types(policies, **kwargs):
     """
     Describe the policies with policy details.
@@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def attach_certificate(name, port, certificate_id, **kwargs):
     """
     Attaches a certificate to a listener, throws exception
@@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def attach_certificate_v2(listener_arn, port, certificates, **kwargs):
     """
     Attaches a certificate to a listener, throws exception

lemur/plugins/lemur_aws/iam.py

@@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
     """
     Upload a certificate to AWS
@@ -64,6 +64,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
     :param path:
     :return:
     """
+    assert isinstance(private_key, str)
     client = kwargs.pop('client')
 
     if not path or path == '/':
@@ -72,8 +73,6 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
         name = name + '-' + path.strip('/')
 
     try:
-        if isinstance(private_key, bytes):
-            private_key = private_key.decode("utf-8")
         if cert_chain:
             return client.upload_server_certificate(
                 Path=path,
@@ -95,7 +94,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def delete_cert(cert_name, **kwargs):
     """
     Delete a certificate from AWS
@@ -112,7 +111,7 @@ def delete_cert(cert_name, **kwargs):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_certificate(name, **kwargs):
     """
     Retrieves an SSL certificate.
@@ -126,7 +125,7 @@ def get_certificate(name, **kwargs):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_certificates(**kwargs):
     """
     Fetches one page of certificate objects for a given account.

lemur/plugins/lemur_aws/plugin.py

@@ -35,8 +35,8 @@
 from flask import current_app
 
 from lemur.plugins import lemur_aws as aws
-from lemur.plugins.lemur_aws import iam, s3, elb, ec2
 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
+from lemur.plugins.lemur_aws import iam, s3, elb, ec2
 
 
 def get_region_from_dns(dns):
@@ -163,7 +163,7 @@ class AWSDestinationPlugin(DestinationPlugin):
             'name': 'accountNumber',
             'type': 'str',
             'required': True,
-            'validation': '/^[0-9]{12,12}$/',
+            'validation': '[0-9]{12}',
             'helpMessage': 'Must be a valid AWS account number!',
         },
         {
@@ -279,14 +279,14 @@ class S3DestinationPlugin(ExportDestinationPlugin):
             'name': 'bucket',
             'type': 'str',
             'required': True,
-            'validation': '/^$|\s+/',
+            'validation': '[0-9a-z.-]{3,63}',
             'helpMessage': 'Must be a valid S3 bucket name!',
         },
         {
             'name': 'accountNumber',
             'type': 'str',
             'required': True,
-            'validation': '/^[0-9]{12,12}$/',
+            'validation': '[0-9]{12}',
             'helpMessage': 'A valid AWS account number with permission to access S3',
         },
         {
@@ -308,7 +308,6 @@ class S3DestinationPlugin(ExportDestinationPlugin):
             'name': 'prefix',
             'type': 'str',
             'required': False,
-            'validation': '/^$|\s+/',
             'helpMessage': 'Must be a valid S3 object prefix!',
         }
     ]

lemur/plugins/lemur_aws/sts.py

@@ -9,14 +9,22 @@ from functools import wraps
 
 import boto3
 
+from botocore.config import Config
 from flask import current_app
 
 
+config = Config(
+    retries=dict(
+        max_attempts=20
+    )
+)
+
+
 def sts_client(service, service_type='client'):
     def decorator(f):
         @wraps(f)
         def decorated_function(*args, **kwargs):
-            sts = boto3.client('sts')
+            sts = boto3.client('sts', config=config)
             arn = 'arn:aws:iam::{0}:role/{1}'.format(
                 kwargs.pop('account_number'),
                 current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
@@ -31,7 +39,8 @@ def sts_client(service, service_type='client'):
                     region_name=kwargs.pop('region', 'us-east-1'),
                     aws_access_key_id=role['Credentials']['AccessKeyId'],
                     aws_secret_access_key=role['Credentials']['SecretAccessKey'],
-                    aws_session_token=role['Credentials']['SessionToken']
+                    aws_session_token=role['Credentials']['SessionToken'],
+                    config=config
                 )
                 kwargs['client'] = client
             elif service_type == 'resource':
@@ -40,7 +49,8 @@ def sts_client(service, service_type='client'):
                     region_name=kwargs.pop('region', 'us-east-1'),
                     aws_access_key_id=role['Credentials']['AccessKeyId'],
                     aws_secret_access_key=role['Credentials']['SecretAccessKey'],
-                    aws_session_token=role['Credentials']['SessionToken']
+                    aws_session_token=role['Credentials']['SessionToken'],
+                    config=config
                 )
                 kwargs['resource'] = resource
             return f(*args, **kwargs)

lemur/plugins/lemur_aws/tests/test_elb.py

@@ -4,7 +4,7 @@ from moto import mock_sts, mock_elb
 
 @mock_sts()
 @mock_elb()
-def test_get_all_elbs(app):
+def test_get_all_elbs(app, aws_credentials):
     from lemur.plugins.lemur_aws.elb import get_all_elbs
     client = boto3.client('elb', region_name='us-east-1')
 

lemur/plugins/lemur_cfssl/plugin.py

@@ -10,6 +10,9 @@
 
 import json
 import requests
+import base64
+import hmac
+import hashlib
 
 from flask import current_app
 
@@ -48,6 +51,21 @@ class CfsslIssuerPlugin(IssuerPlugin):
         data = {'certificate_request': csr}
         data = json.dumps(data)
 
+        try:
+            hex_key = current_app.config.get('CFSSL_KEY')
+            key = bytes.fromhex(hex_key)
+        except (ValueError, NameError):
+            # unable to find CFSSL_KEY in config, continue using normal sign method
+            pass
+        else:
+            data = data.encode()
+
+            token = base64.b64encode(hmac.new(key, data, digestmod=hashlib.sha256).digest())
+            data = base64.b64encode(data)
+
+            data = json.dumps({'token': token.decode('utf-8'), 'request': data.decode('utf-8')})
+
+            url = "{0}{1}".format(current_app.config.get('CFSSL_URL'), '/api/v1/cfssl/authsign')
         response = self.session.post(url, data=data.encode(encoding='utf_8', errors='strict'))
         if response.status_code > 399:
             metrics.send('cfssl_create_certificate_failure', 'counter', 1)

lemur/plugins/lemur_cryptography/plugin.py

@@ -14,6 +14,7 @@ from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
 
+from lemur.common.utils import parse_private_key
 from lemur.plugins.bases import IssuerPlugin
 from lemur.plugins import lemur_cryptography as cryptography_issuer
 
@@ -40,7 +41,8 @@ def issue_certificate(csr, options, private_key=None):
     if options.get("authority"):
         # Issue certificate signed by an existing lemur_certificates authority
         issuer_subject = options['authority'].authority_certificate.subject
-        issuer_private_key = options['authority'].authority_certificate.private_key
+        assert private_key is None, "Private would be ignored, authority key used instead"
+        private_key = options['authority'].authority_certificate.private_key
         chain_cert_pem = options['authority'].authority_certificate.body
         authority_key_identifier_public = options['authority'].authority_certificate.public_key
         authority_key_identifier_subject = x509.SubjectKeyIdentifier.from_public_key(authority_key_identifier_public)
@@ -52,7 +54,6 @@ def issue_certificate(csr, options, private_key=None):
     else:
         # Issue certificate that is self-signed (new lemur_certificates root authority)
         issuer_subject = csr.subject
-        issuer_private_key = private_key
         chain_cert_pem = ""
         authority_key_identifier_public = csr.public_key()
         authority_key_identifier_subject = None
@@ -112,11 +113,7 @@ def issue_certificate(csr, options, private_key=None):
             # FIXME: Not implemented in lemur/schemas.py yet https://github.com/Netflix/lemur/issues/662
             pass
 
-    private_key = serialization.load_pem_private_key(
+    private_key = parse_private_key(private_key)
-        bytes(str(issuer_private_key).encode('utf-8')),
-        password=None,
-        backend=default_backend()
-    )
 
     cert = builder.sign(private_key, hashes.SHA256(), default_backend())
     cert_pem = cert.public_bytes(

lemur/plugins/lemur_csr/plugin.py

@@ -38,14 +38,9 @@ def create_csr(cert, chain, csr_tmp, key):
     :param csr_tmp:
     :param key:
     """
-    if isinstance(cert, bytes):
+    assert isinstance(cert, str)
-        cert = cert.decode('utf-8')
+    assert isinstance(chain, str)
-
+    assert isinstance(key, str)
-    if isinstance(chain, bytes):
-        chain = chain.decode('utf-8')
-
-    if isinstance(key, bytes):
-        key = key.decode('utf-8')
 
     with mktempfile() as key_tmp:
         with open(key_tmp, 'w') as f:

lemur/plugins/lemur_java/plugin.py

@@ -59,11 +59,8 @@ def split_chain(chain):
 
 
 def create_truststore(cert, chain, jks_tmp, alias, passphrase):
-    if isinstance(cert, bytes):
+    assert isinstance(cert, str)
-        cert = cert.decode('utf-8')
+    assert isinstance(chain, str)
-
-    if isinstance(chain, bytes):
-        chain = chain.decode('utf-8')
 
     with mktempfile() as cert_tmp:
         with open(cert_tmp, 'w') as f:
@@ -98,14 +95,9 @@ def create_truststore(cert, chain, jks_tmp, alias, passphrase):
 
 
 def create_keystore(cert, chain, jks_tmp, key, alias, passphrase):
-    if isinstance(cert, bytes):
+    assert isinstance(cert, str)
-        cert = cert.decode('utf-8')
+    assert isinstance(chain, str)
-
+    assert isinstance(key, str)
-    if isinstance(chain, bytes):
-        chain = chain.decode('utf-8')
-
-    if isinstance(key, bytes):
-        key = key.decode('utf-8')
 
     # Create PKCS12 keystore from private key and public certificate
     with mktempfile() as cert_tmp:

lemur/plugins/lemur_kubernetes/plugin.py

@@ -11,31 +11,37 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 """
 import base64
-import os
-import urllib
-import requests
 import itertools
+import os
 
-from lemur.certificates.models import Certificate
+import requests
+from flask import current_app
+
+from lemur.common.defaults import common_name
+from lemur.common.utils import parse_certificate
 from lemur.plugins.bases import DestinationPlugin
 
 DEFAULT_API_VERSION = 'v1'
 
 
 def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data):
-
     # _resolve_uri(k8s_base_uri, namespace, kind, name, api_ver=DEFAULT_API_VERSION)
     url = _resolve_uri(k8s_base_uri, namespace, kind)
+    current_app.logger.debug("K8S POST request URL: %s", url)
 
     create_resp = k8s_api.post(url, json=data)
+    current_app.logger.debug("K8S POST response: %s", create_resp)
 
     if 200 <= create_resp.status_code <= 299:
         return None
-
+    elif create_resp.json().get('reason', '') != 'AlreadyExists':
-    elif create_resp.json()['reason'] != 'AlreadyExists':
         return create_resp.content
 
-    update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data)
+    url = _resolve_uri(k8s_base_uri, namespace, kind, name)
+    current_app.logger.debug("K8S PUT request URL: %s", url)
+
+    update_resp = k8s_api.put(url, json=data)
+    current_app.logger.debug("K8S PUT response: %s", update_resp)
 
     if not 200 <= update_resp.status_code <= 299:
         return update_resp.content
@@ -43,11 +49,12 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data):
     return
 
 
-def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,):
+def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION):
     api_group = 'api'
     if '/' in api_ver:
         api_group = 'apis'
-    return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ('/' + namespace if namespace else '')
+    return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + (
+        '/' + namespace if namespace else '')
 
 
 def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION):
@@ -61,6 +68,41 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V
     ]))
 
 
+# Performs Base64 encoding of string to string using the base64.b64encode() function
+# which encodes bytes to bytes.
+def base64encode(string):
+    return base64.b64encode(string.encode()).decode()
+
+
+def build_secret(secret_format, secret_name, body, private_key, cert_chain):
+    secret = {
+        'apiVersion': 'v1',
+        'kind': 'Secret',
+        'type': 'Opaque',
+        'metadata': {
+            'name': secret_name,
+        }
+    }
+    if secret_format == 'Full':
+        secret['data'] = {
+            'combined.pem': base64encode('%s\n%s' % (body, private_key)),
+            'ca.crt': base64encode(cert_chain),
+            'service.key': base64encode(private_key),
+            'service.crt': base64encode(body),
+        }
+    if secret_format == 'TLS':
+        secret['type'] = 'kubernetes.io/tls'
+        secret['data'] = {
+            'tls.crt': base64encode(cert_chain),
+            'tls.key': base64encode(private_key)
+        }
+    if secret_format == 'Certificate':
+        secret['data'] = {
+            'tls.crt': base64encode(cert_chain),
+        }
+    return secret
+
+
 class KubernetesDestinationPlugin(DestinationPlugin):
     title = 'Kubernetes'
     slug = 'kubernetes-destination'
@@ -70,35 +112,81 @@ class KubernetesDestinationPlugin(DestinationPlugin):
     author_url = 'https://github.com/mik373/lemur'
 
     options = [
+        {
+            'name': 'secretNameFormat',
+            'type': 'str',
+            'required': False,
+            # Validation is difficult. This regex is used by kubectl to validate secret names:
+            #  [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*
+            # Allowing the insertion of "{common_name}" (or any other such placeholder}
+            # at any point in the string proved very challenging and had a tendency to
+            # cause my browser to hang. The specified expression will allow any valid string
+            # but will also accept many invalid strings.
+            'validation': '(?:[a-z0-9.-]|\\{common_name\\})+',
+            'helpMessage': 'Must be a valid secret name, possibly including "{common_name}"',
+            'default': '{common_name}'
+        },
         {
             'name': 'kubernetesURL',
             'type': 'str',
-            'required': True,
+            'required': False,
-            'validation': '@(https?|http)://(-\.)?([^\s/?\.#-]+\.?)+(/[^\s]*)?$@iS',
+            'validation': 'https?://[a-zA-Z0-9.-]+(?::[0-9]+)?',
             'helpMessage': 'Must be a valid Kubernetes server URL!',
+            'default': 'https://kubernetes.default'
         },
         {
             'name': 'kubernetesAuthToken',
             'type': 'str',
-            'required': True,
+            'required': False,
-            'validation': '/^$|\s+/',
+            'validation': '[0-9a-zA-Z-_.]+',
             'helpMessage': 'Must be a valid Kubernetes server Token!',
         },
         {
-            'name': 'kubernetesServerCertificate',
+            'name': 'kubernetesAuthTokenFile',
             'type': 'str',
-            'required': True,
+            'required': False,
-            'validation': '/^$|\s+/',
+            'validation': '(/[^/]+)+',
+            'helpMessage': 'Must be a valid file path!',
+            'default': '/var/run/secrets/kubernetes.io/serviceaccount/token'
+        },
+        {
+            'name': 'kubernetesServerCertificate',
+            'type': 'textarea',
+            'required': False,
+            'validation': '-----BEGIN CERTIFICATE-----[a-zA-Z0-9/+\\s\\r\\n]+-----END CERTIFICATE-----',
             'helpMessage': 'Must be a valid Kubernetes server Certificate!',
         },
+        {
+            'name': 'kubernetesServerCertificateFile',
+            'type': 'str',
+            'required': False,
+            'validation': '(/[^/]+)+',
+            'helpMessage': 'Must be a valid file path!',
+            'default': '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+        },
         {
             'name': 'kubernetesNamespace',
             'type': 'str',
-            'required': True,
+            'required': False,
-            'validation': '/^$|\s+/',
+            'validation': '[a-z0-9]([-a-z0-9]*[a-z0-9])?',
             'helpMessage': 'Must be a valid Kubernetes Namespace!',
         },
-
+        {
+            'name': 'kubernetesNamespaceFile',
+            'type': 'str',
+            'required': False,
+            'validation': '(/[^/]+)+',
+            'helpMessage': 'Must be a valid file path!',
+            'default': '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
+        },
+        {
+            'name': 'secretFormat',
+            'type': 'select',
+            'required': True,
+            'available': ['Full', 'TLS', 'Certificate'],
+            'helpMessage': 'The type of Secret to create.',
+            'default': 'Full'
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -106,56 +194,91 @@ class KubernetesDestinationPlugin(DestinationPlugin):
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
 
-        k8_bearer = self.get_option('kubernetesAuthToken', options)
+        try:
-        k8_cert = self.get_option('kubernetesServerCertificate', options)
-        k8_namespace = self.get_option('kubernetesNamespace', options)
             k8_base_uri = self.get_option('kubernetesURL', options)
+            secret_format = self.get_option('secretFormat', options)
+            k8s_api = K8sSession(
+                self.k8s_bearer(options),
+                self.k8s_cert(options)
+            )
+            cn = common_name(parse_certificate(body))
+            secret_name_format = self.get_option('secretNameFormat', options)
+            secret_name = secret_name_format.format(common_name=cn)
+            secret = build_secret(secret_format, secret_name, body, private_key, cert_chain)
+            err = ensure_resource(
+                k8s_api,
+                k8s_base_uri=k8_base_uri,
+                namespace=self.k8s_namespace(options),
+                kind="secret",
+                name=secret_name,
+                data=secret
+            )
 
-        k8s_api = K8sSession(k8_bearer, k8_cert)
+        except Exception as e:
-
+            current_app.logger.exception("Exception in upload: {}".format(e), exc_info=True)
-        cert = Certificate(body=body)
+            raise
-
-        # in the future once runtime properties can be passed-in - use passed-in secret name
-        secret_name = 'certs-' + urllib.quote_plus(cert.name)
-
-        err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={
-            'apiVersion': 'v1',
-            'kind': 'Secret',
-            'metadata': {
-                'name': secret_name,
-            },
-            'data': {
-                'combined.pem': base64.b64encode(body + private_key),
-                'ca.crt': base64.b64encode(cert_chain),
-                'service.key': base64.b64encode(private_key),
-                'service.crt': base64.b64encode(body),
-            }
-        })
 
         if err is not None:
+            current_app.logger.error("Error deploying resource: %s", err)
             raise Exception("Error uploading secret: " + err)
 
+    def k8s_bearer(self, options):
+        bearer = self.get_option('kubernetesAuthToken', options)
+        if not bearer:
+            bearer_file = self.get_option('kubernetesAuthTokenFile', options)
+            with open(bearer_file, "r") as file:
+                bearer = file.readline()
+            if bearer:
+                current_app.logger.debug("Using token read from %s", bearer_file)
+            else:
+                raise Exception("Unable to locate token in options or from %s", bearer_file)
+        else:
+            current_app.logger.debug("Using token from options")
+        return bearer
+
+    def k8s_cert(self, options):
+        cert_file = self.get_option('kubernetesServerCertificateFile', options)
+        cert = self.get_option('kubernetesServerCertificate', options)
+        if cert:
+            cert_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert')
+            with open(cert_file, "w") as text_file:
+                text_file.write(cert)
+            current_app.logger.debug("Using certificate from options")
+        else:
+            current_app.logger.debug("Using certificate from %s", cert_file)
+        return cert_file
+
+    def k8s_namespace(self, options):
+        namespace = self.get_option('kubernetesNamespace', options)
+        if not namespace:
+            namespace_file = self.get_option('kubernetesNamespaceFile', options)
+            with open(namespace_file, "r") as file:
+                namespace = file.readline()
+            if namespace:
+                current_app.logger.debug("Using namespace %s from %s", namespace, namespace_file)
+            else:
+                raise Exception("Unable to locate namespace in options or from %s", namespace_file)
+        else:
+            current_app.logger.debug("Using namespace %s from options", namespace)
+        return namespace
+
 
 class K8sSession(requests.Session):
 
-    def __init__(self, bearer, cert):
+    def __init__(self, bearer, cert_file):
         super(K8sSession, self).__init__()
 
         self.headers.update({
             'Authorization': 'Bearer %s' % bearer
         })
 
-        k8_ca = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert')
+        self.verify = cert_file
 
-        with open(k8_ca, "w") as text_file:
+    def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None,
-            text_file.write(cert)
+                timeout=30, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None,
-
+                json=None):
-        self.verify = k8_ca
-
-    def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=30, allow_redirects=True, proxies=None,
-                hooks=None, stream=None, verify=None, cert=None, json=None):
         """
         This method overrides the default timeout to be 10s.
         """
-        return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream,
+        return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout,
-                                               verify, cert, json)
+                                               allow_redirects, proxies, hooks, stream, verify, cert, json)

lemur/plugins/lemur_openssl/plugin.py

@@ -44,14 +44,9 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase):
     :param alias:
     :param passphrase:
     """
-    if isinstance(cert, bytes):
+    assert isinstance(cert, str)
-        cert = cert.decode('utf-8')
+    assert isinstance(chain, str)
-
+    assert isinstance(key, str)
-    if isinstance(chain, bytes):
-        chain = chain.decode('utf-8')
-
-    if isinstance(key, bytes):
-        key = key.decode('utf-8')
 
     with mktempfile() as key_tmp:
         with open(key_tmp, 'w') as f:

lemur/plugins/lemur_verisign/plugin.py

@@ -111,10 +111,19 @@ def process_options(options):
 
     data['subject_alt_names'] = ",".join(get_additional_names(options))
 
+    if options.get('validity_end') > arrow.utcnow().replace(years=2):
+        raise Exception("Verisign issued certificates cannot exceed two years in validity")
+
     if options.get('validity_end'):
+        # VeriSign (Symantec) only accepts strictly smaller than 2 year end date
+        if options.get('validity_end') < arrow.utcnow().replace(years=2).replace(days=-1):
             period = get_default_issuance(options)
             data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY")
             data['validityPeriod'] = period
+        else:
+            # allowing Symantec website setting the end date, given the validity period
+            data['validityPeriod'] = str(get_default_issuance(options))
+            options.pop('validity_end', None)
 
     elif options.get('validity_years'):
         if options['validity_years'] in [1, 2]:

lemur/sources/cli.py

@@ -93,6 +93,7 @@ def sync(source_strings):
             )
 
             sentry.captureException()
+            metrics.send('source_sync_fail', 'counter', 1, metric_tags={'source': source.label, 'status': status})
 
         metrics.send('source_sync', 'counter', 1, metric_tags={'source': source.label, 'status': status})
 

lemur/sources/service.py

@@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service
 from lemur.destinations import service as destination_service
 
 from lemur.certificates.schemas import CertificateUploadInputSchema
-from lemur.common.utils import parse_certificate
+from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate
 from lemur.common.defaults import serial
 
 from lemur.plugins.base import plugins
@@ -131,7 +131,8 @@ def sync_certificates(source, user):
 
         if not exists:
             cert = parse_certificate(certificate['body'])
-            exists = certificate_service.get_by_serial(serial(cert))
+            matching_serials = certificate_service.get_by_serial(serial(cert))
+            exists = find_matching_certificates_by_hash(cert, matching_serials)
 
         if not certificate.get('owner'):
             certificate['owner'] = user.email

lemur/static/app/angular/certificates/view/view.tpl.html

@@ -83,6 +83,8 @@
                     </div>
                     <!-- Certificate fields -->
                     <div class="list-group-item">
+                      <dt>Distinguished Name</dt>
+                      <dd>{{ certificate.distinguishedName }}</dd>
                       <dt>Certificate Authority</dt>
                       <dd>{{ certificate.authority ? certificate.authority.name : "Imported" }} <span class="text-muted">({{ certificate.issuer }})</span></dd>
                       <dt>Serial</dt>

lemur/static/app/angular/destinations/destination/destination.tpl.html

@@ -47,7 +47,9 @@
             <select name="sub" ng-if="item.type == 'select'" class="form-control" ng-options="i for i in item.available"
                     ng-model="item.value"></select>
             <input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox" ng-model="item.value">
-            <input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value"/>
+            <input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value" ng-pattern="item.validation"/>
+            <textarea name="sub" ng-if="item.type == 'textarea'" class="form-control"
+                      ng-model="item.value" ng-pattern="item.validation"></textarea>
             <div ng-if="item.type == 'export-plugin'">
               <form name="exportForm" class="form-horizontal" role="form" novalidate>
                 <select class="form-control" ng-model="item.value"
@@ -69,6 +71,8 @@
                                  ng-model="item.value">
                           <input name="sub" ng-if="item.type == 'str'" type="text" class="form-control"
                                  ng-model="item.value" ng-pattern="item.validation"/>
+                          <textarea name="sub" ng-if="item.type == 'textarea'" class="form-control"
+                                    ng-model="item.value" ng-pattern="item.validation"></textarea>
                           <p ng-show="subForm.sub.$invalid && !subForm.sub.$pristine"
                              class="help-block">{{ item.helpMessage }}</p>
                         </div>

lemur/tests/conf.py

@@ -1,7 +1,7 @@
-
 # This is just Python which means you can inherit and tweak settings
 
 import os
+
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
 THREADS_PER_PAGE = 8
@@ -78,14 +78,12 @@ DIGICERT_API_KEY = 'api-key'
 DIGICERT_ORG_ID = 111111
 DIGICERT_ROOT = "ROOT"
 
-
 VERISIGN_URL = 'http://example.com'
 VERISIGN_PEM_PATH = '~/'
 VERISIGN_FIRST_NAME = 'Jim'
 VERISIGN_LAST_NAME = 'Bob'
 VERSIGN_EMAIL = 'jim@example.com'
 
-
 ACME_AWS_ACCOUNT_NUMBER = '11111111111'
 
 ACME_PRIVATE_KEY = '''
@@ -180,6 +178,7 @@ ACME_URL = 'https://acme-v01.api.letsencrypt.org'
 ACME_EMAIL = 'jim@example.com'
 ACME_TEL = '4088675309'
 ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org'
+ACME_DISABLE_AUTORESOLVE = True
 
 LDAP_AUTH = True
 LDAP_BIND_URI = 'ldap://localhost'

lemur/tests/conftest.py

@@ -3,19 +3,18 @@ import os
 import datetime
 import pytest
 from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from flask import current_app
 from flask_principal import identity_changed, Identity
 
 from lemur import create_app
+from lemur.common.utils import parse_private_key
 from lemur.database import db as _db
 from lemur.auth.service import create_token
-from lemur.tests.vectors import SAN_CERT_KEY
+from lemur.tests.vectors import SAN_CERT_KEY, INTERMEDIATE_KEY
 
 from .factories import ApiKeyFactory, AuthorityFactory, NotificationFactory, DestinationFactory, \
     CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, \
-    RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory
+    RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory, CryptoAuthorityFactory
 
 
 def pytest_runtest_setup(item):
@@ -91,6 +90,13 @@ def authority(session):
     return a
 
 
+@pytest.fixture
+def crypto_authority(session):
+    a = CryptoAuthorityFactory()
+    session.commit()
+    return a
+
+
 @pytest.fixture
 def async_authority(session):
     a = AsyncAuthorityFactory()
@@ -228,7 +234,12 @@ def logged_in_admin(session, app):
 
 @pytest.fixture
 def private_key():
-    return load_pem_private_key(SAN_CERT_KEY.encode(), password=None, backend=default_backend())
+    return parse_private_key(SAN_CERT_KEY)
+
+
+@pytest.fixture
+def issuer_private_key():
+    return parse_private_key(INTERMEDIATE_KEY)
 
 
 @pytest.fixture
@@ -240,3 +251,11 @@ def cert_builder(private_key):
             .public_key(private_key.public_key())
             .not_valid_before(datetime.datetime(2017, 12, 22))
             .not_valid_after(datetime.datetime(2040, 1, 1)))
+
+
+@pytest.fixture(scope='function')
+def aws_credentials():
+    os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
+    os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
+    os.environ['AWS_SECURITY_TOKEN'] = 'testing'
+    os.environ['AWS_SESSION_TOKEN'] = 'testing'

lemur/tests/factories.py

@@ -168,6 +168,11 @@ class AsyncAuthorityFactory(AuthorityFactory):
     authority_certificate = SubFactory(CertificateFactory)
 
 
+class CryptoAuthorityFactory(AuthorityFactory):
+    """Authority factory based on 'cryptography' plugin."""
+    plugin = {'slug': 'cryptography-issuer'}
+
+
 class DestinationFactory(BaseFactory):
     """Destination factory."""
     plugin_name = 'test-destination'

lemur/tests/test_certificates.py

@@ -18,7 +18,7 @@ from lemur.domains.models import Domain
 
 
 from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \
-    INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY
+    INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
 
 
 def test_get_or_increase_name(session, certificate):
@@ -448,6 +448,85 @@ def test_certificate_sensitive_name(client, authority, session, logged_in_user):
     assert errors['common_name'][0].startswith("Domain sensitive.example.com has been marked as sensitive")
 
 
+def test_certificate_upload_schema_ok(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'name': 'Jane',
+        'owner': 'pwner@example.com',
+        'body': SAN_CERT_STR,
+        'privateKey': SAN_CERT_KEY,
+        'chain': INTERMEDIATE_CERT_STR,
+        'external_id': '1234',
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert not errors
+
+
+def test_certificate_upload_schema_minimal(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'owner': 'pwner@example.com',
+        'body': SAN_CERT_STR,
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert not errors
+
+
+def test_certificate_upload_schema_long_chain(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'owner': 'pwner@example.com',
+        'body': SAN_CERT_STR,
+        'chain': INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert not errors
+
+
+def test_certificate_upload_schema_invalid_body(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'owner': 'pwner@example.com',
+        'body': 'Hereby I certify that this is a valid body',
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert errors == {'body': ['Public certificate presented is not valid.']}
+
+
+def test_certificate_upload_schema_invalid_pkey(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'owner': 'pwner@example.com',
+        'body': SAN_CERT_STR,
+        'privateKey': 'Look at me Im a private key!!111',
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert errors == {'private_key': ['Private key presented is not valid.']}
+
+
+def test_certificate_upload_schema_invalid_chain(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'body': SAN_CERT_STR,
+        'chain': 'CHAINSAW',
+        'owner': 'pwner@example.com',
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert errors == {'chain': ['Public certificate presented is not valid.']}
+
+
+def test_certificate_upload_schema_wrong_pkey(client):
+    from lemur.certificates.schemas import CertificateUploadInputSchema
+    data = {
+        'body': SAN_CERT_STR,
+        'privateKey': ROOTCA_KEY,
+        'chain': INTERMEDIATE_CERT_STR,
+        'owner': 'pwner@example.com',
+    }
+    data, errors = CertificateUploadInputSchema().load(data)
+    assert errors == {'_schema': ['Private key does not match certificate.']}
+
+
 def test_create_basic_csr(client):
     csr_config = dict(
         common_name='example.com',
@@ -545,8 +624,11 @@ def test_create_certificate(issuer_plugin, authority, user):
     assert cert.name == 'ACustomName1'
 
 
-def test_reissue_certificate(issuer_plugin, authority, certificate):
+def test_reissue_certificate(issuer_plugin, crypto_authority, certificate, logged_in_user):
     from lemur.certificates.service import reissue_certificate
+
+    # test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead.
+    certificate.authority = crypto_authority
     new_cert = reissue_certificate(certificate)
     assert new_cert
 
@@ -570,7 +652,7 @@ def test_import(user):
     assert str(cert.not_after) == '2047-12-31T22:00:00+00:00'
     assert str(cert.not_before) == '2017-12-31T22:00:00+00:00'
     assert cert.issuer == 'LemurTrustUnittestsClass1CA2018'
-    assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2'
+    assert cert.name.startswith('SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231')
 
     cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, owner='joe@example.com', name='ACustomName2', creator=user['user'])
     assert cert.name == 'ACustomName2'
@@ -620,6 +702,12 @@ def test_certificate_get_body(client):
     response_body = client.get(api.url_for(Certificates, certificate_id=1), headers=VALID_USER_HEADER_TOKEN).json
     assert response_body['serial'] == '211983098819107449768450703123665283596'
     assert response_body['serialHex'] == '9F7A75B39DAE4C3F9524C68B06DA6A0C'
+    assert response_body['distinguishedName'] == ('CN=LemurTrust Unittests Class 1 CA 2018,'
+                                                  'O=LemurTrust Enterprises Ltd,'
+                                                  'OU=Unittesting Operations Center,'
+                                                  'C=EE,'
+                                                  'ST=N/A,'
+                                                  'L=Earth')
 
 
 @pytest.mark.parametrize("token,status", [

lemur/tests/test_defaults.py

@@ -1,3 +1,7 @@
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+
 from .vectors import SAN_CERT, WILDCARD_CERT, INTERMEDIATE_CERT
 
 
@@ -41,12 +45,14 @@ def test_cert_issuer(client):
 def test_text_to_slug(client):
     from lemur.common.defaults import text_to_slug
     assert text_to_slug('test - string') == 'test-string'
+    assert text_to_slug('test - string', '') == 'teststring'
     # Accented characters are decomposed
     assert text_to_slug('föö bär') == 'foo-bar'
     # Melt away the Unicode Snowman
     assert text_to_slug('\u2603') == ''
     assert text_to_slug('\u2603test\u2603') == 'test'
     assert text_to_slug('snow\u2603man') == 'snow-man'
+    assert text_to_slug('snow\u2603man', '') == 'snowman'
     # IDNA-encoded domain names should be kept as-is
     assert text_to_slug('xn--i1b6eqas.xn--xmpl-loa9b3671b.com') == 'xn--i1b6eqas.xn--xmpl-loa9b3671b.com'
 
@@ -75,3 +81,29 @@ def test_create_name(client):
         datetime(2015, 5, 12, 0, 0, 0),
         False
     ) == 'xn--mnchen-3ya.de-VertrauenswurdigAutoritat-20150507-20150512'
+
+
+def test_issuer(client, cert_builder, issuer_private_key):
+    from lemur.common.defaults import issuer
+
+    assert issuer(INTERMEDIATE_CERT) == 'LemurTrustUnittestsRootCA2018'
+
+    # We need to override builder's issuer name
+    cert_builder._issuer_name = None
+    # Unicode issuer name
+    cert = (cert_builder
+            .issuer_name(x509.Name([x509.NameAttribute(x509.NameOID.COMMON_NAME, 'Vertrauenswürdig Autorität')]))
+            .sign(issuer_private_key, hashes.SHA256(), default_backend()))
+    assert issuer(cert) == 'VertrauenswurdigAutoritat'
+
+    # Fallback to 'Organization' field when issuer CN is missing
+    cert = (cert_builder
+            .issuer_name(x509.Name([x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, 'No Such Organization')]))
+            .sign(issuer_private_key, hashes.SHA256(), default_backend()))
+    assert issuer(cert) == 'NoSuchOrganization'
+
+    # Missing issuer name
+    cert = (cert_builder
+            .issuer_name(x509.Name([]))
+            .sign(issuer_private_key, hashes.SHA256(), default_backend()))
+    assert issuer(cert) == 'Unknown'

lemur/tests/test_pending_certificates.py

@@ -2,11 +2,10 @@ import json
 
 import pytest
 
+from lemur.pending_certificates.views import *  # noqa
 from .vectors import CSR_STR, INTERMEDIATE_CERT_STR, VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, \
     VALID_USER_HEADER_TOKEN, WILDCARD_CERT_STR
 
-from lemur.pending_certificates.views import *  # noqa
-
 
 def test_increment_attempt(pending_certificate):
     from lemur.pending_certificates.service import increment_attempt
@@ -17,7 +16,8 @@ def test_increment_attempt(pending_certificate):
 
 def test_create_pending_certificate(async_issuer_plugin, async_authority, user):
     from lemur.certificates.service import create
-    pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], common_name='ACommonName')
+    pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'],
+                          common_name='ACommonName')
     assert pending_cert.external_id == '12345'
 
 

lemur/tests/test_validators.py

@@ -1,16 +1,28 @@
-import pytest
 from datetime import datetime
-from .vectors import SAN_CERT_KEY
+
+import pytest
 from marshmallow.exceptions import ValidationError
 
+from lemur.common.utils import parse_private_key
+from lemur.common.validators import verify_private_key_match
+from lemur.tests.vectors import INTERMEDIATE_CERT, SAN_CERT, SAN_CERT_KEY
+
 
 def test_private_key(session):
-    from lemur.common.validators import private_key
+    parse_private_key(SAN_CERT_KEY)
 
-    private_key(SAN_CERT_KEY)
+    with pytest.raises(ValueError):
+        parse_private_key('invalid_private_key')
+
+
+def test_validate_private_key(session):
+    key = parse_private_key(SAN_CERT_KEY)
+
+    verify_private_key_match(key, SAN_CERT)
 
     with pytest.raises(ValidationError):
-        private_key('invalid_private_key')
+        # Wrong key for certificate
+        verify_private_key_match(key, INTERMEDIATE_CERT)
 
 
 def test_sub_alt_type(session):

requirements-dev.in

@@ -1,6 +1,6 @@
 # Run `make up-reqs` to update pinned dependencies in requirement text files
 
-flake8>=3.2,<4.0
+flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors.
 pre-commit
 invoke
 twine

requirements-dev.txt

@@ -4,36 +4,34 @@
 #
 #    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
-aspy.yaml==1.1.1          # via pre-commit
+aspy.yaml==1.1.2          # via pre-commit
-bleach==3.0.2             # via readme-renderer
+bleach==3.1.0             # via readme-renderer
-cached-property==1.5.1    # via pre-commit
+certifi==2018.11.29       # via requests
-certifi==2018.10.15       # via requests
+cfgv==1.4.0               # via pre-commit
-cffi==1.11.5              # via cmarkgfm
-cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
-cmarkgfm==0.4.2           # via readme-renderer
 docutils==0.14            # via readme-renderer
 flake8==3.5.0
-future==0.16.0            # via readme-renderer
+identify==1.2.1           # via pre-commit
-identify==1.1.7           # via pre-commit
+idna==2.8                 # via requests
-idna==2.7                 # via requests
+importlib-metadata==0.8   # via pre-commit
+importlib-resources==1.0.2  # via pre-commit
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
-nodeenv==1.3.2
+nodeenv==1.3.3
-pkginfo==1.4.2            # via twine
+pkginfo==1.5.0.1          # via twine
-pre-commit==1.11.2
+pre-commit==1.14.2
 pycodestyle==2.3.1        # via flake8
-pycparser==2.19           # via cffi
 pyflakes==1.6.0           # via flake8
-pygments==2.2.0           # via readme-renderer
+pygments==2.3.1           # via readme-renderer
 pyyaml==3.13              # via aspy.yaml, pre-commit
-readme-renderer==22.0     # via twine
+readme-renderer==24.0     # via twine
-requests-toolbelt==0.8.0  # via twine
+requests-toolbelt==0.9.0  # via twine
-requests==2.20.0          # via requests-toolbelt, twine
+requests==2.21.0          # via requests-toolbelt, twine
-six==1.11.0               # via bleach, cfgv, pre-commit, readme-renderer
+six==1.12.0               # via bleach, cfgv, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
-tqdm==4.28.1              # via twine
+tqdm==4.30.0              # via twine
 twine==1.12.1
-urllib3==1.24             # via requests
+urllib3==1.24.1           # via requests
-virtualenv==16.0.0        # via pre-commit
+virtualenv==16.3.0        # via pre-commit
 webencodings==0.5.1       # via bleach
+zipp==0.3.3               # via importlib-metadata

requirements-docs.txt

@@ -4,93 +4,93 @@
 #
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
-acme==0.27.1
+acme==0.30.2
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.1
+alembic==1.0.7
-amqp==2.3.2
+amqp==2.4.0
-aniso8601==3.0.2
+aniso8601==4.1.0
-arrow==0.12.1
+arrow==0.13.0
 asn1crypto==0.24.0
 asyncpool==1.0
 babel==2.6.0              # via sphinx
-bcrypt==3.1.4
+bcrypt==3.1.6
-billiard==3.5.0.4
+billiard==3.5.0.5
 blinker==1.4
-boto3==1.7.79
+boto3==1.9.86
-botocore==1.10.84
+botocore==1.12.86
 celery[redis]==4.2.1
-certifi==2018.10.15
+certifi==2018.11.29
 cffi==1.11.5
 chardet==3.0.4
 click==7.0
 cloudflare==2.1.0
-cryptography==2.3.1
+cryptography==2.5
 dnspython3==1.15.0
 dnspython==1.15.0
 docutils==0.14
 dyn==1.8.1
 flask-bcrypt==0.7.1
-flask-cors==3.0.6
+flask-cors==3.0.7
 flask-mail==0.9.1
-flask-migrate==2.1.1
+flask-migrate==2.3.1
 flask-principal==0.4.0
-flask-restful==0.3.6
+flask-restful==0.3.7
 flask-script==2.0.6
 flask-sqlalchemy==2.3.2
-flask==0.12
+flask==1.0.2
-future==0.16.0
+future==0.17.1
 gunicorn==19.9.0
-idna==2.7
+idna==2.8
 imagesize==1.1.0          # via sphinx
 inflection==0.3.1
-itsdangerous==1.0.0
+itsdangerous==1.1.0
 jinja2==2.10
 jmespath==0.9.3
 josepy==1.1.0
 jsonlines==1.2.0
-kombu==4.2.1
+kombu==4.2.2.post1
 lockfile==0.12.2
 mako==1.0.7
-markupsafe==1.0
+markupsafe==1.1.0
-marshmallow-sqlalchemy==0.14.1
+marshmallow-sqlalchemy==0.15.0
-marshmallow==2.16.0
+marshmallow==2.18.0
 mock==2.0.0
 ndg-httpsclient==0.5.1
-packaging==18.0           # via sphinx
+packaging==19.0           # via sphinx
 paramiko==2.4.2
-pbr==5.0.0
+pbr==5.1.1
 pem==18.2.0
-psycopg2==2.7.5
+psycopg2==2.7.7
-pyasn1-modules==0.2.2
+pyasn1-modules==0.2.4
-pyasn1==0.4.4
+pyasn1==0.4.5
 pycparser==2.19
-pygments==2.2.0           # via sphinx
+pygments==2.3.1           # via sphinx
-pyjwt==1.6.4
+pyjwt==1.7.1
 pynacl==1.3.0
-pyopenssl==18.0.0
+pyopenssl==19.0.0
-pyparsing==2.2.2          # via packaging
+pyparsing==2.3.1          # via packaging
 pyrfc3339==1.1
-python-dateutil==2.7.3
+python-dateutil==2.7.5
 python-editor==1.0.3
-pytz==2018.5
+pytz==2018.9
 pyyaml==3.13
-raven[flask]==6.9.0
+raven[flask]==6.10.0
 redis==2.10.6
-requests-toolbelt==0.8.0
+requests-toolbelt==0.9.0
-requests[security]==2.20.0
+requests[security]==2.21.0
 retrying==1.3.3
 s3transfer==0.1.13
-six==1.11.0
+six==1.12.0
 snowballstemmer==1.2.1    # via sphinx
 sphinx-rtd-theme==0.4.2
-sphinx==1.8.1
+sphinx==1.8.3
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
-sqlalchemy-utils==0.33.6
+sqlalchemy-utils==0.33.11
-sqlalchemy==1.2.12
+sqlalchemy==1.2.17
-tabulate==0.8.2
+tabulate==0.8.3
-urllib3==1.24
+urllib3==1.24.1
-vine==1.1.4
+vine==1.2.0
 werkzeug==0.14.1
 xmltodict==0.11.0

requirements-tests.in

@@ -4,7 +4,7 @@ coverage
 factory-boy
 Faker
 freezegun
-moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813
+moto
 nose
 pyflakes
 pytest

requirements-tests.txt

@@ -8,59 +8,57 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-biscuits==0.1.1           # via responses
+boto3==1.9.86             # via moto
-boto3==1.9.28             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.28         # via boto3, moto, s3transfer
+botocore==1.12.86         # via boto3, moto, s3transfer
-certifi==2018.10.15       # via requests
+certifi==2018.11.29       # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
 click==7.0                # via flask
-cookies==2.2.1            # via moto
+coverage==4.5.2
-coverage==4.5.1
+cryptography==2.5         # via moto
-cryptography==2.3.1       # via moto
+docker-pycreds==0.4.0     # via docker
-docker-pycreds==0.3.0     # via docker
+docker==3.7.0             # via moto
-docker==3.5.1             # via moto
 docutils==0.14            # via botocore
 ecdsa==0.13               # via python-jose
 factory-boy==2.11.1
-faker==0.9.2
+faker==1.0.2
 flask==1.0.2              # via pytest-flask
 freezegun==0.3.11
-future==0.16.0            # via python-jose
+future==0.17.1            # via python-jose
-idna==2.7                 # via cryptography, requests
+idna==2.8                 # via requests
-itsdangerous==1.0.0       # via flask
+itsdangerous==1.1.0       # via flask
 jinja2==2.10              # via flask, moto
 jmespath==0.9.3           # via boto3, botocore
 jsondiff==1.1.1           # via moto
-jsonpickle==1.0           # via aws-xray-sdk
+jsonpickle==1.1           # via aws-xray-sdk
-markupsafe==1.0           # via jinja2
+markupsafe==1.1.0         # via jinja2
 mock==2.0.0               # via moto
-more-itertools==4.3.0     # via pytest
+more-itertools==5.0.0     # via pytest
-moto==1.3.4
+moto==1.3.7
 nose==1.3.7
-pbr==5.0.0                # via mock
+pbr==5.1.1                # via mock
-pluggy==0.8.0             # via pytest
+pluggy==0.8.1             # via pytest
 py==1.7.0                 # via pytest
-pyaml==17.12.1            # via moto
+pyaml==18.11.0            # via moto
 pycparser==2.19           # via cffi
-pycryptodome==3.6.6       # via python-jose
+pycryptodome==3.7.3       # via python-jose
-pyflakes==2.0.0
+pyflakes==2.1.0
 pytest-flask==0.14.0
 pytest-mock==1.10.0
-pytest==3.9.1
+pytest==4.1.1
-python-dateutil==2.7.3    # via botocore, faker, freezegun, moto
+python-dateutil==2.7.5    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
-pytz==2018.5              # via moto
+pytz==2018.9              # via moto
 pyyaml==3.13              # via pyaml
 requests-mock==1.5.2
-requests==2.20.0          # via aws-xray-sdk, docker, moto, requests-mock, responses
+requests==2.21.0          # via aws-xray-sdk, docker, moto, requests-mock, responses
-responses==0.10.1         # via moto
+responses==0.10.5         # via moto
 s3transfer==0.1.13        # via boto3
-six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
+six==1.12.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker
-urllib3==1.23             # via botocore, requests
+urllib3==1.24.1           # via botocore, requests
-websocket-client==0.53.0  # via docker
+websocket-client==0.54.0  # via docker
 werkzeug==0.14.1          # via flask, moto, pytest-flask
-wrapt==1.10.11            # via aws-xray-sdk
+wrapt==1.11.1             # via aws-xray-sdk
 xmltodict==0.11.0         # via moto

requirements.in

@@ -4,22 +4,22 @@ acme
 alembic-autogenerate-enums
 arrow
 asyncpool
-boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813
+boto3
-botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813
+botocore
 celery[redis]
 certifi
 CloudFlare
 cryptography
 dnspython3
 dyn
-Flask-Bcrypt==0.7.1
+Flask-Bcrypt
-Flask-Mail==0.9.1
+Flask-Mail
-Flask-Migrate==2.1.1
+Flask-Migrate
-Flask-Principal==0.4.0
+Flask-Principal
-Flask-RESTful==0.3.6
+Flask-RESTful
-Flask-Script==2.0.6
+Flask-Script
 Flask-SQLAlchemy
-Flask==0.12
+Flask
 Flask-Cors
 future
 gunicorn
@@ -36,6 +36,7 @@ pyjwt
 pyOpenSSL
 python_ldap
 raven[flask]
+redis<3 # redis>=3 is not compatible with celery
 requests
 retrying
 six

requirements.txt

@@ -4,83 +4,83 @@
 #
 #    pip-compile --no-index --output-file requirements.txt requirements.in
 #
-acme==0.27.1
+acme==0.30.2
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.1            # via flask-migrate
+alembic==1.0.7            # via flask-migrate
-amqp==2.3.2               # via kombu
+amqp==2.4.0               # via kombu
-aniso8601==3.0.2          # via flask-restful
+aniso8601==4.1.0          # via flask-restful
-arrow==0.12.1
+arrow==0.13.0
 asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
-bcrypt==3.1.4             # via flask-bcrypt, paramiko
+bcrypt==3.1.6             # via flask-bcrypt, paramiko
-billiard==3.5.0.4         # via celery
+billiard==3.5.0.5         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.7.79
+boto3==1.9.86
-botocore==1.10.84
+botocore==1.12.86
 celery[redis]==4.2.1
-certifi==2018.10.15
+certifi==2018.11.29
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 click==7.0                # via flask
 cloudflare==2.1.0
-cryptography==2.3.1
+cryptography==2.5
 dnspython3==1.15.0
 dnspython==1.15.0         # via dnspython3
 docutils==0.14            # via botocore
 dyn==1.8.1
 flask-bcrypt==0.7.1
-flask-cors==3.0.6
+flask-cors==3.0.7
 flask-mail==0.9.1
-flask-migrate==2.1.1
+flask-migrate==2.3.1
 flask-principal==0.4.0
-flask-restful==0.3.6
+flask-restful==0.3.7
 flask-script==2.0.6
 flask-sqlalchemy==2.3.2
-flask==0.12
+flask==1.0.2
-future==0.16.0
+future==0.17.1
 gunicorn==19.9.0
-idna==2.7                 # via cryptography, requests
+idna==2.8                 # via requests
 inflection==0.3.1
-itsdangerous==1.0.0       # via flask
+itsdangerous==1.1.0       # via flask
 jinja2==2.10
 jmespath==0.9.3           # via boto3, botocore
 josepy==1.1.0             # via acme
 jsonlines==1.2.0          # via cloudflare
-kombu==4.2.1              # via celery
+kombu==4.2.2.post1        # via celery
 lockfile==0.12.2
 mako==1.0.7               # via alembic
-markupsafe==1.0           # via jinja2, mako
+markupsafe==1.1.0         # via jinja2, mako
-marshmallow-sqlalchemy==0.14.1
+marshmallow-sqlalchemy==0.15.0
-marshmallow==2.16.0
+marshmallow==2.18.0
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.2
-pbr==5.0.0                # via mock
+pbr==5.1.1                # via mock
 pem==18.2.0
-psycopg2==2.7.5
+psycopg2==2.7.7
-pyasn1-modules==0.2.2     # via python-ldap
+pyasn1-modules==0.2.4     # via python-ldap
-pyasn1==0.4.4             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
+pyasn1==0.4.5             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
 pycparser==2.19           # via cffi
-pyjwt==1.6.4
+pyjwt==1.7.1
 pynacl==1.3.0             # via paramiko
-pyopenssl==18.0.0
+pyopenssl==19.0.0
 pyrfc3339==1.1            # via acme
-python-dateutil==2.7.3    # via alembic, arrow, botocore
+python-dateutil==2.7.5    # via alembic, arrow, botocore
 python-editor==1.0.3      # via alembic
 python-ldap==3.1.0
-pytz==2018.5              # via acme, celery, flask-restful, pyrfc3339
+pytz==2018.9              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==3.13              # via cloudflare
-raven[flask]==6.9.0
+raven[flask]==6.10.0
-redis==2.10.6             # via celery
+redis==2.10.6
-requests-toolbelt==0.8.0  # via acme
+requests-toolbelt==0.9.0  # via acme
-requests[security]==2.20.0
+requests[security]==2.21.0
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
-six==1.11.0
+six==1.12.0
-sqlalchemy-utils==0.33.6
+sqlalchemy-utils==0.33.11
-sqlalchemy==1.2.12        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.2.17        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
-tabulate==0.8.2
+tabulate==0.8.3
-urllib3==1.24             # via requests
+urllib3==1.24.1           # via botocore, requests
-vine==1.1.4               # via amqp
+vine==1.2.0               # via amqp
 werkzeug==0.14.1          # via flask
 xmltodict==0.11.0