Merge branch 'master' into lemur_vault_plugin

Makefile

@@ -125,5 +125,9 @@ endif
 	@echo "--> Done installing new dependencies"
 	@echo ""
 
+# Execute with make checkout-pr pr=<pr number>
+checkout-pr:
+	git fetch upstream pull/$(pr)/head:pr-$(pr)
+
 
 .PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

lemur/certificates/schemas.py

@@ -112,10 +112,20 @@ class CertificateInputSchema(CertificateCreationSchema):
         if data.get('replacements'):
             data['replaces'] = data['replacements']  # TODO remove when field is deprecated
         if data.get('csr'):
-            dns_names = cert_utils.get_dns_names_from_csr(data['csr'])
+            csr_sans = cert_utils.get_sans_from_csr(data['csr'])
-            if not data['extensions']['subAltNames']['names']:
+            if not data.get('extensions'):
+                data['extensions'] = {
+                    'subAltNames': {
+                        'names': []
+                    }
+                }
+            elif not data['extensions'].get('subAltNames'):
+                data['extensions']['subAltNames'] = {
+                    'names': []
+                }
+            elif not data['extensions']['subAltNames'].get('names'):
                 data['extensions']['subAltNames']['names'] = []
-            data['extensions']['subAltNames']['names'] += dns_names
+            data['extensions']['subAltNames']['names'] += csr_sans
         return missing.convert_validity_years(data)
 
 
@@ -255,6 +265,7 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
     private_key = fields.String()
     body = fields.String(required=True)
     chain = fields.String(missing=None, allow_none=True)
+    csr = fields.String(required=False, allow_none=True, validate=validators.csr)
 
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)

lemur/certificates/utils.py

@@ -14,14 +14,14 @@ from cryptography.hazmat.backends import default_backend
 from marshmallow.exceptions import ValidationError
 
 
-def get_dns_names_from_csr(data):
+def get_sans_from_csr(data):
     """
-    Fetches DNSNames from CSR.
+    Fetches SubjectAlternativeNames from CSR.
-    Potentially extendable to any kind of SubjectAlternativeName
+    Works with any kind of SubjectAlternativeName
     :param data: PEM-encoded string with CSR
-    :return:
+    :return: List of LemurAPI-compatible subAltNames
     """
-    dns_names = []
+    sub_alt_names = []
     try:
         request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
     except Exception:
@@ -29,14 +29,12 @@ def get_dns_names_from_csr(data):
 
     try:
         alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
-
+        for alt_name in alt_names.value:
-        for name in alt_names.value.get_values_for_type(x509.DNSName):
+            sub_alt_names.append({
-            dns_name = {
+                'nameType': type(alt_name).__name__,
-                'nameType': 'DNSName',
+                'value': alt_name.value
-                'value': name
+            })
-            }
-            dns_names.append(dns_name)
     except x509.ExtensionNotFound:
         pass
 
-    return dns_names
+    return sub_alt_names

lemur/certificates/views.py

@@ -306,6 +306,7 @@ class CertificatesUpload(AuthenticatedResource):
                  "body": "-----BEGIN CERTIFICATE-----...",
                  "chain": "-----BEGIN CERTIFICATE-----...",
                  "privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
+                 "csr": "-----BEGIN CERTIFICATE REQUEST-----..."
                  "destinations": [],
                  "notifications": [],
                  "replacements": [],

lemur/common/celery.py

@@ -18,11 +18,10 @@ from lemur.authorities.service import get as get_authority
 from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
-from lemur.plugins.base import plugins, IPlugin
+from lemur.plugins.base import plugins
 from lemur.sources.cli import clean, sync, validate_sources
 from lemur.destinations import service as destinations_service
-from lemur.sources import service as sources_service
+from lemur.sources.service import add_aws_destination_to_sources
-
 
 if current_app:
     flask_app = current_app
@@ -266,27 +265,13 @@ def sync_source_destination():
     This celery task will sync destination and source, to make sure all new destinations are also present as source.
     Some destinations do not qualify as sources, and hence should be excluded from being added as sources
     We identify qualified destinations based on the sync_as_source attributed of the plugin.
-    The destination sync_as_source_name reviels the name of the suitable source-plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
     We rely on account numbers to avoid duplicates.
     """
-    current_app.logger.debug("Syncing source and destination")
+    current_app.logger.debug("Syncing AWS destinations and sources")
-
-    # a set of all accounts numbers available as sources
-    src_accounts = set()
-    sources = validate_sources("all")
-    for src in sources:
-        src_accounts.add(IPlugin.get_option('accountNumber', src.options))
 
     for dst in destinations_service.get_all():
-        destination_plugin = plugins.get(dst.plugin_name)
+        if add_aws_destination_to_sources(dst):
-        account_number = IPlugin.get_option('accountNumber', dst.options)
+            current_app.logger.debug("Source: %s added", dst.label)
-        if destination_plugin.sync_as_source and (account_number not in src_accounts):
+
-            src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
+    current_app.logger.debug("Completed Syncing AWS destinations and sources")
-            for o in src_options:
-                if o.get('name') == 'accountNumber':
-                    o.update({'value': account_number})
-            sources_service.create(label=dst.label,
-                                   plugin_name=destination_plugin.sync_as_source_name,
-                                   options=src_options,
-                                   description=dst.description)
-            current_app.logger.info("Source: %s added", dst.label)

lemur/destinations/service.py

@@ -6,11 +6,13 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from sqlalchemy import func
+from flask import current_app
 
 from lemur import database
 from lemur.models import certificate_destination_associations
 from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
+from lemur.sources.service import add_aws_destination_to_sources
 
 
 def create(label, plugin_name, options, description=None):
@@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None):
             del option['value']['plugin_object']
 
     destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
+    current_app.logger.info("Destination: %s created", label)
+
+    # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
+    if add_aws_destination_to_sources(destination):
+        current_app.logger.info("Source: %s created", label)
+
     return database.create(destination)
 
 

lemur/plugins/utils.py

@@ -18,4 +18,14 @@ def get_plugin_option(name, options):
     """
     for o in options:
         if o.get('name') == name:
-            return o['value']
+            return o.get('value', o.get('default'))
+
+
+def set_plugin_option(name, value, options):
+    """
+    Set value for option name for options dict.
+    :param options:
+    """
+    for o in options:
+        if o.get('name') == name:
+            o.update({'value': value})

lemur/sources/service.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+import copy
 
 from flask import current_app
 
@@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi
 from lemur.common.defaults import serial
 
 from lemur.plugins.base import plugins
+from lemur.plugins.utils import get_plugin_option, set_plugin_option
 
 
 def certificate_create(certificate, source):
@@ -256,3 +258,35 @@ def render(args):
         query = database.filter(query, Source, terms)
 
     return database.sort_and_page(query, Source, args)
+
+
+def add_aws_destination_to_sources(dst):
+    """
+    Given a destination check, if it can be added as sources, and included it if not already a source
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    :return: true for success and false for not adding the destination as source
+    """
+    # a set of all accounts numbers available as sources
+    src_accounts = set()
+    sources = get_all()
+    for src in sources:
+        src_accounts.add(get_plugin_option('accountNumber', src.options))
+
+    # check
+    destination_plugin = plugins.get(dst.plugin_name)
+    account_number = get_plugin_option('accountNumber', dst.options)
+    if account_number is not None and \
+            destination_plugin.sync_as_source is not None and \
+            destination_plugin.sync_as_source and \
+            (account_number not in src_accounts):
+        src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
+        set_plugin_option('accountNumber', account_number, src_options)
+        create(label=dst.label,
+               plugin_name=destination_plugin.sync_as_source_name,
+               options=src_options,
+               description=dst.description)
+        return True
+
+    return False

lemur/static/app/angular/certificates/certificate/upload.tpl.html

@@ -62,6 +62,19 @@
           a valid certificate.</p>
       </div>
     </div>
+    <div class="form-group"
+         ng-class="{'has-error': uploadForm.csr.$invalid&&uploadForm.csr.$dirty, 'has-success': !uploadForm.csr.$invalid&&uploadForm.csr.$dirty}">
+      <label class="control-label col-sm-2">
+        Certificate Signing Request (CSR)
+      </label>
+      <div class="col-sm-10">
+                <textarea name="csr" ng-model="certificate.csr" placeholder="PEM encoded string..."
+                          class="form-control"
+                          ng-pattern="/^-----BEGIN CERTIFICATE REQUEST-----/"></textarea>
+        <p ng-show="uploadForm.csr.$invalid && !uploadForm.csr.$pristine"
+             class="help-block">Enter a valid certificate signing request.</p>
+      </div>
+    </div>
     <div class="form-group"
          ng-class="{'has-error': uploadForm.owner.$invalid&&uploadform.intermediateCert.$dirty, 'has-success': !uploadForm.intermediateCert.$invalid&&uploadForm.intermediateCert.$dirty}">
       <label class="control-label col-sm-2">

lemur/tests/test_certificates.py

@@ -18,7 +18,7 @@ from lemur.domains.models import Domain
 
 
 from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \
-    INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
+    INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_CSR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR
 
 
 def test_get_or_increase_name(session, certificate):
@@ -284,6 +284,31 @@ def test_certificate_input_with_extensions(client, authority):
     assert not errors
 
 
+def test_certificate_input_schema_parse_csr(authority):
+    from lemur.certificates.schemas import CertificateInputSchema
+
+    test_san_dns = 'foobar.com'
+    extensions = {'sub_alt_names': {'names': x509.SubjectAlternativeName([x509.DNSName(test_san_dns)])}}
+    csr, private_key = create_csr(owner='joe@example.com', common_name='ACommonName', organization='test',
+                                  organizational_unit='Meters', country='NL', state='Noord-Holland', location='Amsterdam',
+                                  key_type='RSA2048', extensions=extensions)
+
+    input_data = {
+        'commonName': 'test.example.com',
+        'owner': 'jim@example.com',
+        'authority': {'id': authority.id},
+        'description': 'testtestest',
+        'csr': csr,
+        'dnsProvider': None,
+    }
+
+    data, errors = CertificateInputSchema().load(input_data)
+
+    for san in data['extensions']['sub_alt_names']['names']:
+        assert san.value == test_san_dns
+    assert not errors
+
+
 def test_certificate_out_of_range_date(client, authority):
     from lemur.certificates.schemas import CertificateInputSchema
     input_data = {
@@ -456,6 +481,7 @@ def test_certificate_upload_schema_ok(client):
         'body': SAN_CERT_STR,
         'privateKey': SAN_CERT_KEY,
         'chain': INTERMEDIATE_CERT_STR,
+        'csr': SAN_CERT_CSR,
         'external_id': '1234',
     }
     data, errors = CertificateUploadInputSchema().load(data)

lemur/tests/vectors.py

@@ -137,6 +137,26 @@ eMVHHbWm1CpGO294R+vMBv4jcuhIBOx63KZE4VaoJuaazF6TE5czDw==
 
 
 #: CN=san.example.org, issued by LemurTrust Unittests Class 1 CA 2018
+SAN_CERT_CSR = """\
+-----BEGIN CERTIFICATE REQUEST-----
+MIICvTCCAaUCAQAweDELMAkGA1UEBhMCRUUxDDAKBgNVBAgMA04vQTEOMAwGA1UE
+BwwFRWFydGgxGDAWBgNVBAoMD0RhbmllbCBTYW4gJiBjbzEXMBUGA1UECwwOS2Fy
+YXRlIExlc3NvbnMxGDAWBgNVBAMMD3Nhbi5leGFtcGxlLm9yZzCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAMia9BcpypZUU9xJoknzdEp+AevQE93XSAyl
+IlXji80ZlYS/T/mVWtu6hNwz2IJDBFh6nPaHT1Ud/AI4YanDMa+fF4KJxzlkKPbY
+quWx4EOjTZ2sFBBCivwxlo1So8r5Hf4NZ9Ewu4AIma3zmk+dzxJTpnWbTIFJGsDG
+LwJO9iu6uqf79VdYkGELCusq3dyF2j2DNDiGHoRcQYFMMhDKR6uYmCTYvwjf0+sf
+6k1zk2EK1X+ZWUyjP+Nl2NB6bpL0TydF75fuplWROczceiO6BKO4YT2uNPdF4BAH
+p/kQCkqnjw5FCX7PONRT4wTW/AjDkt5WOgY+AB90zQBPxvXWbUMCAwEAAaAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQAFYgEafwRmsqdK1i1xrLFYbNNLkzmAZyL+6gXUBVIJ
+TbGVVWSNNIcEmHIX8O9X4lN52qDYWOsxH/OKPVxpXqoHm/ztczFlte76wOYg+VAS
+yK8DwQRP/+n+j6J40o1cZwnilPWqHgee5zbIL7lpCVxuFDofWpskwP5PLbxibFq8
+4TWynhjKKUw4+q4h4iCHG3PQhbV0ExWOyqX05QyDtJdkEwgJUWz1m9caHU2Jl7kX
+5bWKOtXORpCYA7ed3WqktKQIxBD6vCVbQ+LuLZPYeWzGHYjfOejL6usD32KmNa2E
+ZhDsC0fjqSX0FJKz6gOhP88bkbbapyHuGB71o2dwhCKV
+-----END CERTIFICATE REQUEST-----
+"""
+
 SAN_CERT_STR = """\
 -----BEGIN CERTIFICATE-----
 MIIESjCCAzKgAwIBAgIRAK/y20+NLU2OgPo4KuJ8IzMwDQYJKoZIhvcNAQELBQAw

requirements-docs.txt

@@ -7,7 +7,7 @@
 acme==0.33.1
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.8
+alembic==1.0.9
 amqp==2.4.2
 aniso8601==6.0.0
 arrow==0.13.1
@@ -53,9 +53,9 @@ josepy==1.1.0
 jsonlines==1.2.0
 kombu==4.5.0
 lockfile==0.12.2
-mako==1.0.8
+mako==1.0.9
 markupsafe==1.1.1
-marshmallow-sqlalchemy==0.16.1
+marshmallow-sqlalchemy==0.16.2
 marshmallow==2.19.2
 mock==2.0.0
 ndg-httpsclient==0.5.1
@@ -63,7 +63,7 @@ packaging==19.0           # via sphinx
 paramiko==2.4.2
 pbr==5.1.3
 pem==19.1.0
-psycopg2==2.8.1
+psycopg2==2.8.2
 pyasn1-modules==0.2.4
 pyasn1==0.4.5
 pycparser==2.19
@@ -89,13 +89,13 @@ sphinx-rtd-theme==0.4.3
 sphinx==2.0.1
 sphinxcontrib-applehelp==1.0.1  # via sphinx
 sphinxcontrib-devhelp==1.0.1  # via sphinx
-sphinxcontrib-htmlhelp==1.0.1  # via sphinx
+sphinxcontrib-htmlhelp==1.0.2  # via sphinx
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.2  # via sphinx
 sphinxcontrib-serializinghtml==1.1.3  # via sphinx
 sqlalchemy-utils==0.33.11
-sqlalchemy==1.3.2
+sqlalchemy==1.3.3
 tabulate==0.8.3
 urllib3==1.24.1
 vine==1.3.0

requirements-tests.txt

@@ -22,7 +22,7 @@ docker==3.7.2             # via moto
 docutils==0.14            # via botocore
 ecdsa==0.13               # via python-jose
 factory-boy==2.11.1
-faker==1.0.4
+faker==1.0.5
 flask==1.0.2              # via pytest-flask
 freezegun==0.3.11
 future==0.17.1            # via python-jose
@@ -46,7 +46,7 @@ pycryptodome==3.8.1       # via python-jose
 pyflakes==2.1.1
 pytest-flask==0.14.0
 pytest-mock==1.10.3
-pytest==4.4.0
+pytest==4.4.1
 python-dateutil==2.8.0    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
 pytz==2019.1              # via moto

requirements.txt

@@ -6,7 +6,7 @@
 #
 acme==0.33.1
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.8            # via flask-migrate
+alembic==1.0.9            # via flask-migrate
 amqp==2.4.2               # via kombu
 aniso8601==6.0.0          # via flask-restful
 arrow==0.13.1
@@ -50,16 +50,16 @@ josepy==1.1.0             # via acme
 jsonlines==1.2.0          # via cloudflare
 kombu==4.5.0
 lockfile==0.12.2
-mako==1.0.8               # via alembic
+mako==1.0.9               # via alembic
 markupsafe==1.1.1         # via jinja2, mako
-marshmallow-sqlalchemy==0.16.1
+marshmallow-sqlalchemy==0.16.2
 marshmallow==2.19.2
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.2
 pbr==5.1.3                # via mock
 pem==19.1.0
-psycopg2==2.8.1
+psycopg2==2.8.2
 pyasn1-modules==0.2.4     # via python-ldap
 pyasn1==0.4.5             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
 pycparser==2.19           # via cffi
@@ -80,7 +80,7 @@ retrying==1.3.3
 s3transfer==0.2.0         # via boto3
 six==1.12.0
 sqlalchemy-utils==0.33.11
-sqlalchemy==1.3.2         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.3.3         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.3
 urllib3==1.24.1           # via botocore, requests
 vine==1.3.0               # via amqp, celery