Merge pull request #3485 from hosseinsh/changelog-0.9.0

release 0.9.0 changelog

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# These owners will be the default owners for everything in the repo.
+*                            @hosseinsh @csine-nflx @charhate @jtschladen

.github/dependabot.yml

@@ -0,0 +1,15 @@
+  version: 2
+  updates:
+    - directory: "/"
+      schedule:
+        interval: "weekly"
+        day: "monday"
+        time: "08:00"
+        timezone: "America/Los_Angeles"
+      package-ecosystem: "pip"
+      reviewers:
+        - "hosseinsh"
+        - "csine-nflx"
+        - "charhate"
+        - "jtschladen"
+      versioning-strategy: lockfile-only

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '15 16 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    
+    # Install prerequisites for python-ldap. See: https://www.python-ldap.org/en/python-ldap-3.3.0/installing.html#build-prerequisites
+    - name: Install python-ldap prerequisites
+      run: sudo apt-get install libldap2-dev libsasl2-dev
+    
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dependabot-auto-merge.yml

@@ -0,0 +1,14 @@
+name: dependabot-auto-merge
+
+on:
+  pull_request:
+
+jobs:
+  auto-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ahmadnassri/action-dependabot-auto-merge@v2
+        with:
+          target: minor
+          github-token: ${{ secrets.DEPENDABOT_GITHUB_TOKEN }}

CHANGELOG.rst

@@ -1,6 +1,13 @@
 Changelog
 =========
 
+0.9.0 - `2021-03-17`
+~~~~~~~~~~~~~~~~~~~~
+
+This release fixes three critical vulnerabilities where an authenticated user could retrieve/access
+unauthorized information. (Issue `#3463 <https://github.com/Netflix/lemur/issues/3463>`_)
+
+
 0.8.1 - `2021-03-12`
 ~~~~~~~~~~~~~~~~~~~~
 

docs/doing-a-release.rst

@@ -1,10 +1,18 @@
 Doing a release
 ===============
 
-Doing a release of ``lemur`` requires a few steps.
+Doing a release of ``lemur`` is now mostly automated and consists of the following steps:
 
-Bumping the version number
+* Raise a PR to add the release date and summary in the :doc:`/changelog`.
---------------------------
+* Merge above PR and create a new `Github release <https://github.com/Netflix/lemur/releaes>`_: set the tag starting with v, e.g., v0.9.0
+
+The `publish workflow <https://github.com/Netflix/lemur/actions/workflows/lemur-publish-release-pypi.yml>`_ uses the git
+tag to set the release version.
+
+The following describes the manual release steps, which is now obsolete:
+
+Manually Bumping the version number
+-----------------------------------
 
 The next step in doing a release is bumping the version number in the
 software.
@@ -14,8 +22,8 @@ software.
 * Do a commit indicating this, and raise a pull request with this.
 * Wait for it to be merged.
 
-Performing the release
+Manually Performing the release
-----------------------
+-------------------------------
 
 The commit that merged the version number bump is now the official release
 commit for this release. You need an `API key <https://pypi.org/manage/account/#api-tokens>`_,

lemur/authorities/service.py

@@ -117,6 +117,12 @@ def create(**kwargs):
     """
     Creates a new authority.
     """
+    ca_name = kwargs.get("name")
+    if get_by_name(ca_name):
+        raise Exception(f"Authority with name {ca_name} already exists")
+    if role_service.get_by_name(f"{ca_name}_admin") or role_service.get_by_name(f"{ca_name}_operator"):
+        raise Exception(f"Admin and/or operator roles for authority {ca_name} already exist")
+
     body, private_key, chain, roles = mint(**kwargs)
 
     kwargs["creator"].roles = list(set(list(kwargs["creator"].roles) + roles))

lemur/certificates/service.py

@@ -679,7 +679,16 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    if kwargs.get("metric") == "not_after":
+
+    # Verify requested metric
+    allow_list = ["bits", "issuer", "not_after", "signing_algorithm"]
+    req_metric = kwargs.get("metric")
+    if req_metric not in allow_list:
+        raise Exception(
+            f"Stats not available for requested metric: {req_metric}"
+        )
+
+    if req_metric == "not_after":
         start = arrow.utcnow()
         end = start.shift(weeks=+32)
         items = (
@@ -691,7 +700,7 @@ def stats(**kwargs):
         )
 
     else:
-        attr = getattr(Certificate, kwargs.get("metric"))
+        attr = getattr(Certificate, req_metric)
         query = database.db.session.query(attr, func.count(attr))
 
         items = query.group_by(attr).all()

lemur/certificates/views.py

@@ -635,7 +635,12 @@ class CertificatesStats(AuthenticatedResource):
 
         args = self.reqparse.parse_args()
 
-        items = service.stats(**args)
+        try:
+            items = service.stats(**args)
+        except Exception as e:
+            sentry.captureException()
+            return dict(message=f"Failed to retrieve stats: {str(e)}"), 400
+
         return dict(items=items, total=len(items))
 
 

lemur/destinations/views.py

@@ -425,7 +425,7 @@ class CertificateDestinations(AuthenticatedResource):
 
 
 class DestinationsStats(AuthenticatedResource):
-    """ Defines the 'certificates' stats endpoint """
+    """ Defines the 'destinations' stats endpoint """
 
     def __init__(self):
         self.reqparse = reqparse.RequestParser()

lemur/dns_providers/schemas.py

@@ -10,9 +10,9 @@ class DnsProvidersNestedOutputSchema(LemurOutputSchema):
     name = fields.String()
     provider_type = fields.String()
     description = fields.String()
-    credentials = fields.String()
     api_endpoint = fields.String()
     date_created = ArrowDateTime()
+    # credentials are intentionally omitted (they are input-only)
 
 
 class DnsProvidersNestedInputSchema(LemurInputSchema):

lemur/tests/conftest.py

@@ -36,6 +36,7 @@ from .factories import (
     InvalidCertificateFactory,
     CryptoAuthorityFactory,
     CACertificateFactory,
+    DnsProviderFactory,
 )
 
 
@@ -183,6 +184,13 @@ def user(session):
     return {"user": u, "token": token}
 
 
+@pytest.fixture
+def dns_provider(session):
+    d = DnsProviderFactory()
+    session.commit()
+    return d
+
+
 @pytest.fixture
 def pending_certificate(session):
     u = UserFactory()

lemur/tests/factories.py

@@ -1,14 +1,15 @@
+import json
 from datetime import date
 
 from factory import Sequence, post_generation, SubFactory
 from factory.alchemy import SQLAlchemyModelFactory
 from factory.fuzzy import FuzzyChoice, FuzzyText, FuzzyDate, FuzzyInteger
 
-
 from lemur.database import db
 from lemur.authorities.models import Authority
 from lemur.certificates.models import Certificate
 from lemur.destinations.models import Destination
+from lemur.dns_providers.models import DnsProvider
 from lemur.sources.models import Source
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
@@ -435,3 +436,17 @@ class PendingCertificateFactory(BaseFactory):
         if extracted:
             for domain in extracted:
                 self.roles.append(domain)
+
+
+class DnsProviderFactory(BaseFactory):
+    """DnsProvider Factory."""
+
+    name = Sequence(lambda n: f"dnsProvider{n}")
+    description = FuzzyText(length=128)
+    provider_type = FuzzyText(length=128)
+    credentials = json.dumps({"account_id": f"{FuzzyInteger(100000, 999999).fuzz()}"})
+
+    class Meta:
+        """Factory Configuration."""
+
+        model = DnsProvider

lemur/tests/test_dns_providers.py

@@ -1,5 +1,7 @@
+import json
 import unittest
 from lemur.dns_providers import util as dnsutil
+from lemur.dns_providers.schemas import dns_provider_output_schema
 
 
 class TestDNSProvider(unittest.TestCase):
@@ -21,3 +23,17 @@ class TestDNSProvider(unittest.TestCase):
         self.assertFalse(dnsutil.is_valid_domain('example..io'))
         self.assertFalse(dnsutil.is_valid_domain('exa mple.io'))
         self.assertFalse(dnsutil.is_valid_domain('-'))
+
+
+def test_output_schema(dns_provider):
+    # no credentials using the output schema dump
+    assert dns_provider.credentials
+    assert json.loads(dns_provider.credentials)["account_id"]
+    dump = dns_provider_output_schema.dump(dns_provider).data
+    assert 'name' in dump
+    assert 'credentials' not in dump
+
+
+def test_json(dns_provider):
+    # we can still get credentials using json.load
+    assert 'account_id' in json.loads(dns_provider.credentials)

requirements-dev.txt

@@ -50,7 +50,7 @@ packaging==20.9
     # via bleach
 pkginfo==1.5.0.1
     # via twine
-pre-commit==2.11.0
+pre-commit==2.11.1
     # via -r requirements-dev.in
 pycodestyle==2.6.0
     # via flake8

requirements-docs.txt

@@ -5,7 +5,10 @@
 #    pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
 #
 acme==1.13.0
-    # via -r requirements-docs.in
+    # via
+    #   -r requirements-docs.in
+    #   -r requirements-tests.txt
+    #   certbot
 alabaster==0.7.12
     # via sphinx
 alembic==1.5.5
@@ -48,7 +51,7 @@ blinker==1.4
     #   flask-mail
     #   flask-principal
     #   raven
-boto3==1.17.22
+boto3==1.17.27
     # via
     #   -r requirements-docs.in
     #   -r requirements-tests.txt
@@ -58,7 +61,7 @@ boto==2.49.0
     # via
     #   -r requirements-tests.txt
     #   moto
-botocore==1.20.22
+botocore==1.20.27
     # via
     #   -r requirements-docs.in
     #   -r requirements-tests.txt
@@ -67,6 +70,9 @@ botocore==1.20.22
     #   moto
     #   s3transfer
 certbot==1.13.0
+    # via
+    #   -r requirements-docs.in
+    #   -r requirements-tests.txt
 certifi==2020.12.5
     # via
     #   -r requirements-tests.txt
@@ -94,6 +100,14 @@ click==7.1.2
     #   flask
 cloudflare==2.8.15
     # via -r requirements-docs.in
+configargparse==1.4
+    # via
+    #   -r requirements-tests.txt
+    #   certbot
+configobj==5.0.6
+    # via
+    #   -r requirements-tests.txt
+    #   certbot
 coverage==5.5
     # via -r requirements-tests.txt
 cryptography==3.4.6
@@ -101,6 +115,7 @@ cryptography==3.4.6
     #   -r requirements-docs.in
     #   -r requirements-tests.txt
     #   acme
+    #   certbot
     #   josepy
     #   moto
     #   paramiko
@@ -111,6 +126,10 @@ decorator==4.4.2
     # via
     #   -r requirements-tests.txt
     #   networkx
+distro==1.5.0
+    # via
+    #   -r requirements-tests.txt
+    #   certbot
 dnspython3==1.15.0
     # via -r requirements-docs.in
 dnspython==1.15.0
@@ -226,7 +245,9 @@ jmespath==0.9.5
 josepy==1.7.0
     # via
     #   -r requirements-docs.in
+    #   -r requirements-tests.txt
     #   acme
+    #   certbot
 jsondiff==1.1.2
     # via
     #   -r requirements-tests.txt
@@ -293,6 +314,10 @@ packaging==20.3
     #   sphinx
 paramiko==2.7.2
     # via -r requirements-docs.in
+parsedatetime==2.6
+    # via
+    #   -r requirements-tests.txt
+    #   certbot
 pathspec==0.8.0
     # via
     #   -r requirements-tests.txt
@@ -339,6 +364,7 @@ pynacl==1.4.0
 pyopenssl==20.0.1
     # via
     #   -r requirements-docs.in
+    #   -r requirements-tests.txt
     #   acme
     #   josepy
 pyparsing==2.4.7
@@ -346,7 +372,10 @@ pyparsing==2.4.7
     #   -r requirements-tests.txt
     #   packaging
 pyrfc3339==1.1
-    # via acme
+    # via
+    #   -r requirements-tests.txt
+    #   acme
+    #   certbot
 pyrsistent==0.16.0
     # via
     #   -r requirements-tests.txt
@@ -382,6 +411,7 @@ pytz==2019.3
     #   -r requirements-tests.txt
     #   acme
     #   babel
+    #   certbot
     #   flask-restful
     #   moto
     #   pyrfc3339
@@ -406,7 +436,9 @@ regex==2020.4.4
 requests-mock==1.8.0
     # via -r requirements-tests.txt
 requests-toolbelt==0.9.1
-    # via acme
+    # via
+    #   -r requirements-tests.txt
+    #   acme
 requests==2.25.1
     # via
     #   -r requirements-tests.txt
@@ -441,6 +473,7 @@ six==1.15.0
     #   bandit
     #   bcrypt
     #   cfn-lint
+    #   configobj
     #   docker
     #   ecdsa
     #   fakeredis
@@ -564,6 +597,36 @@ zipp==3.1.0
     #   -r requirements-tests.txt
     #   importlib-metadata
     #   moto
+zope.component==4.6.2
+    # via
+    #   -r requirements-tests.txt
+    #   certbot
+zope.deferredimport==4.3.1
+    # via
+    #   -r requirements-tests.txt
+    #   zope.component
+zope.deprecation==4.4.0
+    # via
+    #   -r requirements-tests.txt
+    #   zope.component
+zope.event==4.5.0
+    # via
+    #   -r requirements-tests.txt
+    #   zope.component
+zope.hookable==5.0.1
+    # via
+    #   -r requirements-tests.txt
+    #   zope.component
+zope.interface==5.2.0
+    # via
+    #   -r requirements-tests.txt
+    #   certbot
+    #   zope.component
+    #   zope.proxy
+zope.proxy==4.3.5
+    # via
+    #   -r requirements-tests.txt
+    #   zope.deferredimport
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements-tests.txt

@@ -4,6 +4,8 @@
 #
 #    pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in
 #
+acme==1.13.0
+    # via certbot
 appdirs==1.4.3
     # via black
 attrs==19.3.0
@@ -18,19 +20,20 @@ bandit==1.7.0
     # via -r requirements-tests.in
 black==20.8b1
     # via -r requirements-tests.in
-boto3==1.17.22
+boto3==1.17.27
     # via
     #   aws-sam-translator
     #   moto
 boto==2.49.0
     # via moto
-botocore==1.20.22
+botocore==1.20.27
     # via
     #   aws-xray-sdk
     #   boto3
     #   moto
     #   s3transfer
 certbot==1.13.0
+    # via -r requirements-tests.in
 certifi==2020.12.5
     # via requests
 cffi==1.14.0
@@ -43,15 +46,25 @@ click==7.1.2
     # via
     #   black
     #   flask
+configargparse==1.4
+    # via certbot
+configobj==5.0.6
+    # via certbot
 coverage==5.5
     # via -r requirements-tests.in
 cryptography==3.4.6
     # via
+    #   acme
+    #   certbot
+    #   josepy
     #   moto
+    #   pyopenssl
     #   python-jose
     #   sshpubkeys
 decorator==4.4.2
     # via networkx
+distro==1.5.0
+    # via certbot
 docker==4.2.0
     # via moto
 ecdsa==0.14.1
@@ -95,6 +108,10 @@ jmespath==0.9.5
     # via
     #   boto3
     #   botocore
+josepy==1.7.0
+    # via
+    #   acme
+    #   certbot
 jsondiff==1.1.2
     # via moto
 jsonpatch==1.25
@@ -125,6 +142,8 @@ nose==1.3.7
     # via -r requirements-tests.in
 packaging==20.3
     # via pytest
+parsedatetime==2.6
+    # via certbot
 pathspec==0.8.0
     # via black
 pbr==5.4.5
@@ -141,8 +160,16 @@ pycparser==2.20
     # via cffi
 pyflakes==2.2.0
     # via -r requirements-tests.in
+pyopenssl==20.0.1
+    # via
+    #   acme
+    #   josepy
 pyparsing==2.4.7
     # via packaging
+pyrfc3339==1.1
+    # via
+    #   acme
+    #   certbot
 pyrsistent==0.16.0
     # via jsonschema
 pytest-flask==1.2.0
@@ -163,7 +190,11 @@ python-dateutil==2.8.1
 python-jose[cryptography]==3.1.0
     # via moto
 pytz==2019.3
-    # via moto
+    # via
+    #   acme
+    #   certbot
+    #   moto
+    #   pyrfc3339
 pyyaml==5.4.1
     # via
     #   -r requirements-tests.in
@@ -176,11 +207,15 @@ regex==2020.4.4
     # via black
 requests-mock==1.8.0
     # via -r requirements-tests.in
+requests-toolbelt==0.9.1
+    # via acme
 requests==2.25.1
     # via
+    #   acme
     #   docker
     #   moto
     #   requests-mock
+    #   requests-toolbelt
     #   responses
 responses==0.10.12
     # via moto
@@ -193,12 +228,15 @@ six==1.15.0
     #   aws-sam-translator
     #   bandit
     #   cfn-lint
+    #   configobj
     #   docker
     #   ecdsa
     #   fakeredis
+    #   josepy
     #   jsonschema
     #   moto
     #   packaging
+    #   pyopenssl
     #   pyrsistent
     #   python-dateutil
     #   python-jose
@@ -243,6 +281,23 @@ zipp==3.1.0
     # via
     #   importlib-metadata
     #   moto
+zope.component==4.6.2
+    # via certbot
+zope.deferredimport==4.3.1
+    # via zope.component
+zope.deprecation==4.4.0
+    # via zope.component
+zope.event==4.5.0
+    # via zope.component
+zope.hookable==5.0.1
+    # via zope.component
+zope.interface==5.2.0
+    # via
+    #   certbot
+    #   zope.component
+    #   zope.proxy
+zope.proxy==4.3.5
+    # via zope.deferredimport
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements.txt

@@ -5,7 +5,9 @@
 #    pip-compile --no-index --output-file=requirements.txt requirements.in
 #
 acme==1.13.0
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   certbot
 alembic-autogenerate-enums==0.0.2
     # via -r requirements.in
 alembic==1.4.2
@@ -31,9 +33,9 @@ blinker==1.4
     #   flask-mail
     #   flask-principal
     #   raven
-boto3==1.17.22
+boto3==1.17.27
     # via -r requirements.in
-botocore==1.20.22
+botocore==1.20.27
     # via
     #   -r requirements.in
     #   boto3
@@ -41,6 +43,7 @@ botocore==1.20.22
 celery[redis]==4.4.2
     # via -r requirements.in
 certbot==1.13.0
+    # via -r requirements.in
 certifi==2020.12.5
     # via
     #   -r requirements.in
@@ -58,13 +61,20 @@ click==7.1.2
     # via flask
 cloudflare==2.8.15
     # via -r requirements.in
+configargparse==1.4
+    # via certbot
+configobj==5.0.6
+    # via certbot
 cryptography==3.4.6
     # via
     #   -r requirements.in
     #   acme
+    #   certbot
     #   josepy
     #   paramiko
     #   pyopenssl
+distro==1.5.0
+    # via certbot
 dnspython3==1.15.0
     # via -r requirements.in
 dnspython==1.15.0
@@ -126,7 +136,9 @@ jmespath==0.9.5
     #   boto3
     #   botocore
 josepy==1.7.0
-    # via acme
+    # via
+    #   acme
+    #   certbot
 jsonlines==1.2.0
     # via cloudflare
 kombu==4.6.8
@@ -151,6 +163,8 @@ ndg-httpsclient==0.5.1
     # via -r requirements.in
 paramiko==2.7.2
     # via -r requirements.in
+parsedatetime==2.6
+    # via certbot
 pem==21.1.0
     # via -r requirements.in
 psycopg2==2.8.6
@@ -182,7 +196,9 @@ pyopenssl==20.0.1
     #   josepy
     #   ndg-httpsclient
 pyrfc3339==1.1
-    # via acme
+    # via
+    #   acme
+    #   certbot
 python-dateutil==2.8.1
     # via
     #   alembic
@@ -198,6 +214,7 @@ pytz==2019.3
     # via
     #   acme
     #   celery
+    #   certbot
     #   flask-restful
     #   pyrfc3339
 pyyaml==5.4.1
@@ -228,6 +245,7 @@ six==1.15.0
     # via
     #   -r requirements.in
     #   bcrypt
+    #   configobj
     #   flask-cors
     #   flask-restful
     #   hvac
@@ -264,6 +282,22 @@ werkzeug==1.0.1
     # via flask
 xmltodict==0.12.0
     # via -r requirements.in
+zope.component==4.6.2
+    # via certbot
+zope.deferredimport==4.3.1
+    # via zope.component
+zope.deprecation==4.4.0
+    # via zope.component
+zope.event==4.5.0
+    # via zope.component
+zope.hookable==5.0.1
+    # via zope.component
+zope.interface==5.2.0
+    # via
+    #   certbot
+    #   zope.component
+zope.proxy==4.3.5
+    # via zope.deferredimport
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools