diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..46b1d24d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ + version: 2 + updates: + - directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "08:00" + timezone: "America/Los_Angeles" + package-ecosystem: "pip" + reviewers: + - "hosseinsh" + - "csine-nflx" + - "charhate" + - "jtschladen" + versioning-strategy: lockfile-only \ No newline at end of file diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml new file mode 100644 index 00000000..be012941 --- /dev/null +++ b/.github/workflows/dependabot-auto-merge.yml @@ -0,0 +1,14 @@ +name: dependabot-auto-merge + +on: + pull_request: + +jobs: + auto-merge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + target: minor + github-token: ${{ secrets.DEPENDABOT_GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/lemur-publish-release-pypi.yml b/.github/workflows/lemur-publish-release-pypi.yml new file mode 100644 index 00000000..816146d0 --- /dev/null +++ b/.github/workflows/lemur-publish-release-pypi.yml @@ -0,0 +1,41 @@ +# This workflow will upload a Python Package using Twine when a Lemur release is created via github +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Publish Lemur's latest package to PyPI + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Autobump version + run: | + # from refs/tags/v0.8.1 get 0.8.1 + VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + PLACEHOLDER='__version__ = "develop"' + VERSION_FILE='lemur/__about__.py' + # in case placeholder is missing, exists with code 1 and github actions aborts the build + grep "$PLACEHOLDER" "$VERSION_FILE" + sed -i "s/$PLACEHOLDER/__version__ = \"${VERSION}\"/g" "$VERSION_FILE" + shell: bash + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.LEMUR_PYPI_API_USERNAME }} + TWINE_PASSWORD: ${{ secrets.LEMUR_PYPI_API_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.gitignore b/.gitignore index 72e85f26..02a24ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ lemur/tests/tmp /lemur/plugins/lemur_email/tests/expiration-rendered.html /lemur/plugins/lemur_email/tests/rotation-rendered.html +.celerybeat-schedule diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..d41769a8 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,23 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + fail_on_warning: true + +# Build docs in all formats (html, pdf, epub) +formats: all + +# Set the version of Python and requirements required to build the docs +python: + version: 3.7 + install: + - requirements: requirements-docs.txt + - method: setuptools + path: . + system_packages: true \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 67b792f8..a470bdc4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,8 +1,39 @@ Changelog ========= +0.8.1 - `2021-03-12` +~~~~~~~~~~~~~~~~~~~~ + +This release includes improvements on many fronts, such as: + +- Notifications: + - Enhanced SNS flow + - Expiration Summary + - CA expiration email +- EC algorithm as the default +- Improved revocation flow +- Localized AWS STS option +- Improved Lemur doc building +- ACME: + - reduced failed attempts to 3x trials + - support for selecting the chain (Let's Encrypt X1 transition) + - revocation + - http01 documentation +- Entrust: + - Support for cross-signed intermediate CA +- Revised disclosure process +- Dependency updates and conflict resolutions + +Special thanks to all who contributed to this release, notably: + +- `peschmae `_ +- `atugushev `_ +- `sirferl `_ + + + 0.8.0 - `2020-11-13` -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ This release comes after more than two years and contains many interesting new features and improvements. In addition to multiple new plugins, such as ACME-http01, ADCS, PowerDNS, UltraDNS, Entrust, SNS, many of Lemur's existing @@ -84,7 +115,7 @@ Upgrading 0.7 - `2018-05-07` -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality. The linux_dst plugin will also be deprecated and removed. @@ -121,8 +152,7 @@ Happy Holidays! This is a big release with lots of bug fixes and features. Below Features: -* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates. -is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used. +* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used. * Adds per-user API Keys, allows users to issue multiple long-lived API tokens with the same permission as the user creating them. * Adds the ability to revoke certificates from the Lemur UI/API, this is currently only supported for the digicert CIS and cfssl plugins. * Allow destinations to support an export function. Useful for file system destinations e.g. S3 to specify the export plugin you wish to run before being sent to the destination. @@ -166,13 +196,9 @@ Big thanks to neilschelly for quite a lot of improvements to the `lemur-cryptogr Other Highlights: -* Closed `#501 `_ - Endpoint resource as now kept in sync via an -expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never -removed from Lemur. -* Closed `#551 `_ - Added the ability to create a 4096 bit key during certificate -creation. Closed `#528 `_ to ensure that issuer plugins supported the new 4096 bit keys. -* Closed `#566 `_ - Fixed an issue changing the notification status for certificates -without private keys. +* Closed `#501 `_ - Endpoint resource as now kept in sync via an expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never removed from Lemur. +* Closed `#551 `_ - Added the ability to create a 4096 bit key during certificate creation. Closed `#528 `_ to ensure that issuer plugins supported the new 4096 bit keys. +* Closed `#566 `_ - Fixed an issue changing the notification status for certificates without private keys. * Closed `#594 `_ - Added `replaced` field indicating if a certificate has been superseded. * Closed `#602 `_ - AWS plugin added support for ALBs for endpoint tracking. @@ -196,12 +222,8 @@ Upgrading There have been quite a few issues closed in this release. Some notables: -* Closed `#284 `_ - Created new models for `Endpoints` created associated -AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for -future enhancements of Lemur's certificate 'deployment' capabilities. - -* Closed `#334 `_ - Lemur not has the ability -to restrict certificate expiration dates to weekdays. +* Closed `#284 `_ - Created new models for `Endpoints` created associated AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for future enhancements of Lemur's certificate 'deployment' capabilities. +* Closed `#334 `_ - Lemur not has the ability to restrict certificate expiration dates to weekdays. Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!) @@ -256,7 +278,7 @@ these keys should be fairly trivial, additionally pull requests have been submit should be easier to determine what authorities are available and when an authority has actually been selected. * Closed `#254 `_ - Forces certificate names to be generally unique. If a certificate name (generated or otherwise) is found to be a duplicate we increment by appending a counter. -* Closed `#254 `_ - Switched to using Fernet generated passphrases for exported items. +* Closed `#275 `_ - Switched to using Fernet generated passphrases for exported items. These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64. * Closed `#278 `_ - Added ability to specify a custom name to certificate creation, previously this was only available in the certificate import wizard. diff --git a/docs/administration.rst b/docs/administration.rst index 3623f311..bad95026 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -78,13 +78,13 @@ Basic Configuration The default connection pool size is 5 for sqlalchemy managed connections. Depending on the number of Lemur instances, please specify per instance connection pool size. Below is an example to set connection pool size to 10. - :: + :: SQLALCHEMY_POOL_SIZE = 10 .. warning:: -This is an optional setting but important to review and set for optimal database connection usage and for overall database performance. + This is an optional setting but important to review and set for optimal database connection usage and for overall database performance. .. data:: SQLALCHEMY_MAX_OVERFLOW :noindex: @@ -99,7 +99,7 @@ This is an optional setting but important to review and set for optimal database .. note:: -Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size. + Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size. .. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION @@ -174,6 +174,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c .. data:: PUBLIC_CA_MAX_VALIDITY_DAYS :noindex: + Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities. The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity of 397 days and sets it to 365 days. @@ -185,6 +186,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c .. data:: DEFAULT_VALIDITY_DAYS :noindex: + Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity @@ -207,6 +209,11 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed" and deleted certificates will always be visible in the UI. (default: `False`) +.. data:: LEMUR_AWS_REGION + :noindex: + + This is an optional config applicable for settings where Lemur is deployed in AWS. For accessing regionalized + STS endpoints, LEMUR_AWS_REGION defines the region where Lemur is deployed. Certificate Default Options --------------------------- @@ -864,6 +871,17 @@ account. See :ref:`Using a pre-existing ACME account ` for mor This is the registration for the ACME account, the most important part is the uri attribute (in JSON) +.. data:: ACME_PREFERRED_ISSUER + :noindex: + + This is an optional parameter to indicate the preferred chain to retrieve from ACME when finalizing the order. + This is applicable to Let's Encrypts recent `migration `_ to their + own root, where they provide two distinct certificate chains (fullchain_pem vs. alternative_fullchains_pem); + the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain + is rooted in X1 root CA. + Select "X1" to get the shorter chain (currently alternative), leave blank or "DST Root CA X3" for the longer chain. + + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -904,10 +922,12 @@ Active Directory Certificate Services Plugin .. data:: ADCS_START :noindex: + Used in ADCS-Sourceplugin. Minimum id of the first certificate to be returned. ID is increased by one until ADCS_STOP. Missing cert-IDs are ignored .. data:: ADCS_STOP :noindex: + Used for ADCS-Sourceplugin. Maximum id of the certificates returned. @@ -1640,7 +1660,7 @@ Slack AWS (Source) ----- +------------ :Authors: Kevin Glisson , @@ -1653,7 +1673,7 @@ AWS (Source) AWS (Destination) ----- +----------------- :Authors: Kevin Glisson , @@ -1666,7 +1686,7 @@ AWS (Destination) AWS (SNS Notification) ------ +---------------------- :Authors: Jasmine Schladen diff --git a/docs/conf.py b/docs/conf.py index 55bd20d2..077b66f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,6 +32,9 @@ if on_rtd: MOCK_MODULES = ["ldap"] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) +autodoc_mock_imports = ["python-ldap", "acme", "certsrv", "dnspython3", "dyn", "factory-boy", "flask_replicated", + "josepy", "logmatic", "pem"] + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -146,7 +149,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +# html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 8569dda5..1a08ffb5 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -43,6 +43,13 @@ Building Documentation Inside the ``docs`` directory, you can run ``make`` to build the documentation. See ``make help`` for available options and the `Sphinx Documentation `_ for more information. +Adding New Modules to Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a new module is added, it will need to be added to the documentation. +Ideally, we might rely on `sphinx-apidoc `_ to autogenerate our documentation. +Unfortunately, this causes some build problems. +Instead, you'll need to add new modules by hand. Developing Against HEAD ----------------------- diff --git a/docs/developer/internals/lemur.defaults.rst b/docs/developer/internals/lemur.defaults.rst new file mode 100644 index 00000000..0b1767ed --- /dev/null +++ b/docs/developer/internals/lemur.defaults.rst @@ -0,0 +1,29 @@ +defaults Package +================ + +:mod:`defaults` Module +---------------------------------------- + +.. automodule:: lemur.defaults + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +----------------------------- + +.. automodule:: lemur.defaults.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +--------------------------- + +.. automodule:: lemur.defaults.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.deployment.rst b/docs/developer/internals/lemur.deployment.rst new file mode 100644 index 00000000..047ec251 --- /dev/null +++ b/docs/developer/internals/lemur.deployment.rst @@ -0,0 +1,20 @@ +deployment Package +=================== + +:mod:`deployment` Module +---------------------------------------- + +.. automodule:: lemur.deployment + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.deployment.service + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.endpoints.rst b/docs/developer/internals/lemur.endpoints.rst new file mode 100644 index 00000000..14cb67d6 --- /dev/null +++ b/docs/developer/internals/lemur.endpoints.rst @@ -0,0 +1,56 @@ +endpoints Package +=================== + +:mod:`endpoints` Module +---------------------------------------- + +.. automodule:: lemur.endpoints + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cli` Module +-------------------------- + +.. automodule:: lemur.endpoints.cli + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +----------------------------- + +.. automodule:: lemur.endpoints.models + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +------------------------------ + +.. automodule:: lemur.endpoints.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.endpoints.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +---------------------------- + +.. automodule:: lemur.endpoints.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.logs.rst b/docs/developer/internals/lemur.logs.rst new file mode 100644 index 00000000..4e01484a --- /dev/null +++ b/docs/developer/internals/lemur.logs.rst @@ -0,0 +1,47 @@ +logs Package +=================== + +:mod:`logs` Module +-------------------- + +.. automodule:: lemur.logs + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +------------------------------ + +.. automodule:: lemur.logs.models + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +------------------------------ + +.. automodule:: lemur.logs.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.logs.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +------------------------------ + +.. automodule:: lemur.logs.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_acme.rst b/docs/developer/internals/lemur.plugins.lemur_acme.rst new file mode 100644 index 00000000..57f5209d --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_acme.rst @@ -0,0 +1,83 @@ +lemur_acme package +================================= + +:mod:`lemur_acme` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`acme_handlers` Module +----------------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.acme_handlers + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`challenge_types` Module +------------------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.challenge_types + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cloudflare` Module +------------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.cloudflare + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`dyn` Module +------------------------------------ + +.. automodule:: lemur.plugins.lemur_acme.dyn + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +--------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`powerdns` Module +----------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.powerdns + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`route53` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.route53 + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`ultradns` Module +----------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.ultradns + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_atlas.rst b/docs/developer/internals/lemur.plugins.lemur_atlas.rst new file mode 100644 index 00000000..01eaaa79 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_atlas.rst @@ -0,0 +1,20 @@ +lemur_atlas package +================================== + +:mod:`lemur_atlas` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_atlas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_atlas.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_cryptography.rst b/docs/developer/internals/lemur.plugins.lemur_cryptography.rst new file mode 100644 index 00000000..ee9f6c93 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_cryptography.rst @@ -0,0 +1,20 @@ +lemur_cryptography package +================================== + +:mod:`lemur_cryptography` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_cryptography + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_cryptography.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_digicert.rst b/docs/developer/internals/lemur.plugins.lemur_digicert.rst new file mode 100644 index 00000000..232d658b --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_digicert.rst @@ -0,0 +1,20 @@ +lemur_digicert package +================================== + +:mod:`lemur_digicert` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_digicert + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_digicert.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_jks.rst b/docs/developer/internals/lemur.plugins.lemur_jks.rst new file mode 100644 index 00000000..a47f653f --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_jks.rst @@ -0,0 +1,20 @@ +lemur_jks package +================================== + +:mod:`lemur_jks` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_jks + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_jks.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_kubernetes.rst b/docs/developer/internals/lemur.plugins.lemur_kubernetes.rst new file mode 100644 index 00000000..7173befb --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_kubernetes.rst @@ -0,0 +1,20 @@ +lemur_kubernetes package +================================== + +:mod:`lemur_kubernetes` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_kubernetes + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_kubernetes.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_openssl.rst b/docs/developer/internals/lemur.plugins.lemur_openssl.rst new file mode 100644 index 00000000..b94b56b4 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_openssl.rst @@ -0,0 +1,20 @@ +lemur_openssl package +================================== + +:mod:`lemur_openssl` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_openssl + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_openssl.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_slack.rst b/docs/developer/internals/lemur.plugins.lemur_slack.rst new file mode 100644 index 00000000..371a8880 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_slack.rst @@ -0,0 +1,20 @@ +lemur_slack package +================================== + +:mod:`lemur_slack` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_slack + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_slack.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.reporting.rst b/docs/developer/internals/lemur.reporting.rst new file mode 100644 index 00000000..9056e8b6 --- /dev/null +++ b/docs/developer/internals/lemur.reporting.rst @@ -0,0 +1,38 @@ +reporting Package +=================== + +:mod:`reporting` Module +---------------------------------------- + +.. automodule:: lemur.reporting + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cli` Module +------------------------------ + +.. automodule:: lemur.reporting.cli + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.reporting.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +------------------------------ + +.. automodule:: lemur.reporting.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.rst b/docs/developer/internals/lemur.rst index b6517e4b..86b4d000 100644 --- a/docs/developer/internals/lemur.rst +++ b/docs/developer/internals/lemur.rst @@ -28,15 +28,6 @@ lemur Package :undoc-members: :show-inheritance: -:mod:`decorators` Module ------------------------- - -.. automodule:: lemur.decorators - :noindex: - :members: - :undoc-members: - :show-inheritance: - :mod:`exceptions` Module ------------------------ @@ -108,7 +99,7 @@ Subpackages lemur.plugins.lemur_atlas lemur.plugins.lemur_cryptography lemur.plugins.lemur_digicert - lemur.plugins.lemur_java + lemur.plugins.lemur_jks lemur.plugins.lemur_kubernetes lemur.plugins.lemur_openssl lemur.plugins.lemur_slack diff --git a/docs/developer/internals/lemur.sources.rst b/docs/developer/internals/lemur.sources.rst new file mode 100644 index 00000000..6a5c0c42 --- /dev/null +++ b/docs/developer/internals/lemur.sources.rst @@ -0,0 +1,56 @@ +sources Package +=================== + +:mod:`sources` Module +---------------------- + +.. automodule:: lemur.sources + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cli` Module +------------------------------ + +.. automodule:: lemur.sources.cli + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +------------------------------ + +.. automodule:: lemur.sources.models + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +------------------------------ + +.. automodule:: lemur.sources.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.sources.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +------------------------------ + +.. automodule:: lemur.sources.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.tests.rst b/docs/developer/internals/lemur.tests.rst new file mode 100644 index 00000000..0c385694 --- /dev/null +++ b/docs/developer/internals/lemur.tests.rst @@ -0,0 +1,11 @@ +tests Package +============= + +:mod:`tests` Module +-------------------- + +.. automodule:: lemur.tests + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index 3834b0b9..517b5a0d 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -145,8 +145,7 @@ The `IssuerPlugin` doesn't have any options like Destination, Source, and Notifi any fields you might need to submit a request to a third party. If there are additional options you need in your plugin feel free to open an issue, or look into adding additional options to issuers yourself. -Asynchronous Certificates -^^^^^^^^^^^^^^^^^^^^^^^^^ +**Asynchronous Certificates** An issuer may take some time to actually issue a certificate for an order. In this case, a `PendingCertificate` is returned, which holds information to recreate a `Certificate` object at a later time. Then, `get_ordered_certificate()` should be run periodically via `python manage.py pending_certs fetch -i all` to attempt to retrieve an ordered certificate:: def get_ordered_ceriticate(self, order_id): @@ -154,9 +153,10 @@ An issuer may take some time to actually issue a certificate for an order. In t # retrieve an order, and check if there is an issued certificate attached to it `cancel_ordered_certificate()` should be implemented to allow an ordered certificate to be canceled before it is issued:: - def cancel_ordered_certificate(self, pending_cert, **kwargs): - # pending_cert should contain the necessary information to match an order - # kwargs can be given to provide information to the issuer for canceling + + def cancel_ordered_certificate(self, pending_cert, **kwargs): + # pending_cert should contain the necessary information to match an order + # kwargs can be given to provide information to the issuer for canceling Destination ----------- @@ -286,7 +286,7 @@ The `ExportPlugin` object requires the implementation of one function:: Custom TLS Provider ------- +------------------- Managing TLS at the enterprise scale could be hard and often organizations offer custom wrapper implementations. It could be ideal to use those while making calls to internal services. The `TLSPlugin` would help to achieve this. It requires the diff --git a/docs/doing-a-release.rst b/docs/doing-a-release.rst index 747668fb..8e24e7c1 100644 --- a/docs/doing-a-release.rst +++ b/docs/doing-a-release.rst @@ -1,10 +1,18 @@ Doing a release =============== -Doing a release of ``lemur`` requires a few steps. +Doing a release of ``lemur`` is now mostly automated and consists of the following steps: -Bumping the version number --------------------------- +* Raise a PR to add the release date and summary in the :doc:`/changelog`. +* Merge above PR and create a new `Github release `_: set the tag starting with v, e.g., v0.9.0 + +The `publish workflow `_ uses the git +tag to set the release version. + +The following describes the manual release steps, which is now obsolete: + +Manually Bumping the version number +----------------------------------- The next step in doing a release is bumping the version number in the software. @@ -14,8 +22,8 @@ software. * Do a commit indicating this, and raise a pull request with this. * Wait for it to be merged. -Performing the release ----------------------- +Manually Performing the release +------------------------------- The commit that merged the version number bump is now the official release commit for this release. You need an `API key `_, diff --git a/docs/guide/index.rst b/docs/guide/index.rst index b06a95e0..f3efcb14 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -65,6 +65,7 @@ Import an Existing Certificate You can add notification options and upload the created certificate to a destination, both of these are editable features and can be changed after the certificate has been created. +.. _CreateANewUser: Create a New User ~~~~~~~~~~~~~~~~~ diff --git a/docs/license/index.rst b/docs/license/index.rst index 4df00576..3afc9bc8 100644 --- a/docs/license/index.rst +++ b/docs/license/index.rst @@ -17,4 +17,5 @@ A list of additional contributors can be seen on `GitHub `Destination` and click `Create`. The path you provide for the destination needs to -be the exact path that is called when the ACME providers calls ``http:///.well-known/acme-challenge/`. The +be the exact path that is called when the ACME providers calls `http:///.well-known/acme-challenge/`. The token part will be added dynamically by the acme_upload. Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge. diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 3056029d..f972c2ef 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -148,7 +148,7 @@ Before Lemur will run you need to fill in a few required variables in the config LEMUR_DEFAULT_ORGANIZATIONAL_UNIT Set Up Postgres --------------- +--------------- For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on. @@ -186,11 +186,12 @@ In addition to creating a new user, Lemur also creates a few default email notif Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user. .. code-block:: bash + sudo -u postgres -i psql postgres=# ALTER USER lemur WITH SUPERUSER -Additional notifications can be created through the UI or API. See :ref:`Creating Notifications ` and :ref:`Command Line Interface ` for details. +Additional notifications can be created through the UI or API. See :ref:`Notification Options ` and :ref:`Command Line Interface ` for details. **Make note of the password used as this will be used during first login to the Lemur UI.** @@ -202,15 +203,16 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now. .. code-block:: bash + sudo -u postgres -i psql postgres=# ALTER USER lemur WITH NOSUPERUSER -.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. +.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating a New User ` and :ref:`Command Line Interface ` for details. Set Up a Reverse Proxy ---------------------- +---------------------- By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. diff --git a/docs/security.rst b/docs/security.rst index e2712e1f..e4a7ccf6 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -22,7 +22,7 @@ Supported Versions ------------------ At any given time, we will provide security support for the `master`_ branch -as well as the 2 most recent releases. +as well as the most recent release. Disclosure Process ------------------ @@ -30,20 +30,15 @@ Disclosure Process Our process for taking a security issue from private discussion to public disclosure involves multiple steps. -Approximately one week before full public disclosure, we will send advance -notification of the issue to a list of people and organizations, primarily -composed of operating-system vendors and other distributors of -``lemur``. This notification will consist of an email message -containing: +Approximately one week before full public disclosure, we will provide advanced notification that a security issue exists. Depending on the severity of the issue, we may choose to either send a targeted email to known Lemur users and contributors or post an issue to the Lemur repository. In either case, the notification should contain the following. -* A full description of the issue and the affected versions of - ``lemur``. +* A description of the potential impact +* The affected versions of ``lemur``. * The steps we will be taking to remedy the issue. -* The patches, if any, that will be applied to ``lemur``. * The date on which the ``lemur`` team will apply these patches, issue new releases, and publicly disclose the issue. -Simultaneously, the reporter of the issue will receive notification of the date +If the issue was disclosed to us, the reporter will receive notification of the date on which we plan to make the issue public. On the day of disclosure, we will take the following steps: @@ -52,7 +47,7 @@ On the day of disclosure, we will take the following steps: messages for these patches will indicate that they are for security issues, but will not describe the issue in any detail; instead, they will warn of upcoming disclosure. -* Issue the relevant releases. +* Issue an updated release. If a reported issue is believed to be particularly time-sensitive – due to a known exploit in the wild, for example – the time between advance notification diff --git a/gulp/build.js b/gulp/build.js index 405cacc8..13915051 100644 --- a/gulp/build.js +++ b/gulp/build.js @@ -40,7 +40,7 @@ function replaceAll(string, find, replace) { function stringSrc(filename, string) { let src = require('stream').Readable({objectMode: true}); src._read = function () { - this.push(new gutil.File({cwd: '', base: '', path: filename, contents: new Buffer(string)})); + this.push(new gutil.File({cwd: '', base: '', path: filename, contents: Buffer.from(string)})); this.push(null); }; return src; diff --git a/lemur/__about__.py b/lemur/__about__.py index 0926ef33..2a6db3c1 100644 --- a/lemur/__about__.py +++ b/lemur/__about__.py @@ -15,7 +15,7 @@ __title__ = "lemur" __summary__ = "Certificate management and orchestration service" __uri__ = "https://github.com/Netflix/lemur" -__version__ = "0.8.0" +__version__ = "develop" __author__ = "The Lemur developers" __email__ = "security@netflix.com" diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 1705e0c9..6ce9a5b6 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -75,7 +75,7 @@ def create_token(user, aid=None, ttl=None): if ttl == -1: del payload["exp"] else: - payload["exp"] = ttl + payload["exp"] = datetime.utcnow() + timedelta(days=ttl) token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"]) return token @@ -116,9 +116,8 @@ def login_required(f): return dict(message="Token has been revoked"), 403 if access_key.ttl != -1: current_time = datetime.utcnow() - expired_time = datetime.fromtimestamp( - access_key.issued_at + access_key.ttl - ) + # API key uses days + expired_time = datetime.fromtimestamp(access_key.issued_at) + timedelta(days=access_key.ttl) if current_time >= expired_time: return dict(message="Token has expired"), 403 diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index 094a5a74..16441719 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -132,31 +132,31 @@ class AuthoritiesList(AuthenticatedResource): Accept: application/json, text/javascript Content-Type: application/json;charset=UTF-8 - { - "country": "US", - "state": "California", - "location": "Los Gatos", - "organization": "Netflix", - "organizationalUnit": "Operations", - "type": "root", - "signingAlgorithm": "sha256WithRSA", - "sensitivity": "medium", - "keyType": "RSA2048", - "plugin": { - "slug": "cloudca-issuer" - }, - "name": "TimeTestAuthority5", - "owner": "secure@example.com", - "description": "test", - "commonName": "AcommonName", - "validityYears": "20", - "extensions": { - "subAltNames": { - "names": [] - }, - "custom": [] - } - } + { + "country": "US", + "state": "California", + "location": "Los Gatos", + "organization": "Netflix", + "organizationalUnit": "Operations", + "type": "root", + "signingAlgorithm": "sha256WithRSA", + "sensitivity": "medium", + "keyType": "RSA2048", + "plugin": { + "slug": "cloudca-issuer" + }, + "name": "TimeTestAuthority5", + "owner": "secure@example.com", + "description": "test", + "commonName": "AcommonName", + "validityYears": "20", + "extensions": { + "subAltNames": { + "names": [] + }, + "custom": [] + } + } **Example response**: @@ -218,8 +218,7 @@ class AuthoritiesList(AuthenticatedResource): :arg parent: the parent authority if this is to be a subca :arg signingAlgorithm: algorithm used to sign the authority :arg keyType: key type - :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored - in an HSM + :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored in an HSM :arg keyName: name of the key to store in the HSM (CloudCA) :arg serialNumber: serial number of the authority :arg firstSerial: specifies the starting serial number for certificates issued off of this authority @@ -494,23 +493,48 @@ class CertificateAuthority(AuthenticatedResource): class AuthorityVisualizations(AuthenticatedResource): def get(self, authority_id): """ - {"name": "flare", - "children": [ - { - "name": "analytics", - "children": [ - { - "name": "cluster", - "children": [ - {"name": "AgglomerativeCluster", "size": 3938}, - {"name": "CommunityStructure", "size": 3812}, - {"name": "HierarchicalCluster", "size": 6714}, - {"name": "MergeEdge", "size": 743} - ] - } - ] - } - ]} + .. http:get:: /authorities/1/visualize + + Authority visualization + + **Example request**: + + .. sourcecode:: http + + GET /certificates/1/visualize HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + {"name": "flare", + "children": [ + { + "name": "analytics", + "children": [ + { + "name": "cluster", + "children": [ + {"name": "AgglomerativeCluster", "size": 3938}, + {"name": "CommunityStructure", "size": 3812}, + {"name": "HierarchicalCluster", "size": 6714}, + {"name": "MergeEdge", "size": 743} + ] + } + ] + } + ] + } + + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + :statuscode 403: unauthenticated """ authority = service.get(authority_id) return dict( diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 8f21a751..986a8e31 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -153,6 +153,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate(): return ( Certificate.query.filter(Certificate.endpoints.any()) .filter(Certificate.rotation == false()) + .filter(Certificate.revoked == false()) .filter(Certificate.not_after >= arrow.now()) .filter(not_(Certificate.replaced.any())) .all() # noqa diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 3de08003..941eba0f 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -59,8 +59,8 @@ class CertificatesListValid(AuthenticatedResource): **Example request**: .. sourcecode:: http - GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com&page=1&count=20 - HTTP/1.1 + + GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com&page=1&count=20 HTTP/1.1 Host: example.com Accept: application/json, text/javascript diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 7bae57f0..5e302c6d 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -21,7 +21,7 @@ def create(label, plugin_name, options, description=None): :param label: Destination common name :param description: - :rtype : Destination + :rtype: Destination :return: New destination """ # remove any sub-plugin objects before try to save the json options @@ -50,7 +50,7 @@ def update(destination_id, label, plugin_name, options, description): :param plugin_name: :param options: :param description: - :rtype : Destination + :rtype: Destination :return: """ destination = get(destination_id) @@ -81,7 +81,7 @@ def get(destination_id): Retrieves an destination by its lemur assigned ID. :param destination_id: Lemur assigned ID - :rtype : Destination + :rtype: Destination :return: """ return database.get(Destination, destination_id) diff --git a/lemur/dns_providers/service.py b/lemur/dns_providers/service.py index 7052b55b..942161b7 100644 --- a/lemur/dns_providers/service.py +++ b/lemur/dns_providers/service.py @@ -36,7 +36,7 @@ def get_friendly(dns_provider_id): Retrieves a dns provider by its lemur assigned ID. :param dns_provider_id: Lemur assigned ID - :rtype : DnsProvider + :rtype: DnsProvider :return: """ dns_provider = get(dns_provider_id) diff --git a/lemur/dns_providers/views.py b/lemur/dns_providers/views.py index d470aa2f..0784cc02 100644 --- a/lemur/dns_providers/views.py +++ b/lemur/dns_providers/views.py @@ -86,62 +86,79 @@ class DnsProvidersList(AuthenticatedResource): @admin_permission.require(http_exception=403) def post(self, data=None): """ - Creates a DNS Provider + .. http:post:: /dns_providers - **Example request**: - { - "providerType": { - "name": "route53", - "requirements": [ - { - "name": "account_id", - "type": "int", - "required": true, - "helpMessage": "AWS Account number", - "value": 12345 - } - ], - "route": "dns_provider_options", - "reqParams": null, - "restangularized": true, - "fromServer": true, - "parentResource": null, - "restangularCollection": false - }, - "name": "provider_name", - "description": "provider_description" - } + Creates a DNS Provider + + **Example request**: + + .. sourcecode:: http + + POST /dns_providers HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + { + "providerType": { + "name": "route53", + "requirements": [ + { + "name": "account_id", + "type": "int", + "required": true, + "helpMessage": "AWS Account number", + "value": 12345 + } + ], + "route": "dns_provider_options", + "reqParams": null, + "restangularized": true, + "fromServer": true, + "parentResource": null, + "restangularCollection": false + }, + "name": "provider_name", + "description": "provider_description" + } + + **Example request 2**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + { + "providerType": { + "name": "cloudflare", + "requirements": [ + { + "name": "email", + "type": "str", + "required": true, + "helpMessage": "Cloudflare Email", + "value": "test@example.com" + }, + { + "name": "key", + "type": "str", + "required": true, + "helpMessage": "Cloudflare Key", + "value": "secretkey" + } + ], + "route": "dns_provider_options", + "reqParams": null, + "restangularized": true, + "fromServer": true, + "parentResource": null, + "restangularCollection": false + }, + "name": "provider_name", + "description": "provider_description" + } - **Example request 2** - { - "providerType": { - "name": "cloudflare", - "requirements": [ - { - "name": "email", - "type": "str", - "required": true, - "helpMessage": "Cloudflare Email", - "value": "test@example.com" - }, - { - "name": "key", - "type": "str", - "required": true, - "helpMessage": "Cloudflare Key", - "value": "secretkey" - } - ], - "route": "dns_provider_options", - "reqParams": null, - "restangularized": true, - "fromServer": true, - "parentResource": null, - "restangularCollection": false - }, - "name": "provider_name", - "description": "provider_description" - } :return: """ return service.create(data) diff --git a/lemur/domains/views.py b/lemur/domains/views.py index a3e0cdff..bb7eac1f 100644 --- a/lemur/domains/views.py +++ b/lemur/domains/views.py @@ -96,7 +96,7 @@ class DomainsList(AuthenticatedResource): .. sourcecode:: http - GET /domains HTTP/1.1 + POST /domains HTTP/1.1 Host: example.com Accept: application/json, text/javascript diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 1d7bda4c..2e2f37ad 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -200,6 +200,8 @@ def send_plugin_notification(event_type, data, recipients, notification): "notification_type": event_type, "notification_plugin": notification.plugin.slug, "certificate_targets": recipients, + "plugin": notification.plugin.slug, + "notification_id": notification.id, } status = FAILURE_METRIC_STATUS try: diff --git a/lemur/notifications/schemas.py b/lemur/notifications/schemas.py index a3ff4c99..6ef5c506 100644 --- a/lemur/notifications/schemas.py +++ b/lemur/notifications/schemas.py @@ -21,6 +21,8 @@ class NotificationInputSchema(LemurInputSchema): active = fields.Boolean() plugin = fields.Nested(PluginInputSchema, required=True) certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) + added_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) + removed_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) class NotificationOutputSchema(LemurOutputSchema): diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 5bc5f3e1..372c1843 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -94,7 +94,7 @@ def create(label, plugin_name, options, description, certificates): :param options: :param description: :param certificates: - :rtype : Notification + :rtype: Notification :return: """ notification = Notification( @@ -104,7 +104,7 @@ def create(label, plugin_name, options, description, certificates): return database.create(notification) -def update(notification_id, label, plugin_name, options, description, active, certificates): +def update(notification_id, label, plugin_name, options, description, active, added_certificates, removed_certificates): """ Updates an existing notification. @@ -114,8 +114,9 @@ def update(notification_id, label, plugin_name, options, description, active, ce :param options: :param description: :param active: - :param certificates: - :rtype : Notification + :param added_certificates: + :param removed_certificates: + :rtype: Notification :return: """ notification = get(notification_id) @@ -125,7 +126,8 @@ def update(notification_id, label, plugin_name, options, description, active, ce notification.options = options notification.description = description notification.active = active - notification.certificates = certificates + notification.certificates = notification.certificates + added_certificates + notification.certificates = [c for c in notification.certificates if c not in removed_certificates] return database.update(notification) @@ -144,7 +146,7 @@ def get(notification_id): Retrieves an notification by its lemur assigned ID. :param notification_id: Lemur assigned ID - :rtype : Notification + :rtype: Notification :return: """ return database.get(Notification, notification_id) diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index 19111d33..8ac8e06f 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -117,7 +117,7 @@ class NotificationsList(AuthenticatedResource): """ .. http:post:: /notifications - Creates a new account + Creates a new notification **Example request**: @@ -214,9 +214,12 @@ class NotificationsList(AuthenticatedResource): "id": 2 } - :arg accountNumber: aws account number - :arg label: human readable account label - :arg comments: some description about the account + :label label: notification name + :label slug: notification plugin slug + :label plugin_options: notification plugin options + :label description: notification description + :label active: whether or not the notification is active/enabled + :label certificates: certificates to attach to notification :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error """ @@ -239,7 +242,7 @@ class Notifications(AuthenticatedResource): """ .. http:get:: /notifications/1 - Get a specific account + Get a specific notification **Example request**: @@ -306,17 +309,29 @@ class Notifications(AuthenticatedResource): """ .. http:put:: /notifications/1 - Updates an account + Updates a notification **Example request**: .. sourcecode:: http - POST /notifications/1 HTTP/1.1 + PUT /notifications/1 HTTP/1.1 Host: example.com Accept: application/json, text/javascript Content-Type: application/json;charset=UTF-8 + { + "label": "labelChanged", + "plugin": { + "slug": "email-notification", + "plugin_options": "???" + }, + "description": "Sample notification", + "active": "true", + "added_certificates": "???", + "removed_certificates": "???" + } + **Example response**: @@ -328,14 +343,24 @@ class Notifications(AuthenticatedResource): { "id": 1, - "accountNumber": 11111111111, "label": "labelChanged", - "comments": "this is a thing" + "plugin": { + "slug": "email-notification", + "plugin_options": "???" + }, + "description": "Sample notification", + "active": "true", + "added_certificates": "???", + "removed_certificates": "???" } - :arg accountNumber: aws account number - :arg label: human readable account label - :arg comments: some description about the account + :label label: notification name + :label slug: notification plugin slug + :label plugin_options: notification plugin options + :label description: notification description + :label active: whether or not the notification is active/enabled + :label added_certificates: certificates to add + :label removed_certificates: certificates to remove :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error """ @@ -346,7 +371,8 @@ class Notifications(AuthenticatedResource): data["plugin"]["plugin_options"], data["description"], data["active"], - data["certificates"], + data["added_certificates"], + data["removed_certificates"], ) def delete(self, notification_id): diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 8b4d033c..d2cb4638 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -93,11 +93,10 @@ def get_pending_certs(pending_ids): def create_certificate(pending_certificate, certificate, user): """ Create and store a certificate with pending certificate's info - Args: - pending_certificate: PendingCertificate which will populate the certificate - certificate: dict from Authority, which contains the body, chain and external id - user: User that called this function, used as 'creator' of the certificate if it does - not have an owner + + :arg pending_certificate: PendingCertificate which will populate the certificate + :arg certificate: dict from Authority, which contains the body, chain and external id + :arg user: User that called this function, used as 'creator' of the certificate if it does not have an owner """ certificate["owner"] = pending_certificate.owner data, errors = CertificateUploadInputSchema().load(certificate) @@ -158,9 +157,9 @@ def cancel(pending_certificate, **kwargs): """ Cancel a pending certificate. A check should be done prior to this function to decide to revoke the certificate or just abort cancelling. - Args: - pending_certificate: PendingCertificate to be cancelled - Returns: the pending certificate if successful, raises Exception if there was an issue + + :arg pending_certificate: PendingCertificate to be cancelled + :return: the pending certificate if successful, raises Exception if there was an issue """ plugin = plugins.get(pending_certificate.authority.plugin_name) plugin.cancel_ordered_certificate(pending_certificate, **kwargs) diff --git a/lemur/pending_certificates/views.py b/lemur/pending_certificates/views.py index e70c8c3b..7c639477 100644 --- a/lemur/pending_certificates/views.py +++ b/lemur/pending_certificates/views.py @@ -221,7 +221,7 @@ class PendingCertificates(AuthenticatedResource): .. sourcecode:: http - PUT /pending certificates/1 HTTP/1.1 + PUT /pending_certificates/1 HTTP/1.1 Host: example.com Accept: application/json, text/javascript Content-Type: application/json;charset=UTF-8 @@ -338,7 +338,7 @@ class PendingCertificates(AuthenticatedResource): .. sourcecode:: http - DELETE /pending certificates/1 HTTP/1.1 + DELETE /pending_certificates/1 HTTP/1.1 Host: example.com Accept: application/json, text/javascript diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index e00c5090..3555a439 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -31,6 +31,11 @@ class ExportDestinationPlugin(DestinationPlugin): @property def options(self): + """ + Gets/sets options for the plugin. + + :return: + """ return self.default_options + self.additional_options def export(self, body, private_key, cert_chain, options): diff --git a/lemur/plugins/bases/notification.py b/lemur/plugins/bases/notification.py index 03de95ce..87b2b0fc 100644 --- a/lemur/plugins/bases/notification.py +++ b/lemur/plugins/bases/notification.py @@ -57,6 +57,11 @@ class ExpirationNotificationPlugin(NotificationPlugin): @property def options(self): + """ + Gets/sets options for the plugin. + + :return: + """ return self.default_options + self.additional_options def send(self, notification_type, message, excluded_targets, options, **kwargs): diff --git a/lemur/plugins/bases/source.py b/lemur/plugins/bases/source.py index 6f521e40..9f6bf304 100644 --- a/lemur/plugins/bases/source.py +++ b/lemur/plugins/bases/source.py @@ -33,4 +33,9 @@ class SourcePlugin(Plugin): @property def options(self): + """ + Gets/sets options for the plugin. + + :return: + """ return self.default_options + self.additional_options diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 78c160b2..8636d882 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -23,6 +23,7 @@ from acme import challenges, errors, messages from acme.client import BackwardsCompatibleClientV2, ClientNetwork from acme.errors import TimeoutError from acme.messages import Error as AcmeError +from certbot import crypto_util as acme_crypto_util from flask import current_app from lemur.common.utils import generate_private_key @@ -71,7 +72,7 @@ class AcmeHandler(object): return False def strip_wildcard(self, host): - """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" + """Removes the leading wildcard and returns Host and whether it was removed or not (True/False)""" prefix = "*." if host.startswith(prefix): return host[len(prefix):], True @@ -92,7 +93,8 @@ class AcmeHandler(object): deadline = datetime.datetime.now() + datetime.timedelta(seconds=360) try: - orderr = acme_client.poll_and_finalize(order, deadline) + orderr = acme_client.poll_authorizations(order, deadline) + orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True) except (AcmeError, TimeoutError): sentry.captureException(extra={"order_url": str(order.uri)}) @@ -112,14 +114,23 @@ class AcmeHandler(object): f"Successfully resolved Acme order: {order.uri}", exc_info=True ) - pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem) + pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem, + orderr.alternative_fullchains_pem) current_app.logger.debug( "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) ) return pem_certificate, pem_certificate_chain - def extract_cert_and_chain(self, fullchain_pem): + def extract_cert_and_chain(self, fullchain_pem, alternative_fullchains_pem, preferred_issuer=None): + + if not preferred_issuer: + preferred_issuer = current_app.config.get("ACME_PREFERRED_ISSUER", None) + if preferred_issuer: + # returns first chain if not match + fullchain_pem = acme_crypto_util.find_chain_with_issuer([fullchain_pem] + alternative_fullchains_pem, + preferred_issuer) + pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate( @@ -127,12 +138,7 @@ class AcmeHandler(object): ), ).decode() - if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ - and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): - pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") - else: - pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() + pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() return pem_certificate, pem_certificate_chain diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index 538ec236..49ae47a0 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -119,8 +119,10 @@ class AcmeHttpChallenge(AcmeChallenge): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") try: - finalized_orderr = acme_client.poll_and_finalize(orderr, - datetime.datetime.now() + datetime.timedelta(seconds=90)) + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = acme_client.poll_authorizations(orderr, deadline) + finalized_orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True) + except errors.ValidationError as validationError: for authz in validationError.failed_authzrs: for chall in authz.body.challenges: @@ -130,7 +132,8 @@ class AcmeHttpChallenge(AcmeChallenge): ERROR_CODES[chall.error.code])) raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.') - pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem) + pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem, + finalized_orderr.alternative_fullchains_pem) if len(deployed_challenges) != 0: for token_path in deployed_challenges: diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 324af5ac..74211b1b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -5,6 +5,12 @@ from flask import Flask from cryptography.x509 import DNSName from lemur.plugins.lemur_acme import acme_handlers +from lemur.tests.vectors import ( + ACME_CHAIN_SHORT_STR, + ACME_CHAIN_LONG_STR, + SAN_CERT_STR, +) + class TestAcmeHandler(unittest.TestCase): def setUp(self): @@ -110,3 +116,18 @@ class TestAcmeHandler(unittest.TestCase): self.assertEqual( result, [options["common_name"], "test2.netflix.net"] ) + + def test_extract_cert_and_chain(self): + # expecting the short chain + leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR, + [ACME_CHAIN_LONG_STR], + "(STAGING) Artificial Apricot R3") + self.assertEqual(leaf_pem, SAN_CERT_STR) + self.assertEqual(chain_pem, ACME_CHAIN_SHORT_STR[len(leaf_pem):].lstrip()) + + # expecting the long chain + leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR, + [ACME_CHAIN_LONG_STR], + "(STAGING) Doctored Durian Root CA X3") + self.assertEqual(leaf_pem, SAN_CERT_STR) + self.assertEqual(chain_pem, ACME_CHAIN_LONG_STR[len(leaf_pem):].lstrip()) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 0df9e6b2..d81e165d 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -146,7 +146,8 @@ Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4 idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng== -----END CERTIFICATE----- """ - mock_client.poll_and_finalize.return_value = mock_finalized_order + mock_finalized_order.alternative_fullchains_pem = [mock_finalized_order.fullchain_pem] + mock_client.finalize_order.return_value = mock_finalized_order mock_acme.return_value = (mock_client, "") diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index efcce4d0..bf7d89bb 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -450,7 +450,8 @@ class S3DestinationPlugin(ExportDestinationPlugin): def upload_acme_token(self, token_path, token, options, **kwargs): """ - This is called from the acme http challenge + This is called from the acme http challenge + :param self: :param token_path: :param token: @@ -563,4 +564,4 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin): f"{self.get_option('topicName', options)}" current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}") - sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) + sns.publish(topic_arn, message, notification_type, options, region_name=self.get_option("region", options)) diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index fab45b82..5fc4b8dd 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -11,21 +11,24 @@ import arrow import boto3 from flask import current_app +from lemur.plugins.bases import ExpirationNotificationPlugin -def publish(topic_arn, certificates, notification_type, **kwargs): + +def publish(topic_arn, certificates, notification_type, options, **kwargs): sns_client = boto3.client("sns", **kwargs) message_ids = {} subject = "Lemur: {0} Notification".format(notification_type.capitalize()) for certificate in certificates: - message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject) + message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, + subject, options) return message_ids -def publish_single(sns_client, topic_arn, certificate, notification_type, subject): +def publish_single(sns_client, topic_arn, certificate, notification_type, subject, options): response = sns_client.publish( TopicArn=topic_arn, - Message=format_message(certificate, notification_type), + Message=format_message(certificate, notification_type, options), Subject=subject, ) @@ -46,7 +49,7 @@ def create_certificate_url(name): ) -def format_message(certificate, notification_type): +def format_message(certificate, notification_type, options): json_message = { "notification_type": notification_type, "certificate_name": certificate["name"], @@ -57,4 +60,19 @@ def format_message(certificate, notification_type): "owner": certificate["owner"], "details": create_certificate_url(certificate["name"]) } + if notification_type == "expiration": + json_message["notification_interval_days"] = calculate_expiration_days(options) return json.dumps(json_message) + + +def calculate_expiration_days(options): + unit = ExpirationNotificationPlugin.get_option("unit", options) + interval = ExpirationNotificationPlugin.get_option("interval", options) + if unit == "weeks": + return interval * 7 + + elif unit == "months": + return interval * 30 + + elif unit == "days": + return interval diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index c1bd562c..722b5a2c 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -20,7 +20,13 @@ def sts_client(service, service_type="client"): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): - sts = boto3.client("sts", config=config) + if current_app.config.get("LEMUR_AWS_REGION"): + deployment_region = current_app.config.get("LEMUR_AWS_REGION") + sts = boto3.client('sts', region_name=deployment_region, + endpoint_url=f"https://sts.{deployment_region}.amazonaws.com/", + config=config) + else: + sts = boto3.client("sts", config=config) arn = "arn:aws:iam::{0}:role/{1}".format( kwargs.pop("account_number"), current_app.config.get("LEMUR_INSTANCE_PROFILE", "Lemur"), diff --git a/lemur/plugins/lemur_aws/tests/test_sns.py b/lemur/plugins/lemur_aws/tests/test_sns.py index c8688194..66ad3e96 100644 --- a/lemur/plugins/lemur_aws/tests/test_sns.py +++ b/lemur/plugins/lemur_aws/tests/test_sns.py @@ -13,9 +13,31 @@ from lemur.tests.test_messaging import verify_sender_email @mock_sns() -def test_format(certificate, endpoint): +def test_format_nonexpiration(certificate, endpoint): data = [certificate_notification_output_schema.dump(certificate).data] + for certificate in data: + expected_message = { + "notification_type": "not-expiration", + "certificate_name": certificate["name"], + "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), + "issuer": certificate["issuer"], + "id": certificate["id"], + "endpoints_detected": 0, + "owner": certificate["owner"], + "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) + } + # We don't currently support any SNS notifications besides expiration; + # when we do, this test will probably need to be refactored. + # For now, this is a placeholder proving empty options works as long as it's not "expiration" type + assert expected_message == json.loads(format_message(certificate, "not-expiration", None)) + + +@mock_sns() +def test_format_expiration(certificate, endpoint): + data = [certificate_notification_output_schema.dump(certificate).data] + options = get_options() + for certificate in data: expected_message = { "notification_type": "expiration", @@ -25,9 +47,10 @@ def test_format(certificate, endpoint): "id": certificate["id"], "endpoints_detected": 0, "owner": certificate["owner"], - "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) + "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]), + "notification_interval_days": 10 # 10 days specified in options } - assert expected_message == json.loads(format_message(certificate, "expiration")) + assert expected_message == json.loads(format_message(certificate, "expiration", options)) @mock_sns() @@ -52,7 +75,7 @@ def test_publish(certificate, endpoint): topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic() - message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1") + message_ids = publish(topic_arn, data, "expiration", get_options(), region_name="us-east-1") assert len(message_ids) == len(data) received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] @@ -61,7 +84,7 @@ def test_publish(certificate, endpoint): actual_message = next( (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None) actual_json = json.loads(actual_message["Body"]) - assert actual_json["Message"] == format_message(certificate, "expiration") + assert actual_json["Message"] == format_message(certificate, "expiration", get_options()) assert actual_json["Subject"] == "Lemur: Expiration Notification" @@ -98,7 +121,8 @@ def test_send_expiration_notification(): received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] assert len(received_messages) == 1 - expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration") + expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration", + notification.options) actual_message = json.loads(received_messages[0]["Body"])["Message"] assert actual_message == expected_message diff --git a/lemur/roles/views.py b/lemur/roles/views.py index 238bb1b7..24c99c20 100644 --- a/lemur/roles/views.py +++ b/lemur/roles/views.py @@ -114,7 +114,7 @@ class RolesList(AuthenticatedResource): "username": null, "password": null, "users": [ - {'id': 1} + {"id": 1} ] } @@ -177,7 +177,7 @@ class RoleViewCredentials(AuthenticatedResource): Content-Type: text/javascript { - "username: "ausername", + "username": "ausername", "password": "apassword" } diff --git a/lemur/sources/service.py b/lemur/sources/service.py index be0de049..c4c086f7 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -255,7 +255,7 @@ def create(label, plugin_name, options, description=None): :param plugin_name: :param options: :param description: - :rtype : Source + :rtype: Source :return: New source """ source = Source( @@ -273,7 +273,7 @@ def update(source_id, label, plugin_name, options, description): :param options: :param plugin_name: :param description: - :rtype : Source + :rtype: Source :return: """ source = get(source_id) @@ -300,7 +300,7 @@ def get(source_id): Retrieves an source by its lemur assigned ID. :param source_id: Lemur assigned ID - :rtype : Source + :rtype: Source :return: """ return database.get(Source, source_id) diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 9e8c9b33..9c484277 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -8,10 +8,35 @@ angular.module('lemur') if (this.certificates === undefined) { this.certificates = []; } + if (this.addedCertificates === undefined) { + this.addedCertificates = []; + } + if (_.some(this.addedCertificates, function (cert) { + return cert.id === certificate.id; + })) { + return; + } this.certificates.push(certificate); + this.addedCertificates.push(certificate); + if (this.removedCertificates !== undefined) { + const indexInRemovedList = _.findIndex(this.removedCertificates, function (cert) { + return cert.id === certificate.id; + }); + this.removedCertificates.splice(indexInRemovedList, 1); + } }, removeCertificate: function (index) { - this.certificates.splice(index, 1); + if (this.removedCertificates === undefined) { + this.removedCertificates = []; + } + const removedCert = this.certificates.splice(index, 1)[0]; + this.removedCertificates.push(removedCert); + if (this.addedCertificates !== undefined) { + const indexInAddedList = _.findIndex(this.addedCertificates, function (cert) { + return cert.id === removedCert.id; + }); + this.addedCertificates.splice(indexInAddedList, 1); + } } }); }); diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index 3dfb5621..51b61a3d 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -201,6 +201,7 @@ ACME_EMAIL = "jim@example.com" ACME_TEL = "4088675309" ACME_DIRECTORY_URL = "https://acme-v01.api.letsencrypt.org" ACME_DISABLE_AUTORESOLVE = True +ACME_PREFERRED_ISSUER = "R3" LDAP_AUTH = True LDAP_BIND_URI = "ldap://localhost" diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 962c40b4..06a04397 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -84,6 +84,25 @@ def test_get_by_serial(session, certificate): assert found +def test_get_all_certs_attached_to_endpoint_without_autorotate(session): + from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_autorotate, \ + cleanup_after_revoke + from lemur.tests.factories import EndpointFactory + + # add a certificate with endpoint + EndpointFactory() + + list_before = get_all_certs_attached_to_endpoint_without_autorotate() + len_list_before = len(list_before) + assert len_list_before > 0 + # revoked the first certificate + first_cert_with_endpoint = list_before[0] + cleanup_after_revoke(first_cert_with_endpoint) + + list_after = get_all_certs_attached_to_endpoint_without_autorotate() + assert len(list_after) + 1 == len_list_before + + def test_delete_cert(session): from lemur.certificates.service import delete, get from lemur.tests.factories import CertificateFactory diff --git a/lemur/tests/test_endpoints.py b/lemur/tests/test_endpoints.py index af073e53..895ab5b8 100644 --- a/lemur/tests/test_endpoints.py +++ b/lemur/tests/test_endpoints.py @@ -32,7 +32,7 @@ def test_rotate_certificate(client, source_plugin): ) def test_endpoint_get(client, token, status): assert ( - client.get(api.url_for(Endpoints, endpoint_id=1), headers=token).status_code + client.get(api.url_for(Endpoints, endpoint_id=2), headers=token).status_code == status ) diff --git a/lemur/tests/vectors.py b/lemur/tests/vectors.py index 7a78818c..c47017ae 100644 --- a/lemur/tests/vectors.py +++ b/lemur/tests/vectors.py @@ -587,3 +587,102 @@ zwKDoqAD+L4wEg8d890Zy2mbzJnDu2HQiMIROaBldKEAMQA= """ CERT_CHAIN_PKCS7_PEM = CERT_CHAIN_PKCS7_STR.encode('utf-8') + +ACME_CHAIN_LONG_STR = SAN_CERT_STR + """ +-----BEGIN CERTIFICATE----- +MIIFWzCCA0OgAwIBAgIQTfQrldHumzpMLrM7jRBd1jANBgkqhkiG9w0BAQsFADBm +MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy +aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ +ZWFyIFgxMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowWTELMAkGA1UE +BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSgwJgYDVQQD +Ex8oU1RBR0lORykgQXJ0aWZpY2lhbCBBcHJpY290IFIzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAu6TR8+74b46mOE1FUwBrvxzEYLck3iasmKrcQkb+ +gy/z9Jy7QNIAl0B9pVKp4YU76JwxF5DOZZhi7vK7SbCkK6FbHlyU5BiDYIxbbfvO +L/jVGqdsSjNaJQTg3C3XrJja/HA4WCFEMVoT2wDZm8ABC1N+IQe7Q6FEqc8NwmTS +nmmRQm4TQvr06DP+zgFK/MNubxWWDSbSKKTH5im5j2fZfg+j/tM1bGaczFWw8/lS +nukyn5J2L+NJYnclzkXoh9nMFnyPmVbfyDPOc4Y25aTzVoeBKXa/cZ5MM+WddjdL +biWvm19f1sYn1aRaAIrkppv7kkn83vcth8XCG39qC2ZvaQIDAQABo4IBEDCCAQww +DgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAS +BgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTecnpI3zHDplDfn4Uj31c3S10u +ZTAfBgNVHSMEGDAWgBS182Xy/rAKkh/7PH3zRKCsYyXDFDA2BggrBgEFBQcBAQQq +MCgwJgYIKwYBBQUHMAKGGmh0dHA6Ly9zdGcteDEuaS5sZW5jci5vcmcvMCsGA1Ud +HwQkMCIwIKAeoByGGmh0dHA6Ly9zdGcteDEuYy5sZW5jci5vcmcvMCIGA1UdIAQb +MBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCN +DLam9yN0EFxxn/3p+ruWO6n/9goCAM5PT6cC6fkjMs4uas6UGXJjr5j7PoTQf3C1 +vuxiIGRJC6qxV7yc6U0X+w0Mj85sHI5DnQVWN5+D1er7mp13JJA0xbAbHa3Rlczn +y2Q82XKui8WHuWra0gb2KLpfboYj1Ghgkhr3gau83pC/WQ8HfkwcvSwhIYqTqxoZ +Uq8HIf3M82qS9aKOZE0CEmSyR1zZqQxJUT7emOUapkUN9poJ9zGc+FgRZvdro0XB +yphWXDaqMYph0DxW/10ig5j4xmmNDjCRmqIKsKoWA52wBTKKXK1na2ty/lW5dhtA +xkz5rVZFd4sgS4J0O+zm6d5GRkWsNJ4knotGXl8vtS3X40KXeb3A5+/3p0qaD215 +Xq8oSNORfB2oI1kQuyEAJ5xvPTdfwRlyRG3lFYodrRg6poUBD/8fNTXMtzydpRgy +zUQZh/18F6B/iW6cbiRN9r2Hkh05Om+q0/6w0DdZe+8YrNpfhSObr/1eVZbKGMIY +qKmyZbBNu5ysENIK5MPc14mUeKmFjpN840VR5zunoU52lqpLDua/qIM8idk86xGW +xx2ml43DO/Ya/tVZVok0mO0TUjzJIfPqyvr455IsIut4RlCR9Iq0EDTve2/ZwCuG +hSjpTUFGSiQrR2JK2Evp+o6AETUkBCO1aw0PpQBPDQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFVDCCBDygAwIBAgIRAO1dW8lt+99NPs1qSY3Rs8cwDQYJKoZIhvcNAQELBQAw +cTELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1 +cml0eSBSZXNlYXJjaCBHcm91cDEtMCsGA1UEAxMkKFNUQUdJTkcpIERvY3RvcmVk +IER1cmlhbiBSb290IENBIFgzMB4XDTIxMDEyMDE5MTQwM1oXDTI0MDkzMDE4MTQw +M1owZjELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRl +bmQgUGVhciBYMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdD +Ta1QgGBWSYkyMhscZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPW +nL++fgehT0FbRHZgjOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigk +kmx8OiCO68a4QXg4wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZ +GTIf/oRt2/c+dYmDoaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6V +P19sTGy3yfqK5tPtTdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkL +YC0Ft2cYUyHtkstOfRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2 +UPQFxmWFRQnFjaq6rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/ +2dBZKmJqxHkxCuOQFjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRM +EeOXUYvbV4lqfCf8mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEm +QWUOTWIoDQ5FOia/GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eV +EGOIpn26bW5LKerumJxa/CFBaKi4bRvmdJRLAgMBAAGjgfEwge4wDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLXzZfL+sAqSH/s8ffNE +oKxjJcMUMB8GA1UdIwQYMBaAFAhX2onHolN5DE/d4JCPdLriJ3NEMDgGCCsGAQUF +BwEBBCwwKjAoBggrBgEFBQcwAoYcaHR0cDovL3N0Zy1kc3QzLmkubGVuY3Iub3Jn +LzAtBgNVHR8EJjAkMCKgIKAehhxodHRwOi8vc3RnLWRzdDMuYy5sZW5jci5vcmcv +MCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEB +CwUAA4IBAQB7tR8B0eIQSS6MhP5kuvGth+dN02DsIhr0yJtk2ehIcPIqSxRRmHGl +4u2c3QlvEpeRDp2w7eQdRTlI/WnNhY4JOofpMf2zwABgBWtAu0VooQcZZTpQruig +F/z6xYkBk3UHkjeqxzMN3d1EqGusxJoqgdTouZ5X5QTTIee9nQ3LEhWnRSXDx7Y0 +ttR1BGfcdqHopO4IBqAhbkKRjF5zj7OD8cG35omywUbZtOJnftiI0nFcRaxbXo0v +oDfLD0S6+AC2R3tKpqjkNX6/91hrRFglUakyMcZU/xleqbv6+Lr3YD8PsBTub6lI +oZ2lS38fL18Aon458fbc0BPHtenfhKj5 +-----END CERTIFICATE----- +""" + +ACME_CHAIN_SHORT_STR = SAN_CERT_STR + """ +-----BEGIN CERTIFICATE----- +MIIFWzCCA0OgAwIBAgIQTfQrldHumzpMLrM7jRBd1jANBgkqhkiG9w0BAQsFADBm +MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy +aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ +ZWFyIFgxMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowWTELMAkGA1UE +BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSgwJgYDVQQD +Ex8oU1RBR0lORykgQXJ0aWZpY2lhbCBBcHJpY290IFIzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAu6TR8+74b46mOE1FUwBrvxzEYLck3iasmKrcQkb+ +gy/z9Jy7QNIAl0B9pVKp4YU76JwxF5DOZZhi7vK7SbCkK6FbHlyU5BiDYIxbbfvO +L/jVGqdsSjNaJQTg3C3XrJja/HA4WCFEMVoT2wDZm8ABC1N+IQe7Q6FEqc8NwmTS +nmmRQm4TQvr06DP+zgFK/MNubxWWDSbSKKTH5im5j2fZfg+j/tM1bGaczFWw8/lS +nukyn5J2L+NJYnclzkXoh9nMFnyPmVbfyDPOc4Y25aTzVoeBKXa/cZ5MM+WddjdL +biWvm19f1sYn1aRaAIrkppv7kkn83vcth8XCG39qC2ZvaQIDAQABo4IBEDCCAQww +DgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAS +BgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTecnpI3zHDplDfn4Uj31c3S10u +ZTAfBgNVHSMEGDAWgBS182Xy/rAKkh/7PH3zRKCsYyXDFDA2BggrBgEFBQcBAQQq +MCgwJgYIKwYBBQUHMAKGGmh0dHA6Ly9zdGcteDEuaS5sZW5jci5vcmcvMCsGA1Ud +HwQkMCIwIKAeoByGGmh0dHA6Ly9zdGcteDEuYy5sZW5jci5vcmcvMCIGA1UdIAQb +MBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCN +DLam9yN0EFxxn/3p+ruWO6n/9goCAM5PT6cC6fkjMs4uas6UGXJjr5j7PoTQf3C1 +vuxiIGRJC6qxV7yc6U0X+w0Mj85sHI5DnQVWN5+D1er7mp13JJA0xbAbHa3Rlczn +y2Q82XKui8WHuWra0gb2KLpfboYj1Ghgkhr3gau83pC/WQ8HfkwcvSwhIYqTqxoZ +Uq8HIf3M82qS9aKOZE0CEmSyR1zZqQxJUT7emOUapkUN9poJ9zGc+FgRZvdro0XB +yphWXDaqMYph0DxW/10ig5j4xmmNDjCRmqIKsKoWA52wBTKKXK1na2ty/lW5dhtA +xkz5rVZFd4sgS4J0O+zm6d5GRkWsNJ4knotGXl8vtS3X40KXeb3A5+/3p0qaD215 +Xq8oSNORfB2oI1kQuyEAJ5xvPTdfwRlyRG3lFYodrRg6poUBD/8fNTXMtzydpRgy +zUQZh/18F6B/iW6cbiRN9r2Hkh05Om+q0/6w0DdZe+8YrNpfhSObr/1eVZbKGMIY +qKmyZbBNu5ysENIK5MPc14mUeKmFjpN840VR5zunoU52lqpLDua/qIM8idk86xGW +xx2ml43DO/Ya/tVZVok0mO0TUjzJIfPqyvr455IsIut4RlCR9Iq0EDTve2/ZwCuG +hSjpTUFGSiQrR2JK2Evp+o6AETUkBCO1aw0PpQBPDQ== +-----END CERTIFICATE----- +""" diff --git a/lemur/users/views.py b/lemur/users/views.py index 4bb07a0b..b0e2608e 100644 --- a/lemur/users/views.py +++ b/lemur/users/views.py @@ -101,7 +101,7 @@ class UsersList(AuthenticatedResource): Creates a new user - **Example request**: + **Example request with ID**: .. sourcecode:: http @@ -115,7 +115,25 @@ class UsersList(AuthenticatedResource): "email": "user3@example.com", "active": true, "roles": [ - {'id': 1} - or - {'name': 'myRole'} + {"id": 1} + ] + } + + **Example request with name**: + + .. sourcecode:: http + + POST /users HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + Content-Type: application/json;charset=UTF-8 + + { + "username": "user3", + "email": "user3@example.com", + "active": true, + "roles": [ + {"name": "myRole"} ] } @@ -130,7 +148,7 @@ class UsersList(AuthenticatedResource): { "id": 3, "active": True, - "email": "user3@example.com, + "email": "user3@example.com", "username": "user3", "profileImage": null } @@ -202,7 +220,7 @@ class Users(AuthenticatedResource): Update a user - **Example request**: + **Example request with ID**: .. sourcecode:: http @@ -216,7 +234,25 @@ class Users(AuthenticatedResource): "email": "user1@example.com", "active": false, "roles": [ - {'id': 1} - or - {'name': 'myRole'} + {"id": 1} + ] + } + + **Example request with name**: + + .. sourcecode:: http + + PUT /users/1 HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + Content-Type: application/json;charset=UTF-8 + + { + "username": "user1", + "email": "user1@example.com", + "active": false, + "roles": [ + {"name": "myRole"} ] } diff --git a/requirements-dev.txt b/requirements-dev.txt index 576ccd48..f6ea8caa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ chardet==3.0.4 # via requests colorama==0.4.3 # via twine -cryptography==3.4.5 +cryptography==3.4.6 # via secretstorage distlib==0.3.0 # via virtualenv @@ -50,7 +50,7 @@ packaging==20.9 # via bleach pkginfo==1.5.0.1 # via twine -pre-commit==2.10.1 +pre-commit==2.11.1 # via -r requirements-dev.in pycodestyle==2.6.0 # via flake8 diff --git a/requirements-docs.in b/requirements-docs.in index f025a85d..87663485 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -1,7 +1,51 @@ # Note: python-ldap from requirements breaks due to readthedocs.io not having the correct header files # The `make up-reqs` will update all requirement text files, and forcibly remove python-ldap # from requirements-docs.txt -# However, dependabot doesn't use `make up-reqs`, so `-r requirements.txt` has been removed completely. +# However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here +# Without including these dependencies, the docs are unable to include generated autodocs +acme +arrow +boto3 +botocore +certbot +certsrv +CloudFlare +cryptography +dnspython3 +dyn +Flask +Flask-Bcrypt +Flask-Cors +Flask-Mail +Flask-Migrate +Flask-Principal +Flask-RESTful +Flask-Script +Flask-SQLAlchemy +flask_replicated +gunicorn +hvac # required for the vault destination plugin +inflection +josepy +logmatic-python +marshmallow-sqlalchemy +marshmallow<2.20.5 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 +paramiko # required for the SFTP destination plugin +pem +pyjks >= 19 # pyjks < 19 depends on pycryptodome, which conflicts with dyn's usage of pycrypto +pyjwt +pyOpenSSL +raven[flask] +redis +retrying +SQLAlchemy-Utils +tabulate +vine +xmltodict +# Test requirements are needed to allow test docs to build +-r requirements-tests.txt + +# docs specific sphinx sphinxcontrib-httpdomain sphinx-rtd-theme diff --git a/requirements-docs.txt b/requirements-docs.txt index 2e76e73f..9906a3ea 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,43 +4,513 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # +acme==1.13.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # certbot alabaster==0.7.12 # via sphinx +alembic==1.5.5 + # via flask-migrate +aniso8601==9.0.0 + # via flask-restful +appdirs==1.4.3 + # via + # -r requirements-tests.txt + # black +arrow==1.0.3 + # via -r requirements-docs.in +attrs==19.3.0 + # via + # -r requirements-tests.txt + # jsonschema + # pytest +aws-sam-translator==1.22.0 + # via + # -r requirements-tests.txt + # cfn-lint +aws-xray-sdk==2.5.0 + # via + # -r requirements-tests.txt + # moto babel==2.8.0 # via sphinx +bandit==1.7.0 + # via -r requirements-tests.txt +bcrypt==3.2.0 + # via + # flask-bcrypt + # paramiko +beautifulsoup4==4.9.3 + # via cloudflare +black==20.8b1 + # via -r requirements-tests.txt +blinker==1.4 + # via + # flask-mail + # flask-principal + # raven +boto3==1.17.27 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # aws-sam-translator + # moto +boto==2.49.0 + # via + # -r requirements-tests.txt + # moto +botocore==1.20.27 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # aws-xray-sdk + # boto3 + # moto + # s3transfer +certbot==1.13.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt certifi==2020.12.5 - # via requests + # via + # -r requirements-tests.txt + # requests +certsrv==2.1.1 + # via -r requirements-docs.in +cffi==1.14.0 + # via + # -r requirements-tests.txt + # bcrypt + # cryptography + # pynacl +cfn-lint==0.29.5 + # via + # -r requirements-tests.txt + # moto chardet==3.0.4 - # via requests + # via + # -r requirements-tests.txt + # requests +click==7.1.2 + # via + # -r requirements-tests.txt + # black + # flask +cloudflare==2.8.15 + # via -r requirements-docs.in +configargparse==1.4 + # via + # -r requirements-tests.txt + # certbot +configobj==5.0.6 + # via + # -r requirements-tests.txt + # certbot +coverage==5.5 + # via -r requirements-tests.txt +cryptography==3.4.6 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # acme + # certbot + # josepy + # moto + # paramiko + # pyopenssl + # python-jose + # sshpubkeys +decorator==4.4.2 + # via + # -r requirements-tests.txt + # networkx +distro==1.5.0 + # via + # -r requirements-tests.txt + # certbot +dnspython3==1.15.0 + # via -r requirements-docs.in +dnspython==1.15.0 + # via dnspython3 +docker==4.2.0 + # via + # -r requirements-tests.txt + # moto docutils==0.15.2 # via sphinx +dyn==1.8.1 + # via -r requirements-docs.in +ecdsa==0.14.1 + # via + # -r requirements-tests.txt + # moto + # python-jose + # sshpubkeys +factory-boy==3.2.0 + # via -r requirements-tests.txt +faker==6.5.0 + # via + # -r requirements-tests.txt + # factory-boy +fakeredis==1.4.5 + # via -r requirements-tests.txt +flask-bcrypt==0.7.1 + # via -r requirements-docs.in +flask-cors==3.0.10 + # via -r requirements-docs.in +flask-mail==0.9.1 + # via -r requirements-docs.in +flask-migrate==2.7.0 + # via -r requirements-docs.in +flask-principal==0.4.0 + # via -r requirements-docs.in +flask-replicated==1.4 + # via -r requirements-docs.in +flask-restful==0.3.8 + # via -r requirements-docs.in +flask-script==2.0.6 + # via -r requirements-docs.in +flask-sqlalchemy==2.4.4 + # via + # -r requirements-docs.in + # flask-migrate +flask==1.1.2 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # flask-bcrypt + # flask-cors + # flask-mail + # flask-migrate + # flask-principal + # flask-restful + # flask-script + # flask-sqlalchemy + # pytest-flask + # raven +freezegun==1.1.0 + # via -r requirements-tests.txt +future==0.18.2 + # via + # -r requirements-tests.txt + # aws-xray-sdk +gitdb==4.0.4 + # via + # -r requirements-tests.txt + # gitpython +gitpython==3.1.1 + # via + # -r requirements-tests.txt + # bandit +gunicorn==20.0.4 + # via -r requirements-docs.in +hvac==0.10.8 + # via -r requirements-docs.in idna==2.9 - # via requests + # via + # -r requirements-tests.txt + # moto + # requests imagesize==1.2.0 # via sphinx +importlib-metadata==1.6.0 + # via + # -r requirements-tests.txt + # jsonpickle +inflection==0.5.1 + # via -r requirements-docs.in +iniconfig==1.0.1 + # via + # -r requirements-tests.txt + # pytest +itsdangerous==1.1.0 + # via + # -r requirements-tests.txt + # flask +javaobj-py3==0.4.2 + # via pyjks jinja2==2.11.3 - # via sphinx + # via + # -r requirements-tests.txt + # flask + # moto + # sphinx +jmespath==0.9.5 + # via + # -r requirements-tests.txt + # boto3 + # botocore +josepy==1.7.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # acme + # certbot +jsondiff==1.1.2 + # via + # -r requirements-tests.txt + # moto +jsonlines==2.0.0 + # via cloudflare +jsonpatch==1.25 + # via + # -r requirements-tests.txt + # cfn-lint +jsonpickle==1.4 + # via + # -r requirements-tests.txt + # aws-xray-sdk +jsonpointer==2.0 + # via + # -r requirements-tests.txt + # jsonpatch +jsonschema==3.2.0 + # via + # -r requirements-tests.txt + # aws-sam-translator + # cfn-lint +logmatic-python==0.1.7 + # via -r requirements-docs.in +mako==1.1.4 + # via alembic markupsafe==1.1.1 - # via jinja2 + # via + # -r requirements-tests.txt + # jinja2 + # mako + # moto +marshmallow-sqlalchemy==0.23.1 + # via -r requirements-docs.in +marshmallow==2.20.4 + # via + # -r requirements-docs.in + # marshmallow-sqlalchemy +mock==4.0.2 + # via + # -r requirements-tests.txt + # moto +more-itertools==8.2.0 + # via + # -r requirements-tests.txt + # moto +moto[ec2,elb,elbv2,iam,s3,ses,sns,sqs,sts]==1.3.16 + # via -r requirements-tests.txt +mypy-extensions==0.4.3 + # via + # -r requirements-tests.txt + # black +networkx==2.4 + # via + # -r requirements-tests.txt + # cfn-lint +nose==1.3.7 + # via -r requirements-tests.txt packaging==20.3 - # via sphinx + # via + # -r requirements-tests.txt + # pytest + # sphinx +paramiko==2.7.2 + # via -r requirements-docs.in +parsedatetime==2.6 + # via + # -r requirements-tests.txt + # certbot +pathspec==0.8.0 + # via + # -r requirements-tests.txt + # black +pbr==5.4.5 + # via + # -r requirements-tests.txt + # stevedore +pem==21.1.0 + # via -r requirements-docs.in +pluggy==0.13.1 + # via + # -r requirements-tests.txt + # pytest +py==1.9.0 + # via + # -r requirements-tests.txt + # pytest +pyasn1-modules==0.2.8 + # via pyjks +pyasn1==0.4.8 + # via + # -r requirements-tests.txt + # pyasn1-modules + # pyjks + # python-jose + # rsa +pycparser==2.20 + # via + # -r requirements-tests.txt + # cffi +pycryptodomex==3.10.1 + # via pyjks +pyflakes==2.2.0 + # via -r requirements-tests.txt pygments==2.6.1 # via sphinx +pyjks==20.0.0 + # via -r requirements-docs.in +pyjwt==2.0.1 + # via -r requirements-docs.in +pynacl==1.4.0 + # via paramiko +pyopenssl==20.0.1 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # acme + # josepy pyparsing==2.4.7 - # via packaging + # via + # -r requirements-tests.txt + # packaging +pyrfc3339==1.1 + # via + # -r requirements-tests.txt + # acme + # certbot +pyrsistent==0.16.0 + # via + # -r requirements-tests.txt + # jsonschema +pytest-flask==1.2.0 + # via -r requirements-tests.txt +pytest-mock==3.5.1 + # via -r requirements-tests.txt +pytest==6.2.2 + # via + # -r requirements-tests.txt + # pytest-flask + # pytest-mock +python-dateutil==2.8.1 + # via + # -r requirements-tests.txt + # alembic + # arrow + # botocore + # faker + # freezegun + # moto +python-editor==1.0.4 + # via alembic +python-jose[cryptography]==3.1.0 + # via + # -r requirements-tests.txt + # moto +python-json-logger==2.0.1 + # via logmatic-python pytz==2019.3 - # via babel + # via + # -r requirements-tests.txt + # acme + # babel + # certbot + # flask-restful + # moto + # pyrfc3339 +pyyaml==5.4.1 + # via + # -r requirements-tests.txt + # bandit + # cfn-lint + # cloudflare + # moto +raven[flask]==6.10.0 + # via -r requirements-docs.in +redis==3.5.3 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # fakeredis +regex==2020.4.4 + # via + # -r requirements-tests.txt + # black +requests-mock==1.8.0 + # via -r requirements-tests.txt +requests-toolbelt==0.9.1 + # via + # -r requirements-tests.txt + # acme requests==2.25.1 - # via sphinx + # via + # -r requirements-tests.txt + # acme + # certsrv + # cloudflare + # docker + # hvac + # moto + # requests-mock + # requests-toolbelt + # responses + # sphinx +responses==0.10.12 + # via + # -r requirements-tests.txt + # moto +retrying==1.3.3 + # via -r requirements-docs.in +rsa==4.0 + # via + # -r requirements-tests.txt + # python-jose +s3transfer==0.3.3 + # via + # -r requirements-tests.txt + # boto3 six==1.15.0 # via + # -r requirements-tests.txt + # aws-sam-translator + # bandit + # bcrypt + # cfn-lint + # configobj + # docker + # ecdsa + # fakeredis + # flask-cors + # flask-restful + # hvac + # josepy + # jsonschema + # moto # packaging + # pynacl + # pyopenssl + # pyrsistent + # python-dateutil + # python-jose + # requests-mock + # responses + # retrying # sphinxcontrib-httpdomain + # sqlalchemy-utils + # stevedore + # websocket-client +smmap==3.0.2 + # via + # -r requirements-tests.txt + # gitdb snowballstemmer==2.0.0 # via sphinx +sortedcontainers==2.1.0 + # via + # -r requirements-tests.txt + # fakeredis +soupsieve==2.0.1 + # via beautifulsoup4 sphinx-rtd-theme==0.5.1 # via -r requirements-docs.in -sphinx==3.5.0 +sphinx==3.5.2 # via # -r requirements-docs.in # sphinx-rtd-theme @@ -59,8 +529,104 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy-utils==0.36.8 + # via -r requirements-docs.in +sqlalchemy==1.3.16 + # via + # alembic + # flask-sqlalchemy + # marshmallow-sqlalchemy + # sqlalchemy-utils +sshpubkeys==3.1.0 + # via + # -r requirements-tests.txt + # moto +stevedore==1.32.0 + # via + # -r requirements-tests.txt + # bandit +tabulate==0.8.9 + # via -r requirements-docs.in +text-unidecode==1.3 + # via + # -r requirements-tests.txt + # faker +toml==0.10.1 + # via + # -r requirements-tests.txt + # black + # pytest +twofish==0.3.0 + # via pyjks +typed-ast==1.4.1 + # via + # -r requirements-tests.txt + # black +typing-extensions==3.7.4.3 + # via + # -r requirements-tests.txt + # black urllib3==1.25.8 - # via requests + # via + # -r requirements-tests.txt + # botocore + # requests +vine==1.3.0 + # via -r requirements-docs.in +websocket-client==0.57.0 + # via + # -r requirements-tests.txt + # docker +werkzeug==1.0.1 + # via + # -r requirements-tests.txt + # flask + # moto + # pytest-flask +wrapt==1.12.1 + # via + # -r requirements-tests.txt + # aws-xray-sdk +xmltodict==0.12.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # moto +zipp==3.1.0 + # via + # -r requirements-tests.txt + # importlib-metadata + # moto +zope.component==4.6.2 + # via + # -r requirements-tests.txt + # certbot +zope.deferredimport==4.3.1 + # via + # -r requirements-tests.txt + # zope.component +zope.deprecation==4.4.0 + # via + # -r requirements-tests.txt + # zope.component +zope.event==4.5.0 + # via + # -r requirements-tests.txt + # zope.component +zope.hookable==5.0.1 + # via + # -r requirements-tests.txt + # zope.component +zope.interface==5.2.0 + # via + # -r requirements-tests.txt + # certbot + # zope.component + # zope.proxy +zope.proxy==4.3.5 + # via + # -r requirements-tests.txt + # zope.deferredimport # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.in b/requirements-tests.in index 610f26f9..9b2b6988 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -3,11 +3,12 @@ bandit black coverage +certbot factory-boy Faker fakeredis freezegun -moto +moto[sts,ec2,elb,elbv2,iam,s3,sns,sqs,ses] nose pyflakes pytest diff --git a/requirements-tests.txt b/requirements-tests.txt index e7bc7e0c..49f56952 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -4,6 +4,8 @@ # # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # +acme==1.13.0 + # via certbot appdirs==1.4.3 # via black attrs==19.3.0 @@ -18,18 +20,20 @@ bandit==1.7.0 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.17.7 +boto3==1.17.27 # via # aws-sam-translator # moto boto==2.49.0 # via moto -botocore==1.20.7 +botocore==1.20.27 # via # aws-xray-sdk # boto3 # moto # s3transfer +certbot==1.13.0 + # via -r requirements-tests.in certifi==2020.12.5 # via requests cffi==1.14.0 @@ -42,15 +46,25 @@ click==7.1.2 # via # black # flask -coverage==5.4 +configargparse==1.4 + # via certbot +configobj==5.0.6 + # via certbot +coverage==5.5 # via -r requirements-tests.in -cryptography==3.4.5 +cryptography==3.4.6 # via + # acme + # certbot + # josepy # moto + # pyopenssl # python-jose # sshpubkeys decorator==4.4.2 # via networkx +distro==1.5.0 + # via certbot docker==4.2.0 # via moto ecdsa==0.14.1 @@ -60,7 +74,7 @@ ecdsa==0.14.1 # sshpubkeys factory-boy==3.2.0 # via -r requirements-tests.in -faker==6.1.1 +faker==6.5.0 # via # -r requirements-tests.in # factory-boy @@ -94,6 +108,10 @@ jmespath==0.9.5 # via # boto3 # botocore +josepy==1.7.0 + # via + # acme + # certbot jsondiff==1.1.2 # via moto jsonpatch==1.25 @@ -114,7 +132,7 @@ mock==4.0.2 # via moto more-itertools==8.2.0 # via moto -moto==1.3.16 +moto[ec2,elb,elbv2,iam,s3,ses,sns,sqs,sts]==1.3.16 # via -r requirements-tests.in mypy-extensions==0.4.3 # via black @@ -124,6 +142,8 @@ nose==1.3.7 # via -r requirements-tests.in packaging==20.3 # via pytest +parsedatetime==2.6 + # via certbot pathspec==0.8.0 # via black pbr==5.4.5 @@ -140,11 +160,19 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via -r requirements-tests.in +pyopenssl==20.0.1 + # via + # acme + # josepy pyparsing==2.4.7 # via packaging +pyrfc3339==1.1 + # via + # acme + # certbot pyrsistent==0.16.0 # via jsonschema -pytest-flask==1.1.0 +pytest-flask==1.2.0 # via -r requirements-tests.in pytest-mock==3.5.1 # via -r requirements-tests.in @@ -162,7 +190,11 @@ python-dateutil==2.8.1 python-jose[cryptography]==3.1.0 # via moto pytz==2019.3 - # via moto + # via + # acme + # certbot + # moto + # pyrfc3339 pyyaml==5.4.1 # via # -r requirements-tests.in @@ -175,11 +207,15 @@ regex==2020.4.4 # via black requests-mock==1.8.0 # via -r requirements-tests.in +requests-toolbelt==0.9.1 + # via acme requests==2.25.1 # via + # acme # docker # moto # requests-mock + # requests-toolbelt # responses responses==0.10.12 # via moto @@ -192,12 +228,15 @@ six==1.15.0 # aws-sam-translator # bandit # cfn-lint + # configobj # docker # ecdsa # fakeredis + # josepy # jsonschema # moto # packaging + # pyopenssl # pyrsistent # python-dateutil # python-jose @@ -242,6 +281,23 @@ zipp==3.1.0 # via # importlib-metadata # moto +zope.component==4.6.2 + # via certbot +zope.deferredimport==4.3.1 + # via zope.component +zope.deprecation==4.4.0 + # via zope.component +zope.event==4.5.0 + # via zope.component +zope.hookable==5.0.1 + # via zope.component +zope.interface==5.2.0 + # via + # certbot + # zope.component + # zope.proxy +zope.proxy==4.3.5 + # via zope.deferredimport # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.in b/requirements.in index 1eb96f97..91e6309d 100644 --- a/requirements.in +++ b/requirements.in @@ -7,6 +7,7 @@ asyncpool boto3 botocore celery[redis]==4.4.2 # need to first resolve the module not found error https://github.com/celery/celery/issues/6406 +certbot certifi certsrv CloudFlare diff --git a/requirements.txt b/requirements.txt index c6a21ef7..c70b42c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,10 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.12.0 - # via -r requirements.in +acme==1.13.0 + # via + # -r requirements.in + # certbot alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 @@ -14,7 +16,7 @@ amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.17.0 +arrow==1.0.3 # via -r requirements.in asyncpool==1.0 # via -r requirements.in @@ -31,15 +33,17 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.7 +boto3==1.17.27 # via -r requirements.in -botocore==1.20.7 +botocore==1.20.27 # via # -r requirements.in # boto3 # s3transfer celery[redis]==4.4.2 # via -r requirements.in +certbot==1.13.0 + # via -r requirements.in certifi==2020.12.5 # via # -r requirements.in @@ -57,14 +61,20 @@ click==7.1.2 # via flask cloudflare==2.8.15 # via -r requirements.in -cryptography==3.4.5 +configargparse==1.4 + # via certbot +configobj==5.0.6 + # via certbot +cryptography==3.4.6 # via # -r requirements.in # acme + # certbot # josepy # paramiko # pyopenssl - # requests +distro==1.5.0 + # via certbot dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 @@ -77,7 +87,7 @@ flask-cors==3.0.10 # via -r requirements.in flask-mail==0.9.1 # via -r requirements.in -flask-migrate==2.6.0 +flask-migrate==2.7.0 # via -r requirements.in flask-principal==0.4.0 # via -r requirements.in @@ -125,8 +135,10 @@ jmespath==0.9.5 # via # boto3 # botocore -josepy==1.3.0 - # via acme +josepy==1.7.0 + # via + # acme + # certbot jsonlines==1.2.0 # via cloudflare kombu==4.6.8 @@ -151,6 +163,8 @@ ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.2 # via -r requirements.in +parsedatetime==2.6 + # via certbot pem==21.1.0 # via -r requirements.in psycopg2==2.8.6 @@ -181,9 +195,10 @@ pyopenssl==20.0.1 # acme # josepy # ndg-httpsclient - # requests pyrfc3339==1.1 - # via acme + # via + # acme + # certbot python-dateutil==2.8.1 # via # alembic @@ -199,6 +214,7 @@ pytz==2019.3 # via # acme # celery + # certbot # flask-restful # pyrfc3339 pyyaml==5.4.1 @@ -213,7 +229,7 @@ redis==3.5.3 # celery requests-toolbelt==0.9.1 # via acme -requests[security]==2.25.1 +requests==2.25.1 # via # -r requirements.in # acme @@ -228,8 +244,8 @@ s3transfer==0.3.3 six==1.15.0 # via # -r requirements.in - # acme # bcrypt + # configobj # flask-cors # flask-restful # hvac @@ -250,7 +266,7 @@ sqlalchemy==1.3.16 # flask-sqlalchemy # marshmallow-sqlalchemy # sqlalchemy-utils -tabulate==0.8.7 +tabulate==0.8.9 # via -r requirements.in twofish==0.3.0 # via pyjks @@ -266,6 +282,22 @@ werkzeug==1.0.1 # via flask xmltodict==0.12.0 # via -r requirements.in +zope.component==4.6.2 + # via certbot +zope.deferredimport==4.3.1 + # via zope.component +zope.deprecation==4.4.0 + # via zope.component +zope.event==4.5.0 + # via zope.component +zope.hookable==5.0.1 + # via zope.component +zope.interface==5.2.0 + # via + # certbot + # zope.component +zope.proxy==4.3.5 + # via zope.deferredimport # The following packages are considered to be unsafe in a requirements file: # setuptools