Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2 from Netflix/master 

Fxing Issue #2831
This commit is contained in:
sirferl 2020-01-27 10:59:03 +01:00 committed by GitHub
parent 194e2a43e7 21c651a566
commit 6248cbb902
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
276 changed files with 13856 additions and 6193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -26,6 +26,11 @@ package-lock.json
/lemur/static/dist/
/lemur/static/app/vendor/
/wheelhouse
/lemur/lib
/lemur/bin
/lemur/lib64
/lemur/include
docs/_build
.editorconfig
.idea

14
.pre-commit-config.yaml
View File

@ -8,3 +8,17 @@
sha: v2.9.5
hooks:
- id: jshint
- repo: https://github.com/ambv/black
rev: stable
hooks:
- id: black
language_version: python3.7
- repo: local
hooks:
- id: python-bandit-vulnerability-check
name: bandit
entry: bandit
args: ['--ini', 'tox.ini', '-r', 'consoleme']
language: system
pass_filenames: false

7
.travis.yml
View File

@ -1,6 +1,5 @@
language: python
sudo: required
dist: trusty
dist: xenial
node_js:
- "6.2.0"
@ -10,8 +9,8 @@ addons:
matrix:
include:
- python: "3.5"
env: TOXENV=py35
- python: "3.7"
env: TOXENV=py37
cache:
directories:

6
Dockerfile
View File

@ -1,9 +1,9 @@
FROM python:3.5
FROM python:3.7
RUN apt-get update
RUN apt-get install -y make python-software-properties curl
RUN apt-get install -y make software-properties-common curl
RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
RUN apt-get update
RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev
RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
RUN pip install -U setuptools
RUN pip install coveralls bandit
WORKDIR /app

10
Makefile
View File

@ -36,7 +36,7 @@ endif
@echo ""
dev-docs:
pip install -r docs/requirements.txt
pip install -r requirements-docs.txt
reset-db:
@echo "--> Dropping existing 'lemur' database"
@ -46,7 +46,7 @@ reset-db:
@echo "--> Enabling pg_trgm extension"
psql lemur -c "create extension IF NOT EXISTS pg_trgm;"
@echo "--> Applying migrations"
lemur db upgrade
cd lemur && lemur db upgrade
setup-git:
@echo "--> Installing git hooks"
@ -113,10 +113,10 @@ endif
@echo "--> Updating Python requirements"
pip install --upgrade pip
pip install --upgrade pip-tools
pip-compile --output-file requirements.txt requirements.in -U --no-index
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
pip-compile --output-file requirements.txt requirements.in -U --no-index
@echo "--> Done updating Python requirements"
@echo "--> Removing python-ldap from requirements-docs.txt"
grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt
@ -125,5 +125,9 @@ endif
@echo "--> Done installing new dependencies"
@echo ""
# Execute with make checkout-pr pr=<pr number>
checkout-pr:
git fetch upstream pull/$(pr)/head:pr-$(pr)
.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

5
docker-compose.yml
View File

@ -13,10 +13,13 @@ services:
VIRTUAL_ENV: 'true'
postgres:
image: postgres:9.4
image: postgres
restart: always
environment:
POSTGRES_USER: lemur
POSTGRES_PASSWORD: lemur
ports:
- "5432:5432"
redis:
image: "redis:alpine"

64
docker/Dockerfile Normal file
View File

@ -0,0 +1,64 @@
FROM alpine:3.8
ARG VERSION
ENV VERSION master
ENV uid 1337
ENV gid 1337
ENV user lemur
ENV group lemur
COPY entrypoint /
COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
COPY supervisor.conf /
COPY nginx/default.conf /etc/nginx/conf.d/
COPY nginx/default-ssl.conf /etc/nginx/conf.d/
RUN addgroup -S ${group} -g ${gid} && \
adduser -D -S ${user} -G ${group} -u ${uid} && \
apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
apk --update add --virtual build-dependencies \
git \
tar \
curl \
python3-dev \
npm \
bash \
musl-dev \
gcc \
autoconf \
automake \
make \
nasm \
zlib-dev \
postgresql-dev \
libressl-dev \
libffi-dev \
cyrus-sasl-dev \
openldap-dev && \
mkdir -p /opt/lemur /home/lemur/.lemur/ && \
curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
pip3 install --upgrade pip && \
pip3 install --upgrade setuptools && \
chmod +x /entrypoint && \
mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
WORKDIR /opt/lemur
RUN npm install --unsafe-perm && \
pip3 install -e . && \
node_modules/.bin/gulp build && \
node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
apk del build-dependencies
WORKDIR /
HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \
CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
USER root
ENTRYPOINT ["/entrypoint"]
CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

54
docker/entrypoint Normal file
View File

@ -0,0 +1,54 @@
#!/bin/sh
if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
echo "Database vars not set"
exit 1
fi
export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
source /etc/profile
PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
echo " # Create Postgres trgm extension"
PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
echo " # Done"
if [ -z "${SKIP_SSL}" ]; then
if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
fi
mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
fi
# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
# echo "Creating config"
# https://github.com/Netflix/lemur/issues/2257
# python3 /opt/lemur/lemur/manage.py create_config
# echo "Done"
# fi
echo " # Running init"
su lemur -c "python3 /opt/lemur/lemur/manage.py init"
echo " # Done"
# echo "Creating user"
# https://github.com/Netflix/lemur/issues/
# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
# echo "Done"
cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
echo " # Populating crontab"
echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
echo " # Done"
exec "$@"

37
docker/nginx/default-ssl.conf Normal file
View File

@ -0,0 +1,37 @@
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
server {
listen 80;
server_name _;
return 301 https://$host$request_uri;
}
server {
listen 443;
server_name _;
access_log /dev/stdout;
error_log /dev/stderr;
ssl_certificate /etc/nginx/ssl/server.crt;
ssl_certificate_key /etc/nginx/ssl/server.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
location /api {
proxy_pass http://127.0.0.1:8000;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
root /opt/lemur/lemur/static/dist;
include mime.types;
index index.html;
}
}

26
docker/nginx/default.conf Normal file
View File

@ -0,0 +1,26 @@
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
server {
listen 80;
access_log /dev/stdout;
error_log /dev/stderr;
location /api {
proxy_pass http://127.0.0.1:8000;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
root /opt/lemur/lemur/static/dist;
include mime.types;
index index.html;
}
}

31
docker/src/lemur.conf.py Normal file
View File

@ -0,0 +1,31 @@
import os
_basedir = os.path.abspath(os.path.dirname(__file__))
CORS = os.environ.get("CORS") == "True"
debug = os.environ.get("DEBUG") == "True"
SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw=='))
LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ=='))
LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo='))
LEMUR_WHITELISTED_DOMAINS = []
LEMUR_EMAIL = ''
LEMUR_SECURITY_TEAM_EMAIL = []
LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
ACTIVE_PROVIDERS = []
METRIC_PROVIDERS = []
LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')

32
docker/supervisor.conf Normal file
View File

@ -0,0 +1,32 @@
[supervisord]
nodaemon=true
user=root
logfile=/dev/stdout
logfile_maxbytes=0
pidfile = /tmp/supervisord.pid
[program:lemur]
environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
user=lemur
directory=/opt/lemur/lemur
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:nginx]
command=/usr/sbin/nginx -g "daemon off;"
user=root
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:cron]
command=/usr/sbin/crond -f
user=root
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0

134
docs/administration.rst
View File

@ -161,6 +161,13 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
Dump all imported or generated CSR and certificate details to stdout using OpenSSL. (default: `False`)
.. data:: ALLOW_CERT_DELETION
:noindex:
When set to True, certificates can be marked as deleted via the API and deleted certificates will not be displayed
in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed"
and deleted certificates will always be visible in the UI. (default: `False`)
Certificate Default Options
---------------------------
@ -586,8 +593,60 @@ If you are not using a metric provider you do not need to configure any of these
Plugin Specific Options
-----------------------
Active Directory Certificate Services Plugin
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. data:: ADCS_SERVER
:noindex:
FQDN of your ADCS Server
.. data:: ADCS_AUTH_METHOD
:noindex:
The chosen authentication method. Either ‘basic’ (the default), ‘ntlm’ or ‘cert’ (SSL client certificate). The next 2 variables are interpreted differently for different methods.
.. data:: ADCS_USER
:noindex:
The username (basic) or the path to the public cert (cert) of the user accessing PKI
.. data:: ADCS_PWD
:noindex:
The passwd (basic) or the path to the private key (cert) of the user accessing PKI
.. data:: ADCS_TEMPLATE
:noindex:
Template to be used for certificate issuing. Usually display name w/o spaces
.. data:: ADCS_START
:noindex:
.. data:: ADCS_STOP
:noindex:
.. data:: ADCS_ISSUING
:noindex:
Contains the issuing cert of the CA
.. data:: ADCS_ROOT
:noindex:
Contains the root cert of the CA
Verisign Issuer Plugin
^^^^^^^^^^^^^^^^^^^^^^
~~~~~~~~~~~~~~~~~~~~~~
Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
@ -683,7 +742,7 @@ The following configuration properties are required to use the Digicert issuer p
CFSSL Issuer Plugin
^^^^^^^^^^^^^^^^^^^
~~~~~~~~~~~~~~~~~~~
The following configuration properties are required to use the CFSSL issuer plugin.
@ -702,9 +761,36 @@ The following configuration properties are required to use the CFSSL issuer plug
This is the intermediate to be used for your CA chain
.. data:: CFSSL_KEY
:noindex:
This is the hmac key to authenticate to the CFSSL service. (Optional)
Hashicorp Vault Source/Destination Plugin
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination.
.. note:: This plugin does not supersede or overlap the 3rd party Vault Issuer plugin.
.. note:: Vault does not have any configuration properties however it does read from a file on disk for a vault access token. The Lemur service account needs read access to this file.
Vault Source
""""""""""""
The Vault Source Plugin will read from one Vault object location per source defined. There is expected to be one or more certificates defined in each object in Vault.
Vault Destination
"""""""""""""""""
A Vault destination can be one object in Vault or a directory where all certificates will be stored as their own object by CN.
Vault Destination supports a regex filter to prevent certificates with SAN that do not match the regex filter from being deployed. This is an optional feature per destination defined.
AWS Source/Destination Plugin
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
@ -1056,7 +1142,9 @@ Verisign/Symantec
-----------------
:Authors:
Kevin Glisson <kglisson@netflix.com>
Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type:
Issuer
:Description:
@ -1082,6 +1170,8 @@ Acme
:Authors:
Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>,
Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
:Type:
Issuer
@ -1093,7 +1183,9 @@ Atlas
-----
:Authors:
Kevin Glisson <kglisson@netflix.com>
Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type:
Metric
:Description:
@ -1104,7 +1196,9 @@ Email
-----
:Authors:
Kevin Glisson <kglisson@netflix.com>
Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type:
Notification
:Description:
@ -1126,7 +1220,9 @@ AWS
----
:Authors:
Kevin Glisson <kglisson@netflix.com>
Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type:
Source
:Description:
@ -1137,7 +1233,9 @@ AWS
----
:Authors:
Kevin Glisson <kglisson@netflix.com>
Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type:
Destination
:Description:
@ -1187,6 +1285,26 @@ CFSSL
:Description:
Basic support for generating certificates from the private certificate authority CFSSL
Vault
-----
:Authors:
Christopher Jolley <chris@alwaysjolley.com>
:Type:
Source
:Description:
Source plugin imports certificates from Hashicorp Vault secret store.
Vault
-----
:Authors:
Christopher Jolley <chris@alwaysjolley.com>
:Type:
Destination
:Description:
Destination plugin to deploy certificates to Hashicorp Vault secret store.
3rd Party Plugins
=================

154
docs/conf.py
View File

@ -18,48 +18,45 @@ from unittest.mock import MagicMock
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath(".."))
# Mock packages that cannot be installed on rtd
on_rtd = os.environ.get('READTHEDOCS') == 'True'
on_rtd = os.environ.get("READTHEDOCS") == "True"
if on_rtd:
class Mock(MagicMock):
@classmethod
def __getattr__(cls, name):
return MagicMock()
MOCK_MODULES = ['ldap']
MOCK_MODULES = ["ldap"]
sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinxcontrib.autohttp.flask',
'sphinx.ext.todo',
]
extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask", "sphinx.ext.todo"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = u'lemur'
copyright = u'2018, Netflix Inc.'
project = u"lemur"
copyright = u"2018, Netflix Inc."
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@ -68,191 +65,186 @@ copyright = u'2018, Netflix Inc.'
base_dir = os.path.join(os.path.dirname(__file__), os.pardir)
about = {}
with open(os.path.join(base_dir, "lemur", "__about__.py")) as f:
exec(f.read(), about)
exec(f.read(), about) # nosec
version = release = about["__version__"]
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# keep_warnings = False
# -- Options for HTML output ----------------------------------------------
# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
on_rtd = os.environ.get("READTHEDOCS", None) == "True"
if not on_rtd: # only import and set the theme if we're building docs locally
import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []
# html_extra_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'lemurdoc'
htmlhelp_basename = "lemurdoc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'lemur.tex', u'Lemur Documentation',
u'Kevin Glisson', 'manual'),
("index", "lemur.tex", u"Lemur Documentation", u"Netflix Security", "manual")
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'Lemur', u'Lemur Documentation',
[u'Kevin Glisson'], 1)
]
man_pages = [("index", "Lemur", u"Lemur Documentation", [u"Netflix Security"], 1)]
# If true, show URL addresses after external links.
#man_show_urls = False
# man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
@ -261,19 +253,25 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'Lemur', u'Lemur Documentation',
u'Kevin Glisson', 'Lemur', 'SSL Certificate Management',
'Miscellaneous'),
(
"index",
"Lemur",
u"Lemur Documentation",
u"Netflix Security",
"Lemur",
"SSL Certificate Management",
"Miscellaneous",
)
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
# texinfo_no_detailmenu = False

16
docs/developer/index.rst
View File

@ -22,12 +22,18 @@ Once you've got all that, the rest is simple:
# If you have a fork, you'll want to clone it instead
git clone git://github.com/netflix/lemur.git
# Create a python virtualenv
mkvirtualenv lemur
# Create and activate python virtualenv from within the lemur repo
python3 -m venv env
. env/bin/activate
# Install doc requirements
# Make the magic happen
make dev-docs
# Make the docs
cd docs
make html
Running ``make dev-docs`` will install the basic requirements to get Sphinx running.
@ -58,7 +64,7 @@ Once you've got all that, the rest is simple:
git clone git://github.com/lemur/lemur.git
# Create a python virtualenv
mkvirtualenv lemur
python3 -m venv env
# Make the magic happen
make
@ -135,7 +141,7 @@ The test suite consists of multiple parts, testing both the Python and JavaScrip
make test
If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``test-js`` for the JavaScript tests.
If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``make test-js`` for the JavaScript tests.
You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests.

BIN
docs/production/create_dns_provider.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 86 KiB

117
docs/production/index.rst
View File

@ -318,7 +318,7 @@ Periodic Tasks
==============
Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
a cron job that runs the commands.
celery tasks or cron jobs that run these commands.
There are currently three commands that could/should be run on a periodic basis:
@ -326,11 +326,124 @@ There are currently three commands that could/should be run on a periodic basis:
- `check_revoked`
- `sync`
If you are using LetsEncrypt, you must also run the following:
- `fetch_all_pending_acme_certs`
- `remove_old_acme_certs`
How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
`sync` is typically run every 15 minutes.
`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
`remove_old_acme_certs` can be ran more rarely, such as once every week.
Example cron entries::
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
*/15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
Example Celery configuration (To be placed in your configuration file)::
CELERYBEAT_SCHEDULE = {
'fetch_all_pending_acme_certs': {
'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
'options': {
'expires': 180
},
'schedule': crontab(minute="*"),
},
'remove_old_acme_certs': {
'task': 'lemur.common.celery.remove_old_acme_certs',
'options': {
'expires': 180
},
'schedule': crontab(hour=7, minute=30, day_of_week=1),
},
'clean_all_sources': {
'task': 'lemur.common.celery.clean_all_sources',
'options': {
'expires': 180
},
'schedule': crontab(hour=1, minute=0, day_of_week=1),
},
'sync_all_sources': {
'task': 'lemur.common.celery.sync_all_sources',
'options': {
'expires': 180
},
'schedule': crontab(hour="*/3", minute=5),
},
'sync_source_destination': {
'task': 'lemur.common.celery.sync_source_destination',
'options': {
'expires': 180
},
'schedule': crontab(hour="*"),
}
}
To enable celery support, you must also have configuration values that tell Celery which broker and backend to use.
Here are the Celery configuration variables that should be set::
CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379'
CELERY_BROKER_URL = 'redis://your_redis_url:6379'
CELERY_IMPORTS = ('lemur.common.celery')
CELERY_TIMEZONE = 'UTC'
You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
The scheduler can be started with::
LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery beat
And the worker can be started with desired options such as the following::
LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery worker --concurrency 10 -E -n lemurworker1@%%h
supervisor or systemd configurations should be created for these in production environments as appropriate.
Add support for LetsEncrypt
===========================
LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
LetsEncrypt requires that we prove ownership of a domain before we're able to issue a certificate for that domain, each
time we want a certificate.
The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
through the creation of DNS TXT records.
In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
prove ownership of all of these domains through this method. The token is typically written to a TXT record at
-acme_challenge.domain.com. Once we create the appropriate TXT record(s), Lemur will try to validate propagation
before requesting that LetsEncrypt finalize the certificate request and send us the certificate.
.. figure:: letsencrypt_flow.png
To start issuing certificates through LetsEncrypt, you must enable Celery support within Lemur first. After doing so,
you need to create a LetsEncrypt authority. To do this, visit
Authorities -> Create. Set the applicable attributes and click "More Options".
.. figure:: letsencrypt_authority_1.png
You will need to set "Certificate" to LetsEncrypt's active chain of trust for the authority you want to use. To find
the active chain of trust at the time of writing, please visit `LetsEncrypt
<https://letsencrypt.org/certificates/>`_.
Under Acme_url, enter in the appropriate endpoint URL. Lemur supports LetsEncrypt's V2 API, and we recommend you to use
this. At the time of writing, the staging and production URLs for LetsEncrypt V2 are
https://acme-staging-v02.api.letsencrypt.org/directory and https://acme-v02.api.letsencrypt.org/directory.
.. figure:: letsencrypt_authority_2.png
After creating the authorities, we will need to create a DNS provider. Visit `Admin` -> `DNS Providers` and click
`Create`. Lemur comes with a few provider plugins built in, with different options. Create a DNS provider with the
appropriate choices.
.. figure:: create_dns_provider.png
By default, users will need to select the DNS provider that is authoritative over their domain in order for the
LetsEncrypt flow to function. However, Lemur will attempt to automatically determine the appropriate provider if
possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.

BIN
docs/production/letsencrypt_authority_1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 132 KiB

BIN
docs/production/letsencrypt_authority_2.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 218 KiB

BIN
docs/production/letsencrypt_flow.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 89 KiB

29
docs/quickstart/index.rst
View File

@ -12,7 +12,7 @@ Dependencies
Some basic prerequisites which you'll need in order to run Lemur:
* A UNIX-based operating system (we test on Ubuntu, develop on OS X)
* Python 3.5 or greater
* Python 3.7 or greater
* PostgreSQL 9.4 or greater
* Nginx
@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur:
Installing Build Dependencies
-----------------------------
If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
.. code-block:: bash
@ -31,7 +31,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack
.. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git).
.. note:: Installing node from a package manager may creat the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
.. note:: Installing node from a package manager may create the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
sudo ln -s /user/bin/nodejs /usr/bin/node
Now, install Python ``virtualenv`` package:
@ -117,7 +117,7 @@ Simply run:
.. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment.
You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment.
Update your configuration
@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config
LEMUR_DEFAULT_ORGANIZATION
LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
Setup Postgres
Set Up Postgres
--------------
For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
@ -180,6 +180,13 @@ Lemur provides a helpful command that will initialize your database for you. It
In addition to creating a new user, Lemur also creates a few default email notifications. These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``. They basically guarantee that every certificate within Lemur will send one expiration notification to the security team.
Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user.
.. code-block:: bash
sudo -u postgres -i
psql
postgres=# ALTER USER lemur WITH SUPERUSER
Additional notifications can be created through the UI or API. See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
**Make note of the password used as this will be used during first login to the Lemur UI.**
@ -189,14 +196,20 @@ Additional notifications can be created through the UI or API. See :ref:`Creati
cd /www/lemur/lemur
lemur init
.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now.
.. code-block:: bash
sudo -u postgres -i
psql
postgres=# ALTER USER lemur WITH NOSUPERUSER
.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
Setup a Reverse Proxy
Set Up a Reverse Proxy
---------------------
By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
Proxying with Nginx

8
gulp/server.js
View File

@ -6,9 +6,9 @@ var browserSync = require('browser-sync');
var httpProxy = require('http-proxy');
/* This configuration allow you to configure browser sync to proxy your backend */
/*
var proxyTarget = 'http://localhost/context/'; // The location of your backend
var proxyApiPrefix = 'api'; // The element in the URL which differentiate between API request and static file request
var proxyTarget = 'http://localhost:8000/'; // The location of your backend
var proxyApiPrefix = '/api/'; // The element in the URL which differentiate between API request and static file request
var proxy = httpProxy.createProxyServer({
target: proxyTarget
});
@ -19,7 +19,6 @@ var httpProxy = require('http-proxy');
next();
}
}
*/
function browserSyncInit(baseDir, files, browser) {
browser = browser === undefined ? 'default' : browser;
@ -27,6 +26,7 @@ function browserSyncInit(baseDir, files, browser) {
browserSync.instance = browserSync.init(files, {
startPath: '/index.html',
server: {
middleware: [proxyMiddleware],
baseDir: baseDir,
routes: {
'/bower_components': './bower_components'

12
lemur/__about__.py
View File

@ -1,12 +1,18 @@
from __future__ import absolute_import, division, print_function
__all__ = [
"__title__", "__summary__", "__uri__", "__version__", "__author__",
"__email__", "__license__", "__copyright__",
"__title__",
"__summary__",
"__uri__",
"__version__",
"__author__",
"__email__",
"__license__",
"__copyright__",
]
__title__ = "lemur"
__summary__ = ("Certificate management and orchestration service")
__summary__ = "Certificate management and orchestration service"
__uri__ = "https://github.com/Netflix/lemur"
__version__ = "0.7.0"

41
lemur/__init__.py
View File

@ -5,7 +5,8 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
.. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
"""
import time
@ -32,14 +33,26 @@ from lemur.pending_certificates.views import mod as pending_certificates_bp
from lemur.dns_providers.views import mod as dns_providers_bp
from lemur.__about__ import (
__author__, __copyright__, __email__, __license__, __summary__, __title__,
__uri__, __version__
__author__,
__copyright__,
__email__,
__license__,
__summary__,
__title__,
__uri__,
__version__,
)
__all__ = [
"__title__", "__summary__", "__uri__", "__version__", "__author__",
"__email__", "__license__", "__copyright__",
"__title__",
"__summary__",
"__uri__",
"__version__",
"__author__",
"__email__",
"__license__",
"__copyright__",
]
LEMUR_BLUEPRINTS = (
@ -62,8 +75,10 @@ LEMUR_BLUEPRINTS = (
)
def create_app(config=None):
app = factory.create_app(app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config)
def create_app(config_path=None):
app = factory.create_app(
app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config_path
)
configure_hook(app)
return app
@ -93,7 +108,7 @@ def configure_hook(app):
@app.after_request
def after_request(response):
# Return early if we don't have the start time
if not hasattr(g, 'request_start_time'):
if not hasattr(g, "request_start_time"):
return response
# Get elapsed time in milliseconds
@ -102,12 +117,12 @@ def configure_hook(app):
# Collect request/response tags
tags = {
'endpoint': request.endpoint,
'request_method': request.method.lower(),
'status_code': response.status_code
"endpoint": request.endpoint,
"request_method": request.method.lower(),
"status_code": response.status_code,
}
# Record our response time metric
metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags)
metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
metrics.send("response_time", "TIMER", elapsed, metric_tags=tags)
metrics.send("status_code_{}".format(response.status_code), "counter", 1)
return response

21
lemur/api_keys/cli.py
View File

@ -14,23 +14,32 @@ from datetime import datetime
manager = Manager(usage="Handles all api key related tasks.")
@manager.option('-u', '--user-id', dest='uid', help='The User ID this access key belongs too.')
@manager.option('-n', '--name', dest='name', help='The name of this API Key.')
@manager.option('-t', '--ttl', dest='ttl', help='The TTL of this API Key. -1 for forever.')
@manager.option(
"-u", "--user-id", dest="uid", help="The User ID this access key belongs too."
)
@manager.option("-n", "--name", dest="name", help="The name of this API Key.")
@manager.option(
"-t", "--ttl", dest="ttl", help="The TTL of this API Key. -1 for forever."
)
def create(uid, name, ttl):
"""
Create a new api key for a user.
:return:
"""
print("[+] Creating a new api key.")
key = api_key_service.create(user_id=uid, name=name,
ttl=ttl, issued_at=int(datetime.utcnow().timestamp()), revoked=False)
key = api_key_service.create(
user_id=uid,
name=name,
ttl=ttl,
issued_at=int(datetime.utcnow().timestamp()),
revoked=False,
)
print("[+] Successfully created a new api key. Generating a JWT...")
jwt = create_token(uid, key.id, key.ttl)
print("[+] Your JWT is: {jwt}".format(jwt=jwt))
@manager.option('-a', '--api-key-id', dest='aid', help='The API Key ID to revoke.')
@manager.option("-a", "--api-key-id", dest="aid", help="The API Key ID to revoke.")
def revoke(aid):
"""
Revokes an api key for a user.

11
lemur/api_keys/models.py
View File

@ -12,14 +12,19 @@ from lemur.database import db
class ApiKey(db.Model):
__tablename__ = 'api_keys'
__tablename__ = "api_keys"
id = Column(Integer, primary_key=True)
name = Column(String)
user_id = Column(Integer, ForeignKey('users.id'))
user_id = Column(Integer, ForeignKey("users.id"))
ttl = Column(BigInteger)
issued_at = Column(BigInteger)
revoked = Column(Boolean)
def __repr__(self):
return "ApiKey(name={name}, user_id={user_id}, ttl={ttl}, issued_at={iat}, revoked={revoked})".format(
user_id=self.user_id, name=self.name, ttl=self.ttl, iat=self.issued_at, revoked=self.revoked)
user_id=self.user_id,
name=self.name,
ttl=self.ttl,
iat=self.issued_at,
revoked=self.revoked,
)

10
lemur/api_keys/schemas.py
View File

@ -13,12 +13,18 @@ from lemur.users.schemas import UserNestedOutputSchema, UserInputSchema
def current_user_id():
return {'id': g.current_user.id, 'email': g.current_user.email, 'username': g.current_user.username}
return {
"id": g.current_user.id,
"email": g.current_user.email,
"username": g.current_user.username,
}
class ApiKeyInputSchema(LemurInputSchema):
name = fields.String(required=False)
user = fields.Nested(UserInputSchema, missing=current_user_id, default=current_user_id)
user = fields.Nested(
UserInputSchema, missing=current_user_id, default=current_user_id
)
ttl = fields.Integer()

10
lemur/api_keys/service.py
View File

@ -34,7 +34,7 @@ def revoke(aid):
:return:
"""
api_key = get(aid)
setattr(api_key, 'revoked', False)
setattr(api_key, "revoked", False)
return database.update(api_key)
@ -80,10 +80,10 @@ def render(args):
:return:
"""
query = database.session_query(ApiKey)
user_id = args.pop('user_id', None)
aid = args.pop('id', None)
has_permission = args.pop('has_permission', False)
requesting_user_id = args.pop('requesting_user_id')
user_id = args.pop("user_id", None)
aid = args.pop("id", None)
has_permission = args.pop("has_permission", False)
requesting_user_id = args.pop("requesting_user_id")
if user_id:
query = query.filter(ApiKey.user_id == user_id)

94
lemur/api_keys/views.py
View File

@ -19,10 +19,16 @@ from lemur.auth.permissions import ApiKeyCreatorPermission
from lemur.common.schema import validate_schema
from lemur.common.utils import paginated_parser
from lemur.api_keys.schemas import api_key_input_schema, api_key_revoke_schema, api_key_output_schema, \
api_keys_output_schema, api_key_described_output_schema, user_api_key_input_schema
from lemur.api_keys.schemas import (
api_key_input_schema,
api_key_revoke_schema,
api_key_output_schema,
api_keys_output_schema,
api_key_described_output_schema,
user_api_key_input_schema,
)
mod = Blueprint('api_keys', __name__)
mod = Blueprint("api_keys", __name__)
api = Api(mod)
@ -81,8 +87,8 @@ class ApiKeyList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args['has_permission'] = ApiKeyCreatorPermission().can()
args['requesting_user_id'] = g.current_user.id
args["has_permission"] = ApiKeyCreatorPermission().can()
args["requesting_user_id"] = g.current_user.id
return service.render(args)
@validate_schema(api_key_input_schema, api_key_output_schema)
@ -124,12 +130,26 @@ class ApiKeyList(AuthenticatedResource):
:statuscode 403: unauthenticated
"""
if not ApiKeyCreatorPermission().can():
if data['user']['id'] != g.current_user.id:
return dict(message="You are not authorized to create tokens for: {0}".format(data['user']['username'])), 403
if data["user"]["id"] != g.current_user.id:
return (
dict(
message="You are not authorized to create tokens for: {0}".format(
data["user"]["username"]
)
),
403,
)
access_token = service.create(name=data['name'], user_id=data['user']['id'], ttl=data['ttl'],
revoked=False, issued_at=int(datetime.utcnow().timestamp()))
return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
access_token = service.create(
name=data["name"],
user_id=data["user"]["id"],
ttl=data["ttl"],
revoked=False,
issued_at=int(datetime.utcnow().timestamp()),
)
return dict(
jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
)
class ApiKeyUserList(AuthenticatedResource):
@ -186,9 +206,9 @@ class ApiKeyUserList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args['has_permission'] = ApiKeyCreatorPermission().can()
args['requesting_user_id'] = g.current_user.id
args['user_id'] = user_id
args["has_permission"] = ApiKeyCreatorPermission().can()
args["requesting_user_id"] = g.current_user.id
args["user_id"] = user_id
return service.render(args)
@validate_schema(user_api_key_input_schema, api_key_output_schema)
@ -230,11 +250,25 @@ class ApiKeyUserList(AuthenticatedResource):
"""
if not ApiKeyCreatorPermission().can():
if user_id != g.current_user.id:
return dict(message="You are not authorized to create tokens for: {0}".format(user_id)), 403
return (
dict(
message="You are not authorized to create tokens for: {0}".format(
user_id
)
),
403,
)
access_token = service.create(name=data['name'], user_id=user_id, ttl=data['ttl'],
revoked=False, issued_at=int(datetime.utcnow().timestamp()))
return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
access_token = service.create(
name=data["name"],
user_id=user_id,
ttl=data["ttl"],
revoked=False,
issued_at=int(datetime.utcnow().timestamp()),
)
return dict(
jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
)
class ApiKeys(AuthenticatedResource):
@ -329,7 +363,9 @@ class ApiKeys(AuthenticatedResource):
if not ApiKeyCreatorPermission().can():
return dict(message="You are not authorized to update this token!"), 403
service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
service.update(
access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
)
return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
def delete(self, aid):
@ -371,7 +407,7 @@ class ApiKeys(AuthenticatedResource):
return dict(message="You are not authorized to delete this token!"), 403
service.delete(access_key)
return {'result': True}
return {"result": True}
class UserApiKeys(AuthenticatedResource):
@ -472,7 +508,9 @@ class UserApiKeys(AuthenticatedResource):
if access_key.user_id != uid:
return dict(message="You are not authorized to update this token!"), 403
service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
service.update(
access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
)
return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
def delete(self, uid, aid):
@ -517,7 +555,7 @@ class UserApiKeys(AuthenticatedResource):
return dict(message="You are not authorized to delete this token!"), 403
service.delete(access_key)
return {'result': True}
return {"result": True}
class ApiKeysDescribed(AuthenticatedResource):
@ -572,8 +610,12 @@ class ApiKeysDescribed(AuthenticatedResource):
return access_key
api.add_resource(ApiKeyList, '/keys', endpoint='api_keys')
api.add_resource(ApiKeys, '/keys/<int:aid>', endpoint='api_key')
api.add_resource(ApiKeysDescribed, '/keys/<int:aid>/described', endpoint='api_key_described')
api.add_resource(ApiKeyUserList, '/users/<int:user_id>/keys', endpoint='user_api_keys')
api.add_resource(UserApiKeys, '/users/<int:uid>/keys/<int:aid>', endpoint='user_api_key')
api.add_resource(ApiKeyList, "/keys", endpoint="api_keys")
api.add_resource(ApiKeys, "/keys/<int:aid>", endpoint="api_key")
api.add_resource(
ApiKeysDescribed, "/keys/<int:aid>/described", endpoint="api_key_described"
)
api.add_resource(ApiKeyUserList, "/users/<int:user_id>/keys", endpoint="user_api_keys")
api.add_resource(
UserApiKeys, "/users/<int:uid>/keys/<int:aid>", endpoint="user_api_key"
)

112
lemur/auth/ldap.py
View File

@ -14,35 +14,41 @@ from lemur.roles import service as role_service
from lemur.common.utils import validate_conf, get_psuedo_random_string
class LdapPrincipal():
class LdapPrincipal:
"""
Provides methods for authenticating against an LDAP server.
"""
def __init__(self, args):
self._ldap_validate_conf()
# setup ldap config
if not args['username']:
if not args["username"]:
raise Exception("missing ldap username")
if not args['password']:
if not args["password"]:
self.error_message = "missing ldap password"
raise Exception("missing ldap password")
self.ldap_principal = args['username']
self.ldap_principal = args["username"]
self.ldap_email_domain = current_app.config.get("LDAP_EMAIL_DOMAIN", None)
if '@' not in self.ldap_principal:
self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
self.ldap_username = args['username']
if '@' in self.ldap_username:
self.ldap_username = args['username'].split("@")[0]
self.ldap_password = args['password']
self.ldap_server = current_app.config.get('LDAP_BIND_URI', None)
if "@" not in self.ldap_principal:
self.ldap_principal = "%s@%s" % (
self.ldap_principal,
self.ldap_email_domain,
)
self.ldap_username = args["username"]
if "@" in self.ldap_username:
self.ldap_username = args["username"].split("@")[0]
self.ldap_password = args["password"]
self.ldap_server = current_app.config.get("LDAP_BIND_URI", None)
self.ldap_base_dn = current_app.config.get("LDAP_BASE_DN", None)
self.ldap_use_tls = current_app.config.get("LDAP_USE_TLS", False)
self.ldap_cacert_file = current_app.config.get("LDAP_CACERT_FILE", None)
self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
self.ldap_is_active_directory = current_app.config.get("LDAP_IS_ACTIVE_DIRECTORY", False)
self.ldap_attrs = ['memberOf']
self.ldap_is_active_directory = current_app.config.get(
"LDAP_IS_ACTIVE_DIRECTORY", False
)
self.ldap_attrs = ["memberOf"]
self.ldap_client = None
self.ldap_groups = None
@ -60,8 +66,8 @@ class LdapPrincipal():
get_psuedo_random_string(),
self.ldap_principal,
True,
'', # thumbnailPhotoUrl
list(roles)
"", # thumbnailPhotoUrl
list(roles),
)
else:
# we add 'lemur' specific roles, so they do not get marked as removed
@ -76,7 +82,7 @@ class LdapPrincipal():
self.ldap_principal,
user.active,
user.profile_picture,
list(roles)
list(roles),
)
return user
@ -99,15 +105,18 @@ class LdapPrincipal():
role = role_service.get_by_name(self.ldap_default_role)
if role:
if not role.third_party:
role = role.set_third_party(role.id, third_party_status=True)
role = role_service.set_third_party(role.id, third_party_status=True)
roles.add(role)
# update their 'roles'
role = role_service.get_by_name(self.ldap_principal)
if not role:
description = "auto generated role based on owner: {0}".format(self.ldap_principal)
role = role_service.create(self.ldap_principal, description=description,
third_party=True)
description = "auto generated role based on owner: {0}".format(
self.ldap_principal
)
role = role_service.create(
self.ldap_principal, description=description, third_party=True
)
if not role.third_party:
role = role_service.set_third_party(role.id, third_party_status=True)
roles.add(role)
@ -118,9 +127,15 @@ class LdapPrincipal():
role = role_service.get_by_name(role_name)
if role:
if ldap_group_name in self.ldap_groups:
current_app.logger.debug("assigning role {0} to ldap user {1}".format(self.ldap_principal, role))
current_app.logger.debug(
"assigning role {0} to ldap user {1}".format(
self.ldap_principal, role
)
)
if not role.third_party:
role = role_service.set_third_party(role.id, third_party_status=True)
role = role_service.set_third_party(
role.id, third_party_status=True
)
roles.add(role)
return roles
@ -132,7 +147,7 @@ class LdapPrincipal():
self._bind()
roles = self._authorize()
if not roles:
raise Exception('ldap authorization failed')
raise Exception("ldap authorization failed")
return self._update_user(roles)
def _bind(self):
@ -141,9 +156,12 @@ class LdapPrincipal():
list groups for a user.
raise an exception on error.
"""
if '@' not in self.ldap_principal:
self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
ldap_filter = 'userPrincipalName=%s' % self.ldap_principal
if "@" not in self.ldap_principal:
self.ldap_principal = "%s@%s" % (
self.ldap_principal,
self.ldap_email_domain,
)
ldap_filter = "userPrincipalName=%s" % self.ldap_principal
# query ldap for auth
try:
@ -159,37 +177,47 @@ class LdapPrincipal():
self.ldap_client.set_option(ldap.OPT_X_TLS_DEMAND, True)
self.ldap_client.set_option(ldap.OPT_DEBUG_LEVEL, 255)
if self.ldap_cacert_file:
self.ldap_client.set_option(ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file)
self.ldap_client.set_option(
ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file
)
self.ldap_client.simple_bind_s(self.ldap_principal, self.ldap_password)
except ldap.INVALID_CREDENTIALS:
self.ldap_client.unbind()
raise Exception('The supplied ldap credentials are invalid')
raise Exception("The supplied ldap credentials are invalid")
except ldap.SERVER_DOWN:
raise Exception('ldap server unavailable')
raise Exception("ldap server unavailable")
except ldap.LDAPError as e:
raise Exception("ldap error: {0}".format(e))
if self.ldap_is_active_directory:
# Lookup user DN, needed to search for group membership
userdn = self.ldap_client.search_s(self.ldap_base_dn,
ldap.SCOPE_SUBTREE, ldap_filter,
['distinguishedName'])[0][1]['distinguishedName'][0]
userdn = userdn.decode('utf-8')
userdn = self.ldap_client.search_s(
self.ldap_base_dn,
ldap.SCOPE_SUBTREE,
ldap_filter,
["distinguishedName"],
)[0][1]["distinguishedName"][0]
userdn = userdn.decode("utf-8")
# Search all groups that have the userDN as a member
groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn)
lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn'])
groupfilter = "(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))".format(
userdn
)
lgroups = self.ldap_client.search_s(
self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ["cn"]
)
# Create a list of group CN's from the result
self.ldap_groups = []
for group in lgroups:
(dn, values) = group
self.ldap_groups.append(values['cn'][0].decode('ascii'))
self.ldap_groups.append(values["cn"][0].decode("ascii"))
else:
lgroups = self.ldap_client.search_s(self.ldap_base_dn,
ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
lgroups = self.ldap_client.search_s(
self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs
)[0][1]["memberOf"]
# lgroups is a list of utf-8 encoded strings
# convert to a single string of groups to allow matching
self.ldap_groups = b''.join(lgroups).decode('ascii')
self.ldap_groups = b"".join(lgroups).decode("ascii")
self.ldap_client.unbind()
@ -197,9 +225,5 @@ class LdapPrincipal():
"""
Confirms required ldap config settings exist.
"""
required_vars = [
'LDAP_BIND_URI',
'LDAP_BASE_DN',
'LDAP_EMAIL_DOMAIN',
]
required_vars = ["LDAP_BIND_URI", "LDAP_BASE_DN", "LDAP_EMAIL_DOMAIN"]
validate_conf(current_app, required_vars)

38
lemur/auth/permissions.py
View File

@ -9,24 +9,32 @@
from functools import partial
from collections import namedtuple
from flask import current_app
from flask_principal import Permission, RoleNeed
# Permissions
operator_permission = Permission(RoleNeed('operator'))
admin_permission = Permission(RoleNeed('admin'))
operator_permission = Permission(RoleNeed("operator"))
admin_permission = Permission(RoleNeed("admin"))
CertificateOwner = namedtuple('certificate', ['method', 'value'])
CertificateOwnerNeed = partial(CertificateOwner, 'role')
CertificateOwner = namedtuple("certificate", ["method", "value"])
CertificateOwnerNeed = partial(CertificateOwner, "role")
class SensitiveDomainPermission(Permission):
def __init__(self):
super(SensitiveDomainPermission, self).__init__(RoleNeed('admin'))
needs = [RoleNeed("admin")]
sensitive_domain_roles = current_app.config.get("SENSITIVE_DOMAIN_ROLES", [])
if sensitive_domain_roles:
for role in sensitive_domain_roles:
needs.append(RoleNeed(role))
super(SensitiveDomainPermission, self).__init__(*needs)
class CertificatePermission(Permission):
def __init__(self, owner, roles):
needs = [RoleNeed('admin'), RoleNeed(owner), RoleNeed('creator')]
needs = [RoleNeed("admin"), RoleNeed(owner), RoleNeed("creator")]
for r in roles:
needs.append(CertificateOwnerNeed(str(r)))
# Backwards compatibility with mixed-case role names
@ -38,29 +46,29 @@ class CertificatePermission(Permission):
class ApiKeyCreatorPermission(Permission):
def __init__(self):
super(ApiKeyCreatorPermission, self).__init__(RoleNeed('admin'))
super(ApiKeyCreatorPermission, self).__init__(RoleNeed("admin"))
RoleMember = namedtuple('role', ['method', 'value'])
RoleMemberNeed = partial(RoleMember, 'member')
RoleMember = namedtuple("role", ["method", "value"])
RoleMemberNeed = partial(RoleMember, "member")
class RoleMemberPermission(Permission):
def __init__(self, role_id):
needs = [RoleNeed('admin'), RoleMemberNeed(role_id)]
needs = [RoleNeed("admin"), RoleMemberNeed(role_id)]
super(RoleMemberPermission, self).__init__(*needs)
AuthorityCreator = namedtuple('authority', ['method', 'value'])
AuthorityCreatorNeed = partial(AuthorityCreator, 'authorityUse')
AuthorityCreator = namedtuple("authority", ["method", "value"])
AuthorityCreatorNeed = partial(AuthorityCreator, "authorityUse")
AuthorityOwner = namedtuple('authority', ['method', 'value'])
AuthorityOwnerNeed = partial(AuthorityOwner, 'role')
AuthorityOwner = namedtuple("authority", ["method", "value"])
AuthorityOwnerNeed = partial(AuthorityOwner, "role")
class AuthorityPermission(Permission):
def __init__(self, authority_id, roles):
needs = [RoleNeed('admin'), AuthorityCreatorNeed(str(authority_id))]
needs = [RoleNeed("admin"), AuthorityCreatorNeed(str(authority_id))]
for r in roles:
needs.append(AuthorityOwnerNeed(str(r)))

85
lemur/auth/service.py
View File

@ -39,13 +39,13 @@ def get_rsa_public_key(n, e):
:param e:
:return: a RSA Public Key in PEM format
"""
n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, 'utf-8'))), 16)
e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, 'utf-8'))), 16)
n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, "utf-8"))), 16)
e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, "utf-8"))), 16)
pub = RSAPublicNumbers(e, n).public_key(default_backend())
return pub.public_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PublicFormat.SubjectPublicKeyInfo
format=serialization.PublicFormat.SubjectPublicKeyInfo,
)
@ -57,28 +57,27 @@ def create_token(user, aid=None, ttl=None):
:param user:
:return:
"""
expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
payload = {
'iat': datetime.utcnow(),
'exp': datetime.utcnow() + expiration_delta
}
expiration_delta = timedelta(
days=int(current_app.config.get("LEMUR_TOKEN_EXPIRATION", 1))
)
payload = {"iat": datetime.utcnow(), "exp": datetime.utcnow() + expiration_delta}
# Handle Just a User ID & User Object.
if isinstance(user, int):
payload['sub'] = user
payload["sub"] = user
else:
payload['sub'] = user.id
payload["sub"] = user.id
if aid is not None:
payload['aid'] = aid
payload["aid"] = aid
# Custom TTLs are only supported on Access Keys.
if ttl is not None and aid is not None:
# Tokens that are forever until revoked.
if ttl == -1:
del payload['exp']
del payload["exp"]
else:
payload['exp'] = ttl
token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
return token.decode('unicode_escape')
payload["exp"] = ttl
token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"])
return token.decode("unicode_escape")
def login_required(f):
@ -88,49 +87,54 @@ def login_required(f):
:param f:
:return:
"""
@wraps(f)
def decorated_function(*args, **kwargs):
if not request.headers.get('Authorization'):
response = jsonify(message='Missing authorization header')
if not request.headers.get("Authorization"):
response = jsonify(message="Missing authorization header")
response.status_code = 401
return response
try:
token = request.headers.get('Authorization').split()[1]
token = request.headers.get("Authorization").split()[1]
except Exception as e:
return dict(message='Token is invalid'), 403
return dict(message="Token is invalid"), 403
try:
payload = jwt.decode(token, current_app.config['LEMUR_TOKEN_SECRET'])
payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"])
except jwt.DecodeError:
return dict(message='Token is invalid'), 403
return dict(message="Token is invalid"), 403
except jwt.ExpiredSignatureError:
return dict(message='Token has expired'), 403
return dict(message="Token has expired"), 403
except jwt.InvalidTokenError:
return dict(message='Token is invalid'), 403
return dict(message="Token is invalid"), 403
if 'aid' in payload:
access_key = api_key_service.get(payload['aid'])
if "aid" in payload:
access_key = api_key_service.get(payload["aid"])
if access_key.revoked:
return dict(message='Token has been revoked'), 403
return dict(message="Token has been revoked"), 403
if access_key.ttl != -1:
current_time = datetime.utcnow()
expired_time = datetime.fromtimestamp(access_key.issued_at + access_key.ttl)
expired_time = datetime.fromtimestamp(
access_key.issued_at + access_key.ttl
)
if current_time >= expired_time:
return dict(message='Token has expired'), 403
return dict(message="Token has expired"), 403
user = user_service.get(payload['sub'])
user = user_service.get(payload["sub"])
if not user.active:
return dict(message='User is not currently active'), 403
return dict(message="User is not currently active"), 403
g.current_user = user
if not g.current_user:
return dict(message='You are not logged in'), 403
return dict(message="You are not logged in"), 403
# Tell Flask-Principal the identity changed
identity_changed.send(current_app._get_current_object(), identity=Identity(g.current_user.id))
identity_changed.send(
current_app._get_current_object(), identity=Identity(g.current_user.id)
)
return f(*args, **kwargs)
@ -144,18 +148,18 @@ def fetch_token_header(token):
:param token:
:return: :raise jwt.DecodeError:
"""
token = token.encode('utf-8')
token = token.encode("utf-8")
try:
signing_input, crypto_segment = token.rsplit(b'.', 1)
header_segment, payload_segment = signing_input.split(b'.', 1)
signing_input, crypto_segment = token.rsplit(b".", 1)
header_segment, payload_segment = signing_input.split(b".", 1)
except ValueError:
raise jwt.DecodeError('Not enough segments')
raise jwt.DecodeError("Not enough segments")
try:
return json.loads(jwt.utils.base64url_decode(header_segment).decode('utf-8'))
return json.loads(jwt.utils.base64url_decode(header_segment).decode("utf-8"))
except TypeError as e:
current_app.logger.exception(e)
raise jwt.DecodeError('Invalid header padding')
raise jwt.DecodeError("Invalid header padding")
@identity_loaded.connect
@ -174,13 +178,13 @@ def on_identity_loaded(sender, identity):
identity.provides.add(UserNeed(identity.id))
# identity with the roles that the user provides
if hasattr(user, 'roles'):
if hasattr(user, "roles"):
for role in user.roles:
identity.provides.add(RoleNeed(role.name))
identity.provides.add(RoleMemberNeed(role.id))
# apply ownership for authorities
if hasattr(user, 'authorities'):
if hasattr(user, "authorities"):
for authority in user.authorities:
identity.provides.add(AuthorityCreatorNeed(authority.id))
@ -191,6 +195,7 @@ class AuthenticatedResource(Resource):
"""
Inherited by all resources that need to be protected by authentication.
"""
method_decorators = [login_required]
def __init__(self):

373
lemur/auth/views.py
View File

@ -24,11 +24,13 @@ from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_
from lemur.auth import ldap
mod = Blueprint('auth', __name__)
mod = Blueprint("auth", __name__)
api = Api(mod)
def exchange_for_access_token(code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True):
def exchange_for_access_token(
code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True
):
"""
Exchanges authorization code for access token.
@ -43,28 +45,32 @@ def exchange_for_access_token(code, redirect_uri, client_id, secret, access_toke
"""
# take the information we have received from the provider to create a new request
params = {
'grant_type': 'authorization_code',
'scope': 'openid email profile address',
'code': code,
'redirect_uri': redirect_uri,
'client_id': client_id
"grant_type": "authorization_code",
"scope": "openid email profile address",
"code": code,
"redirect_uri": redirect_uri,
"client_id": client_id,
}
# the secret and cliendId will be given to you when you signup for the provider
token = '{0}:{1}'.format(client_id, secret)
token = "{0}:{1}".format(client_id, secret)
basic = base64.b64encode(bytes(token, 'utf-8'))
basic = base64.b64encode(bytes(token, "utf-8"))
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'authorization': 'basic {0}'.format(basic.decode('utf-8'))
"Content-Type": "application/x-www-form-urlencoded",
"authorization": "basic {0}".format(basic.decode("utf-8")),
}
# exchange authorization code for access token.
r = requests.post(access_token_url, headers=headers, params=params, verify=verify_cert)
r = requests.post(
access_token_url, headers=headers, params=params, verify=verify_cert
)
if r.status_code == 400:
r = requests.post(access_token_url, headers=headers, data=params, verify=verify_cert)
id_token = r.json()['id_token']
access_token = r.json()['access_token']
r = requests.post(
access_token_url, headers=headers, data=params, verify=verify_cert
)
id_token = r.json()["id_token"]
access_token = r.json()["access_token"]
return id_token, access_token
@ -83,23 +89,25 @@ def validate_id_token(id_token, client_id, jwks_url):
# retrieve the key material as specified by the token header
r = requests.get(jwks_url)
for key in r.json()['keys']:
if key['kid'] == header_data['kid']:
secret = get_rsa_public_key(key['n'], key['e'])
algo = header_data['alg']
for key in r.json()["keys"]:
if key["kid"] == header_data["kid"]:
secret = get_rsa_public_key(key["n"], key["e"])
algo = header_data["alg"]
break
else:
return dict(message='Key not found'), 401
return dict(message="Key not found"), 401
# validate your token based on the key it was signed with
try:
jwt.decode(id_token, secret.decode('utf-8'), algorithms=[algo], audience=client_id)
jwt.decode(
id_token, secret.decode("utf-8"), algorithms=[algo], audience=client_id
)
except jwt.DecodeError:
return dict(message='Token is invalid'), 401
return dict(message="Token is invalid"), 401
except jwt.ExpiredSignatureError:
return dict(message='Token has expired'), 401
return dict(message="Token has expired"), 401
except jwt.InvalidTokenError:
return dict(message='Token is invalid'), 401
return dict(message="Token is invalid"), 401
def retrieve_user(user_api_url, access_token):
@ -110,13 +118,18 @@ def retrieve_user(user_api_url, access_token):
:param access_token:
:return:
"""
user_params = dict(access_token=access_token, schema='profile')
user_params = dict(access_token=access_token, schema="profile")
headers = {}
if current_app.config.get("PING_INCLUDE_BEARER_TOKEN"):
headers = {"Authorization": f"Bearer {access_token}"}
# retrieve information about the current user.
r = requests.get(user_api_url, params=user_params)
r = requests.get(user_api_url, params=user_params, headers=headers)
profile = r.json()
user = user_service.get_by_email(profile['email'])
user = user_service.get_by_email(profile["email"])
return user, profile
@ -129,28 +142,44 @@ def create_user_roles(profile):
roles = []
# update their google 'roles'
for group in profile['googleGroups']:
if "googleGroups" in profile:
for group in profile["googleGroups"]:
role = role_service.get_by_name(group)
if not role:
role = role_service.create(group, description='This is a google group based role created by Lemur', third_party=True)
role = role_service.create(
group,
description="This is a google group based role created by Lemur",
third_party=True,
)
if not role.third_party:
role = role_service.set_third_party(role.id, third_party_status=True)
roles.append(role)
else:
current_app.logger.warning(
"'googleGroups' not sent by identity provider, no specific roles will assigned to the user."
)
role = role_service.get_by_name(profile['email'])
role = role_service.get_by_name(profile["email"])
if not role:
role = role_service.create(profile['email'], description='This is a user specific role', third_party=True)
role = role_service.create(
profile["email"],
description="This is a user specific role",
third_party=True,
)
if not role.third_party:
role = role_service.set_third_party(role.id, third_party_status=True)
roles.append(role)
# every user is an operator (tied to a default role)
if current_app.config.get('LEMUR_DEFAULT_ROLE'):
default = role_service.get_by_name(current_app.config['LEMUR_DEFAULT_ROLE'])
if current_app.config.get("LEMUR_DEFAULT_ROLE"):
default = role_service.get_by_name(current_app.config["LEMUR_DEFAULT_ROLE"])
if not default:
default = role_service.create(current_app.config['LEMUR_DEFAULT_ROLE'], description='This is the default Lemur role.')
default = role_service.create(
current_app.config["LEMUR_DEFAULT_ROLE"],
description="This is the default Lemur role.",
)
if not default.third_party:
role_service.set_third_party(default.id, third_party_status=True)
roles.append(default)
@ -169,12 +198,12 @@ def update_user(user, profile, roles):
# if we get an sso user create them an account
if not user:
user = user_service.create(
profile['email'],
profile["email"],
get_psuedo_random_string(),
profile['email'],
profile["email"],
True,
profile.get('thumbnailPhotoUrl'),
roles
profile.get("thumbnailPhotoUrl"),
roles,
)
else:
@ -186,11 +215,11 @@ def update_user(user, profile, roles):
# update any changes to the user
user_service.update(
user.id,
profile['email'],
profile['email'],
profile["email"],
profile["email"],
True,
profile.get('thumbnailPhotoUrl'), # profile isn't google+ enabled
roles
profile.get("thumbnailPhotoUrl"), # profile isn't google+ enabled
roles,
)
@ -211,6 +240,7 @@ class Login(Resource):
on your uses cases but. It is important to not that there is currently no build in method to revoke a users token \
and force re-authentication.
"""
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(Login, self).__init__()
@ -251,23 +281,26 @@ class Login(Resource):
:statuscode 401: invalid credentials
:statuscode 200: no error
"""
self.reqparse.add_argument('username', type=str, required=True, location='json')
self.reqparse.add_argument('password', type=str, required=True, location='json')
self.reqparse.add_argument("username", type=str, required=True, location="json")
self.reqparse.add_argument("password", type=str, required=True, location="json")
args = self.reqparse.parse_args()
if '@' in args['username']:
user = user_service.get_by_email(args['username'])
if "@" in args["username"]:
user = user_service.get_by_email(args["username"])
else:
user = user_service.get_by_username(args['username'])
user = user_service.get_by_username(args["username"])
# default to local authentication
if user and user.check_password(args['password']) and user.active:
if user and user.check_password(args["password"]) and user.active:
# Tell Flask-Principal the identity changed
identity_changed.send(current_app._get_current_object(),
identity=Identity(user.id))
identity_changed.send(
current_app._get_current_object(), identity=Identity(user.id)
)
metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
metrics.send(
"login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
)
return dict(token=create_token(user))
# try ldap login
@ -277,19 +310,29 @@ class Login(Resource):
user = ldap_principal.authenticate()
if user and user.active:
# Tell Flask-Principal the identity changed
identity_changed.send(current_app._get_current_object(),
identity=Identity(user.id))
metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
identity_changed.send(
current_app._get_current_object(), identity=Identity(user.id)
)
metrics.send(
"login",
"counter",
1,
metric_tags={"status": SUCCESS_METRIC_STATUS},
)
return dict(token=create_token(user))
except Exception as e:
current_app.logger.error("ldap error: {0}".format(e))
ldap_message = 'ldap error: %s' % e
metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
ldap_message = "ldap error: %s" % e
metrics.send(
"login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
)
return dict(message=ldap_message), 403
# if not valid user - no certificates for you
metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
return dict(message='The supplied credentials are invalid'), 403
metrics.send(
"login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
)
return dict(message="The supplied credentials are invalid"), 403
class Ping(Resource):
@ -302,49 +345,59 @@ class Ping(Resource):
provider uses for its callbacks.
2. Add or change the Lemur AngularJS Configuration to point to your new provider
"""
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(Ping, self).__init__()
def get(self):
return 'Redirecting...'
return "Redirecting..."
def post(self):
self.reqparse.add_argument('clientId', type=str, required=True, location='json')
self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
self.reqparse.add_argument('code', type=str, required=True, location='json')
self.reqparse.add_argument("clientId", type=str, required=True, location="json")
self.reqparse.add_argument(
"redirectUri", type=str, required=True, location="json"
)
self.reqparse.add_argument("code", type=str, required=True, location="json")
args = self.reqparse.parse_args()
# you can either discover these dynamically or simply configure them
access_token_url = current_app.config.get('PING_ACCESS_TOKEN_URL')
user_api_url = current_app.config.get('PING_USER_API_URL')
access_token_url = current_app.config.get("PING_ACCESS_TOKEN_URL")
user_api_url = current_app.config.get("PING_USER_API_URL")
secret = current_app.config.get('PING_SECRET')
secret = current_app.config.get("PING_SECRET")
id_token, access_token = exchange_for_access_token(
args['code'],
args['redirectUri'],
args['clientId'],
args["code"],
args["redirectUri"],
args["clientId"],
secret,
access_token_url=access_token_url
access_token_url=access_token_url,
)
jwks_url = current_app.config.get('PING_JWKS_URL')
validate_id_token(id_token, args['clientId'], jwks_url)
jwks_url = current_app.config.get("PING_JWKS_URL")
error_code = validate_id_token(id_token, args["clientId"], jwks_url)
if error_code:
return error_code
user, profile = retrieve_user(user_api_url, access_token)
roles = create_user_roles(profile)
update_user(user, profile, roles)
if not user or not user.active:
metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
return dict(message='The supplied credentials are invalid'), 403
metrics.send(
"login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
)
return dict(message="The supplied credentials are invalid"), 403
# Tell Flask-Principal the identity changed
identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
identity_changed.send(
current_app._get_current_object(), identity=Identity(user.id)
)
metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
metrics.send(
"login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
)
return dict(token=create_token(user))
@ -354,46 +407,56 @@ class OAuth2(Resource):
super(OAuth2, self).__init__()
def get(self):
return 'Redirecting...'
return "Redirecting..."
def post(self):
self.reqparse.add_argument('clientId', type=str, required=True, location='json')
self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
self.reqparse.add_argument('code', type=str, required=True, location='json')
self.reqparse.add_argument("clientId", type=str, required=True, location="json")
self.reqparse.add_argument(
"redirectUri", type=str, required=True, location="json"
)
self.reqparse.add_argument("code", type=str, required=True, location="json")
args = self.reqparse.parse_args()
# you can either discover these dynamically or simply configure them
access_token_url = current_app.config.get('OAUTH2_ACCESS_TOKEN_URL')
user_api_url = current_app.config.get('OAUTH2_USER_API_URL')
verify_cert = current_app.config.get('OAUTH2_VERIFY_CERT')
access_token_url = current_app.config.get("OAUTH2_ACCESS_TOKEN_URL")
user_api_url = current_app.config.get("OAUTH2_USER_API_URL")
verify_cert = current_app.config.get("OAUTH2_VERIFY_CERT")
secret = current_app.config.get('OAUTH2_SECRET')
secret = current_app.config.get("OAUTH2_SECRET")
id_token, access_token = exchange_for_access_token(
args['code'],
args['redirectUri'],
args['clientId'],
args["code"],
args["redirectUri"],
args["clientId"],
secret,
access_token_url=access_token_url,
verify_cert=verify_cert
verify_cert=verify_cert,
)
jwks_url = current_app.config.get('PING_JWKS_URL')
validate_id_token(id_token, args['clientId'], jwks_url)
jwks_url = current_app.config.get("PING_JWKS_URL")
error_code = validate_id_token(id_token, args["clientId"], jwks_url)
if error_code:
return error_code
user, profile = retrieve_user(user_api_url, access_token)
roles = create_user_roles(profile)
update_user(user, profile, roles)
if not user.active:
metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
return dict(message='The supplied credentials are invalid'), 403
metrics.send(
"login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
)
return dict(message="The supplied credentials are invalid"), 403
# Tell Flask-Principal the identity changed
identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
identity_changed.send(
current_app._get_current_object(), identity=Identity(user.id)
)
metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
metrics.send(
"login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
)
return dict(token=create_token(user))
@ -404,44 +467,52 @@ class Google(Resource):
super(Google, self).__init__()
def post(self):
access_token_url = 'https://accounts.google.com/o/oauth2/token'
people_api_url = 'https://www.googleapis.com/plus/v1/people/me/openIdConnect'
access_token_url = "https://accounts.google.com/o/oauth2/token"
people_api_url = "https://www.googleapis.com/plus/v1/people/me/openIdConnect"
self.reqparse.add_argument('clientId', type=str, required=True, location='json')
self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
self.reqparse.add_argument('code', type=str, required=True, location='json')
self.reqparse.add_argument("clientId", type=str, required=True, location="json")
self.reqparse.add_argument(
"redirectUri", type=str, required=True, location="json"
)
self.reqparse.add_argument("code", type=str, required=True, location="json")
args = self.reqparse.parse_args()
# Step 1. Exchange authorization code for access token
payload = {
'client_id': args['clientId'],
'grant_type': 'authorization_code',
'redirect_uri': args['redirectUri'],
'code': args['code'],
'client_secret': current_app.config.get('GOOGLE_SECRET')
"client_id": args["clientId"],
"grant_type": "authorization_code",
"redirect_uri": args["redirectUri"],
"code": args["code"],
"client_secret": current_app.config.get("GOOGLE_SECRET"),
}
r = requests.post(access_token_url, data=payload)
token = r.json()
# Step 2. Retrieve information about the current user
headers = {'Authorization': 'Bearer {0}'.format(token['access_token'])}
headers = {"Authorization": "Bearer {0}".format(token["access_token"])}
r = requests.get(people_api_url, headers=headers)
profile = r.json()
user = user_service.get_by_email(profile['email'])
user = user_service.get_by_email(profile["email"])
if not (user and user.active):
metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
return dict(message='The supplied credentials are invalid.'), 403
metrics.send(
"login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
)
return dict(message="The supplied credentials are invalid."), 403
if user:
metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
metrics.send(
"login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
)
return dict(token=create_token(user))
metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
metrics.send(
"login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
)
class Providers(Resource):
@ -452,47 +523,57 @@ class Providers(Resource):
provider = provider.lower()
if provider == "google":
active_providers.append({
'name': 'google',
'clientId': current_app.config.get("GOOGLE_CLIENT_ID"),
'url': api.url_for(Google)
})
active_providers.append(
{
"name": "google",
"clientId": current_app.config.get("GOOGLE_CLIENT_ID"),
"url": api.url_for(Google),
}
)
elif provider == "ping":
active_providers.append({
'name': current_app.config.get("PING_NAME"),
'url': current_app.config.get('PING_REDIRECT_URI'),
'redirectUri': current_app.config.get("PING_REDIRECT_URI"),
'clientId': current_app.config.get("PING_CLIENT_ID"),
'responseType': 'code',
'scope': ['openid', 'email', 'profile', 'address'],
'scopeDelimiter': ' ',
'authorizationEndpoint': current_app.config.get("PING_AUTH_ENDPOINT"),
'requiredUrlParams': ['scope'],
'type': '2.0'
})
active_providers.append(
{
"name": current_app.config.get("PING_NAME"),
"url": current_app.config.get("PING_REDIRECT_URI"),
"redirectUri": current_app.config.get("PING_REDIRECT_URI"),
"clientId": current_app.config.get("PING_CLIENT_ID"),
"responseType": "code",
"scope": ["openid", "email", "profile", "address"],
"scopeDelimiter": " ",
"authorizationEndpoint": current_app.config.get(
"PING_AUTH_ENDPOINT"
),
"requiredUrlParams": ["scope"],
"type": "2.0",
}
)
elif provider == "oauth2":
active_providers.append({
'name': current_app.config.get("OAUTH2_NAME"),
'url': current_app.config.get('OAUTH2_REDIRECT_URI'),
'redirectUri': current_app.config.get("OAUTH2_REDIRECT_URI"),
'clientId': current_app.config.get("OAUTH2_CLIENT_ID"),
'responseType': 'code',
'scope': ['openid', 'email', 'profile', 'groups'],
'scopeDelimiter': ' ',
'authorizationEndpoint': current_app.config.get("OAUTH2_AUTH_ENDPOINT"),
'requiredUrlParams': ['scope', 'state', 'nonce'],
'state': 'STATE',
'nonce': get_psuedo_random_string(),
'type': '2.0'
})
active_providers.append(
{
"name": current_app.config.get("OAUTH2_NAME"),
"url": current_app.config.get("OAUTH2_REDIRECT_URI"),
"redirectUri": current_app.config.get("OAUTH2_REDIRECT_URI"),
"clientId": current_app.config.get("OAUTH2_CLIENT_ID"),
"responseType": "code",
"scope": ["openid", "email", "profile", "groups"],
"scopeDelimiter": " ",
"authorizationEndpoint": current_app.config.get(
"OAUTH2_AUTH_ENDPOINT"
),
"requiredUrlParams": ["scope", "state", "nonce"],
"state": "STATE",
"nonce": get_psuedo_random_string(),
"type": "2.0",
}
)
return active_providers
api.add_resource(Login, '/auth/login', endpoint='login')
api.add_resource(Ping, '/auth/ping', endpoint='ping')
api.add_resource(Google, '/auth/google', endpoint='google')
api.add_resource(OAuth2, '/auth/oauth2', endpoint='oauth2')
api.add_resource(Providers, '/auth/providers', endpoint='providers')
api.add_resource(Login, "/auth/login", endpoint="login")
api.add_resource(Ping, "/auth/ping", endpoint="ping")
api.add_resource(Google, "/auth/google", endpoint="google")
api.add_resource(OAuth2, "/auth/oauth2", endpoint="oauth2")
api.add_resource(Providers, "/auth/providers", endpoint="providers")

62
lemur/authorities/models.py
View File

@ -7,7 +7,17 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime, PassiveDefault, Boolean
from sqlalchemy import (
Column,
Integer,
String,
Text,
func,
ForeignKey,
DateTime,
PassiveDefault,
Boolean,
)
from sqlalchemy.dialects.postgresql import JSON
from lemur.database import db
@ -16,7 +26,7 @@ from lemur.models import roles_authorities
class Authority(db.Model):
__tablename__ = 'authorities'
__tablename__ = "authorities"
id = Column(Integer, primary_key=True)
owner = Column(String(128), nullable=False)
name = Column(String(128), unique=True)
@ -27,22 +37,44 @@ class Authority(db.Model):
description = Column(Text)
options = Column(JSON)
date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
roles = relationship('Role', secondary=roles_authorities, passive_deletes=True, backref=db.backref('authority'), lazy='dynamic')
user_id = Column(Integer, ForeignKey('users.id'))
authority_certificate = relationship("Certificate", backref='root_authority', uselist=False, foreign_keys='Certificate.root_authority_id')
certificates = relationship("Certificate", backref='authority', foreign_keys='Certificate.authority_id')
roles = relationship(
"Role",
secondary=roles_authorities,
passive_deletes=True,
backref=db.backref("authority"),
lazy="dynamic",
)
user_id = Column(Integer, ForeignKey("users.id"))
authority_certificate = relationship(
"Certificate",
backref="root_authority",
uselist=False,
foreign_keys="Certificate.root_authority_id",
)
certificates = relationship(
"Certificate", backref="authority", foreign_keys="Certificate.authority_id"
)
authority_pending_certificate = relationship("PendingCertificate", backref='root_authority', uselist=False, foreign_keys='PendingCertificate.root_authority_id')
pending_certificates = relationship('PendingCertificate', backref='authority', foreign_keys='PendingCertificate.authority_id')
authority_pending_certificate = relationship(
"PendingCertificate",
backref="root_authority",
uselist=False,
foreign_keys="PendingCertificate.root_authority_id",
)
pending_certificates = relationship(
"PendingCertificate",
backref="authority",
foreign_keys="PendingCertificate.authority_id",
)
def __init__(self, **kwargs):
self.owner = kwargs['owner']
self.roles = kwargs.get('roles', [])
self.name = kwargs.get('name')
self.description = kwargs.get('description')
self.authority_certificate = kwargs['authority_certificate']
self.plugin_name = kwargs['plugin']['slug']
self.options = kwargs.get('options')
self.owner = kwargs["owner"]
self.roles = kwargs.get("roles", [])
self.name = kwargs.get("name")
self.description = kwargs.get("description")
self.authority_certificate = kwargs["authority_certificate"]
self.plugin_name = kwargs["plugin"]["slug"]
self.options = kwargs.get("options")
@property
def plugin(self):

49
lemur/authorities/schemas.py
View File

@ -11,7 +11,13 @@ from marshmallow import fields, validates_schema, pre_load
from marshmallow import validate
from marshmallow.exceptions import ValidationError
from lemur.schemas import PluginInputSchema, PluginOutputSchema, ExtensionSchema, AssociatedAuthoritySchema, AssociatedRoleSchema
from lemur.schemas import (
PluginInputSchema,
PluginOutputSchema,
ExtensionSchema,
AssociatedAuthoritySchema,
AssociatedRoleSchema,
)
from lemur.users.schemas import UserNestedOutputSchema
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.common import validators, missing
@ -30,21 +36,36 @@ class AuthorityInputSchema(LemurInputSchema):
validity_years = fields.Integer()
# certificate body fields
organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
organizational_unit = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
)
organization = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
)
location = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_LOCATION")
)
country = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
)
state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
plugin = fields.Nested(PluginInputSchema)
# signing related options
type = fields.String(validate=validate.OneOf(['root', 'subca']), missing='root')
type = fields.String(validate=validate.OneOf(["root", "subca"]), missing="root")
parent = fields.Nested(AssociatedAuthoritySchema)
signing_algorithm = fields.String(validate=validate.OneOf(['sha256WithRSA', 'sha1WithRSA']), missing='sha256WithRSA')
key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
signing_algorithm = fields.String(
validate=validate.OneOf(["sha256WithRSA", "sha1WithRSA"]),
missing="sha256WithRSA",
)
key_type = fields.String(
validate=validate.OneOf(["RSA2048", "RSA4096"]), missing="RSA2048"
)
key_name = fields.String()
sensitivity = fields.String(validate=validate.OneOf(['medium', 'high']), missing='medium')
sensitivity = fields.String(
validate=validate.OneOf(["medium", "high"]), missing="medium"
)
serial_number = fields.Integer()
first_serial = fields.Integer(missing=1)
@ -58,9 +79,11 @@ class AuthorityInputSchema(LemurInputSchema):
@validates_schema
def validate_subca(self, data):
if data['type'] == 'subca':
if not data.get('parent'):
raise ValidationError("If generating a subca, parent 'authority' must be specified.")
if data["type"] == "subca":
if not data.get("parent"):
raise ValidationError(
"If generating a subca, parent 'authority' must be specified."
)
@pre_load
def ensure_dates(self, data):

75
lemur/authorities/service.py
View File

@ -15,6 +15,7 @@ from lemur import database
from lemur.common.utils import truthiness
from lemur.extensions import metrics
from lemur.authorities.models import Authority
from lemur.certificates.models import Certificate
from lemur.roles import service as role_service
from lemur.certificates.service import upload
@ -42,7 +43,7 @@ def mint(**kwargs):
"""
Creates the authority based on the plugin provided.
"""
issuer = kwargs['plugin']['plugin_object']
issuer = kwargs["plugin"]["plugin_object"]
values = issuer.create_authority(kwargs)
# support older plugins
@ -52,7 +53,12 @@ def mint(**kwargs):
elif len(values) == 4:
body, private_key, chain, roles = values
roles = create_authority_roles(roles, kwargs['owner'], kwargs['plugin']['plugin_object'].title, kwargs['creator'])
roles = create_authority_roles(
roles,
kwargs["owner"],
kwargs["plugin"]["plugin_object"].title,
kwargs["creator"],
)
return body, private_key, chain, roles
@ -65,16 +71,17 @@ def create_authority_roles(roles, owner, plugin_title, creator):
"""
role_objs = []
for r in roles:
role = role_service.get_by_name(r['name'])
role = role_service.get_by_name(r["name"])
if not role:
role = role_service.create(
r['name'],
password=r['password'],
r["name"],
password=r["password"],
description="Auto generated role for {0}".format(plugin_title),
username=r['username'])
username=r["username"],
)
# the user creating the authority should be able to administer it
if role.username == 'admin':
if role.username == "admin":
creator.roles.append(role)
role_objs.append(role)
@ -83,8 +90,7 @@ def create_authority_roles(roles, owner, plugin_title, creator):
owner_role = role_service.get_by_name(owner)
if not owner_role:
owner_role = role_service.create(
owner,
description="Auto generated role based on owner: {0}".format(owner)
owner, description="Auto generated role based on owner: {0}".format(owner)
)
role_objs.append(owner_role)
@ -97,27 +103,29 @@ def create(**kwargs):
"""
body, private_key, chain, roles = mint(**kwargs)
kwargs['creator'].roles = list(set(list(kwargs['creator'].roles) + roles))
kwargs["creator"].roles = list(set(list(kwargs["creator"].roles) + roles))
kwargs['body'] = body
kwargs['private_key'] = private_key
kwargs['chain'] = chain
kwargs["body"] = body
kwargs["private_key"] = private_key
kwargs["chain"] = chain
if kwargs.get('roles'):
kwargs['roles'] += roles
if kwargs.get("roles"):
kwargs["roles"] += roles
else:
kwargs['roles'] = roles
kwargs["roles"] = roles
cert = upload(**kwargs)
kwargs['authority_certificate'] = cert
if kwargs.get('plugin', {}).get('plugin_options', []):
kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
kwargs["authority_certificate"] = cert
if kwargs.get("plugin", {}).get("plugin_options", []):
kwargs["options"] = json.dumps(kwargs["plugin"]["plugin_options"])
authority = Authority(**kwargs)
authority = database.create(authority)
kwargs['creator'].authorities.append(authority)
kwargs["creator"].authorities.append(authority)
metrics.send('authority_created', 'counter', 1, metric_tags=dict(owner=authority.owner))
metrics.send(
"authority_created", "counter", 1, metric_tags=dict(owner=authority.owner)
)
return authority
@ -149,7 +157,7 @@ def get_by_name(authority_name):
:param authority_name:
:return:
"""
return database.get(Authority, authority_name, field='name')
return database.get(Authority, authority_name, field="name")
def get_authority_role(ca_name, creator=None):
@ -172,24 +180,31 @@ def render(args):
:return:
"""
query = database.session_query(Authority)
filt = args.pop('filter')
filt = args.pop("filter")
if filt:
terms = filt.split(';')
if 'active' in filt:
terms = filt.split(";")
if "active" in filt:
query = query.filter(Authority.active == truthiness(terms[1]))
elif 'cn' in filt:
query = query.join(Authority.active == truthiness(terms[1]))
elif "cn" in filt:
term = "%{0}%".format(terms[1])
sub_query = (
database.session_query(Certificate.root_authority_id)
.filter(Certificate.cn.ilike(term))
.subquery()
)
query = query.filter(Authority.id.in_(sub_query))
else:
query = database.filter(query, Authority, terms)
# we make sure that a user can only use an authority they either own are a member of - admins can see all
if not args['user'].is_admin:
if not args["user"].is_admin:
authority_ids = []
for authority in args['user'].authorities:
for authority in args["user"].authorities:
authority_ids.append(authority.id)
for role in args['user'].roles:
for role in args["user"].roles:
for authority in role.authorities:
authority_ids.append(authority.id)
query = query.filter(Authority.id.in_(authority_ids))

45
lemur/authorities/views.py
View File

@ -16,15 +16,21 @@ from lemur.auth.permissions import AuthorityPermission
from lemur.certificates import service as certificate_service
from lemur.authorities import service
from lemur.authorities.schemas import authority_input_schema, authority_output_schema, authorities_output_schema, authority_update_schema
from lemur.authorities.schemas import (
authority_input_schema,
authority_output_schema,
authorities_output_schema,
authority_update_schema,
)
mod = Blueprint('authorities', __name__)
mod = Blueprint("authorities", __name__)
api = Api(mod)
class AuthoritiesList(AuthenticatedResource):
""" Defines the 'authorities' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(AuthoritiesList, self).__init__()
@ -107,7 +113,7 @@ class AuthoritiesList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args['user'] = g.current_user
args["user"] = g.current_user
return service.render(args)
@validate_schema(authority_input_schema, authority_output_schema)
@ -220,7 +226,7 @@ class AuthoritiesList(AuthenticatedResource):
:statuscode 403: unauthenticated
:statuscode 200: no error
"""
data['creator'] = g.current_user
data["creator"] = g.current_user
return service.create(**data)
@ -388,7 +394,7 @@ class Authorities(AuthenticatedResource):
authority = service.get(authority_id)
if not authority:
return dict(message='Not Found'), 404
return dict(message="Not Found"), 404
# all the authority role members should be allowed
roles = [x.name for x in authority.roles]
@ -397,10 +403,10 @@ class Authorities(AuthenticatedResource):
if permission.can():
return service.update(
authority_id,
owner=data['owner'],
description=data['description'],
active=data['active'],
roles=data['roles']
owner=data["owner"],
description=data["description"],
active=data["active"],
roles=data["roles"],
)
return dict(message="You are not authorized to update this authority."), 403
@ -505,10 +511,21 @@ class AuthorityVisualizations(AuthenticatedResource):
]}
"""
authority = service.get(authority_id)
return dict(name=authority.name, children=[{"name": c.name} for c in authority.certificates])
return dict(
name=authority.name,
children=[{"name": c.name} for c in authority.certificates],
)
api.add_resource(AuthoritiesList, '/authorities', endpoint='authorities')
api.add_resource(Authorities, '/authorities/<int:authority_id>', endpoint='authority')
api.add_resource(AuthorityVisualizations, '/authorities/<int:authority_id>/visualize', endpoint='authority_visualizations')
api.add_resource(CertificateAuthority, '/certificates/<int:certificate_id>/authority', endpoint='certificateAuthority')
api.add_resource(AuthoritiesList, "/authorities", endpoint="authorities")
api.add_resource(Authorities, "/authorities/<int:authority_id>", endpoint="authority")
api.add_resource(
AuthorityVisualizations,
"/authorities/<int:authority_id>/visualize",
endpoint="authority_visualizations",
)
api.add_resource(
CertificateAuthority,
"/certificates/<int:certificate_id>/authority",
endpoint="certificateAuthority",
)

2
lemur/authorizations/models.py
View File

@ -13,7 +13,7 @@ from lemur.plugins.base import plugins
class Authorization(db.Model):
__tablename__ = 'pending_dns_authorizations'
__tablename__ = "pending_dns_authorizations"
id = Column(Integer, primary_key=True, autoincrement=True)
account_number = Column(String(128))
domains = Column(JSONType)

248
lemur/certificates/cli.py
View File

@ -34,7 +34,7 @@ from lemur.certificates.service import (
get_all_pending_reissue,
get_by_name,
get_all_certs,
get
get,
)
from lemur.certificates.verify import verify_string
@ -56,11 +56,14 @@ def print_certificate_details(details):
"\t[+] Authority: {authority_name}\n"
"\t[+] Validity Start: {validity_start}\n"
"\t[+] Validity End: {validity_end}\n".format(
common_name=details['commonName'],
sans=",".join(x['value'] for x in details['extensions']['subAltNames']['names']) or None,
authority_name=details['authority']['name'],
validity_start=details['validityStart'],
validity_end=details['validityEnd']
common_name=details["commonName"],
sans=",".join(
x["value"] for x in details["extensions"]["subAltNames"]["names"]
)
or None,
authority_name=details["authority"]["name"],
validity_start=details["validityStart"],
validity_end=details["validityEnd"],
)
)
@ -120,13 +123,11 @@ def request_rotation(endpoint, certificate, message, commit):
except Exception as e:
print(
"[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
endpoint.name,
certificate.name,
e
endpoint.name, certificate.name, e
)
)
metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': status})
metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status})
def request_reissue(certificate, commit):
@ -153,22 +154,53 @@ def request_reissue(certificate, commit):
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error reissuing certificate.", exc_info=True)
print(
"[!] Failed to reissue certificates. Reason: {}".format(
e
sentry.captureException(extra={"certificate_name": str(certificate.name)})
current_app.logger.exception(
f"Error reissuing certificate: {certificate.name}", exc_info=True
)
print(f"[!] Failed to reissue certificate: {certificate.name}. Reason: {e}")
metrics.send(
"certificate_reissue",
"counter",
1,
metric_tags={"status": status, "certificate": certificate.name},
)
metrics.send('certificate_reissue', 'counter', 1, metric_tags={'status': status})
@manager.option('-e', '--endpoint', dest='endpoint_name', help='Name of the endpoint you wish to rotate.')
@manager.option('-n', '--new-certificate', dest='new_certificate_name', help='Name of the certificate you wish to rotate to.')
@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to rotate.')
@manager.option('-a', '--notify', dest='message', action='store_true', help='Send a rotation notification to the certificates owner.')
@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
@manager.option(
"-e",
"--endpoint",
dest="endpoint_name",
help="Name of the endpoint you wish to rotate.",
)
@manager.option(
"-n",
"--new-certificate",
dest="new_certificate_name",
help="Name of the certificate you wish to rotate to.",
)
@manager.option(
"-o",
"--old-certificate",
dest="old_certificate_name",
help="Name of the certificate you wish to rotate.",
)
@manager.option(
"-a",
"--notify",
dest="message",
action="store_true",
help="Send a rotation notification to the certificates owner.",
)
@manager.option(
"-c",
"--commit",
dest="commit",
action="store_true",
default=False,
help="Persist changes.",
)
def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, commit):
"""
Rotates an endpoint and reissues it if it has not already been replaced. If it has
@ -187,39 +219,90 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
request_rotation(endpoint, new_cert, message, commit)
elif old_cert and new_cert:
print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
for endpoint in old_cert.endpoints:
print("[+] Rotating {0}".format(endpoint.name))
print(f"[+] Rotating {endpoint.name}")
request_rotation(endpoint, new_cert, message, commit)
else:
print("[+] Rotating all endpoints that have new certificates available")
for endpoint in endpoint_service.get_all_pending_rotation():
if len(endpoint.certificate.replaced) == 1:
print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
request_rotation(
endpoint, endpoint.certificate.replaced[0], message, commit
)
else:
metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
endpoint.name
))
metrics.send(
"endpoint_rotation",
"counter",
1,
metric_tags={
"status": FAILURE_METRIC_STATUS,
"old_certificate_name": str(old_cert),
"new_certificate_name": str(
endpoint.certificate.replaced[0].name
),
"endpoint_name": str(endpoint.name),
"message": str(message),
},
)
print(
f"[!] Failed to rotate endpoint {endpoint.name} reason: "
"Multiple replacement certificates found."
)
status = SUCCESS_METRIC_STATUS
print("[+] Done!")
except Exception as e:
sentry.captureException()
sentry.captureException(
extra={
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
}
)
metrics.send('endpoint_rotation_job', 'counter', 1, metric_tags={'status': status})
metrics.send(
"endpoint_rotation_job",
"counter",
1,
metric_tags={
"status": status,
"old_certificate_name": str(old_certificate_name),
"new_certificate_name": str(new_certificate_name),
"endpoint_name": str(endpoint_name),
"message": str(message),
"endpoint": str(globals().get("endpoint")),
},
)
@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to reissue.')
@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
@manager.option(
"-o",
"--old-certificate",
dest="old_certificate_name",
help="Name of the certificate you wish to reissue.",
)
@manager.option(
"-c",
"--commit",
dest="commit",
action="store_true",
default=False,
help="Persist changes.",
)
def reissue(old_certificate_name, commit):
"""
Reissues certificate with the same parameters as it was originally issued with.
@ -247,76 +330,94 @@ def reissue(old_certificate_name, commit):
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error reissuing certificate.", exc_info=True)
print(
"[!] Failed to reissue certificates. Reason: {}".format(
e
)
print("[!] Failed to reissue certificates. Reason: {}".format(e))
metrics.send(
"certificate_reissue_job", "counter", 1, metric_tags={"status": status}
)
metrics.send('certificate_reissue_job', 'counter', 1, metric_tags={'status': status})
@manager.option('-f', '--fqdns', dest='fqdns', help='FQDNs to query. Multiple fqdns specified via comma.')
@manager.option('-i', '--issuer', dest='issuer', help='Issuer to query for.')
@manager.option('-o', '--owner', dest='owner', help='Owner to query for.')
@manager.option('-e', '--expired', dest='expired', type=bool, default=False, help='Include expired certificates.')
@manager.option(
"-f",
"--fqdns",
dest="fqdns",
help="FQDNs to query. Multiple fqdns specified via comma.",
)
@manager.option("-i", "--issuer", dest="issuer", help="Issuer to query for.")
@manager.option("-o", "--owner", dest="owner", help="Owner to query for.")
@manager.option(
"-e",
"--expired",
dest="expired",
type=bool,
default=False,
help="Include expired certificates.",
)
def query(fqdns, issuer, owner, expired):
"""Prints certificates that match the query params."""
table = []
q = database.session_query(Certificate)
if issuer:
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike('%{0}%'.format(issuer))) \
sub_query = (
database.session_query(Authority.id)
.filter(Authority.name.ilike("%{0}%".format(issuer)))
.subquery()
)
q = q.filter(
or_(
Certificate.issuer.ilike('%{0}%'.format(issuer)),
Certificate.authority_id.in_(sub_query)
Certificate.issuer.ilike("%{0}%".format(issuer)),
Certificate.authority_id.in_(sub_query),
)
)
if owner:
q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
q = q.filter(Certificate.owner.ilike("%{0}%".format(owner)))
if not expired:
q = q.filter(Certificate.expired == False) # noqa
if fqdns:
for f in fqdns.split(','):
for f in fqdns.split(","):
q = q.filter(
or_(
Certificate.cn.ilike('%{0}%'.format(f)),
Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f)))
Certificate.cn.ilike("%{0}%".format(f)),
Certificate.domains.any(Domain.name.ilike("%{0}%".format(f))),
)
)
for c in q.all():
table.append([c.id, c.name, c.owner, c.issuer])
print(tabulate(table, headers=['Id', 'Name', 'Owner', 'Issuer'], tablefmt='csv'))
print(tabulate(table, headers=["Id", "Name", "Owner", "Issuer"], tablefmt="csv"))
def worker(data, commit, reason):
parts = [x for x in data.split(' ') if x]
parts = [x for x in data.split(" ") if x]
try:
cert = get(int(parts[0].strip()))
plugin = plugins.get(cert.authority.plugin_name)
print('[+] Revoking certificate. Id: {0} Name: {1}'.format(cert.id, cert.name))
print("[+] Revoking certificate. Id: {0} Name: {1}".format(cert.id, cert.name))
if commit:
plugin.revoke_certificate(cert, reason)
metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
metrics.send(
"certificate_revoke",
"counter",
1,
metric_tags={"status": SUCCESS_METRIC_STATUS},
)
except Exception as e:
sentry.captureException()
metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
print(
"[!] Failed to revoke certificates. Reason: {}".format(
e
)
metrics.send(
"certificate_revoke",
"counter",
1,
metric_tags={"status": FAILURE_METRIC_STATUS},
)
print("[!] Failed to revoke certificates. Reason: {}".format(e))
@manager.command
@ -325,13 +426,22 @@ def clear_pending():
Function clears all pending certificates.
:return:
"""
v = plugins.get('verisign-issuer')
v = plugins.get("verisign-issuer")
v.clear_pending_certificates()
@manager.option('-p', '--path', dest='path', help='Absolute file path to a Lemur query csv.')
@manager.option('-r', '--reason', dest='reason', help='Reason to revoke certificate.')
@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
@manager.option(
"-p", "--path", dest="path", help="Absolute file path to a Lemur query csv."
)
@manager.option("-r", "--reason", dest="reason", help="Reason to revoke certificate.")
@manager.option(
"-c",
"--commit",
dest="commit",
action="store_true",
default=False,
help="Persist changes.",
)
def revoke(path, reason, commit):
"""
Revokes given certificate.
@ -341,7 +451,7 @@ def revoke(path, reason, commit):
print("[+] Starting certificate revocation.")
with open(path, 'r') as f:
with open(path, "r") as f:
args = [[x, commit, reason] for x in f.readlines()[2:]]
with multiprocessing.Pool(processes=3) as pool:
@ -364,11 +474,11 @@ def check_revoked():
else:
status = verify_string(cert.body, "")
cert.status = 'valid' if status else 'revoked'
cert.status = "valid" if status else "revoked"
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
cert.status = 'unknown'
cert.status = "unknown"
database.update(cert)

19
lemur/certificates/hooks.py
View File

@ -12,21 +12,30 @@ import subprocess
from flask import current_app
from lemur.certificates.service import csr_created, csr_imported, certificate_issued, certificate_imported
from lemur.certificates.service import (
csr_created,
csr_imported,
certificate_issued,
certificate_imported,
)
def csr_dump_handler(sender, csr, **kwargs):
try:
subprocess.run(['openssl', 'req', '-text', '-noout', '-reqopt', 'no_sigdump,no_pubkey'],
input=csr.encode('utf8'))
subprocess.run(
["openssl", "req", "-text", "-noout", "-reqopt", "no_sigdump,no_pubkey"],
input=csr.encode("utf8"),
)
except Exception as err:
current_app.logger.warning("Error inspecting CSR: %s", err)
def cert_dump_handler(sender, certificate, **kwargs):
try:
subprocess.run(['openssl', 'x509', '-text', '-noout', '-certopt', 'no_sigdump,no_pubkey'],
input=certificate.body.encode('utf8'))
subprocess.run(
["openssl", "x509", "-text", "-noout", "-certopt", "no_sigdump,no_pubkey"],
input=certificate.body.encode("utf8"),
)
except Exception as err:
current_app.logger.warning("Error inspecting certificate: %s", err)

300
lemur/certificates/models.py
View File

@ -12,32 +12,49 @@ from cryptography import x509
from cryptography.hazmat.primitives.asymmetric import rsa
from flask import current_app
from idna.core import InvalidCodepoint
from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index
from sqlalchemy import (
event,
Integer,
ForeignKey,
String,
PassiveDefault,
func,
Column,
Text,
Boolean,
Index,
)
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
from sqlalchemy.sql.expression import case, extract
from sqlalchemy_utils.types.arrow import ArrowType
from werkzeug.utils import cached_property
from lemur.common import defaults, utils
from lemur.common import defaults, utils, validators
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.database import db
from lemur.domains.models import Domain
from lemur.extensions import metrics
from lemur.extensions import sentry
from lemur.models import certificate_associations, certificate_source_associations, \
certificate_destination_associations, certificate_notification_associations, \
certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
from lemur.models import (
certificate_associations,
certificate_source_associations,
certificate_destination_associations,
certificate_notification_associations,
certificate_replacement_associations,
roles_certificates,
pending_cert_replacement_associations,
)
from lemur.plugins.base import plugins
from lemur.policies.models import RotationPolicy
from lemur.utils import Vault
def get_sequence(name):
if '-' not in name:
if "-" not in name:
return name, None
parts = name.split('-')
parts = name.split("-")
# see if we have an int at the end of our name
try:
@ -49,22 +66,26 @@ def get_sequence(name):
if len(parts[-1]) == 8:
return name, None
root = '-'.join(parts[:-1])
root = "-".join(parts[:-1])
return root, seq
def get_or_increase_name(name, serial):
certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).all()
certificates = Certificate.query.filter(Certificate.name == name).all()
if not certificates:
return name
serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper())
certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(serial_name))).all()
serial_name = "{0}-{1}".format(name, hex(int(serial))[2:].upper())
certificates = Certificate.query.filter(Certificate.name == serial_name).all()
if not certificates:
return serial_name
certificates = Certificate.query.filter(
Certificate.name.ilike("{0}%".format(serial_name))
).all()
ends = [0]
root, end = get_sequence(serial_name)
for cert in certificates:
@ -72,21 +93,29 @@ def get_or_increase_name(name, serial):
if end:
ends.append(end)
return '{0}-{1}'.format(root, max(ends) + 1)
return "{0}-{1}".format(root, max(ends) + 1)
class Certificate(db.Model):
__tablename__ = 'certificates'
__tablename__ = "certificates"
__table_args__ = (
Index('ix_certificates_cn', "cn",
Index(
"ix_certificates_cn",
"cn",
postgresql_ops={"cn": "gin_trgm_ops"},
postgresql_using='gin'),
Index('ix_certificates_name', "name",
postgresql_using="gin",
),
Index(
"ix_certificates_name",
"name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
postgresql_using="gin",
),
)
id = Column(Integer, primary_key=True)
ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
ix = Index(
"ix_certificates_id_desc", id.desc(), postgresql_using="btree", unique=True
)
external_id = Column(String(128))
owner = Column(String(128), nullable=False)
name = Column(String(256), unique=True)
@ -101,11 +130,15 @@ class Certificate(db.Model):
issuer = Column(String(128))
serial = Column(String(128))
cn = Column(String(128))
deleted = Column(Boolean, index=True)
dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='CASCADE'), nullable=True)
deleted = Column(Boolean, index=True, default=False)
dns_provider_id = Column(
Integer(), ForeignKey("dns_providers.id", ondelete="CASCADE"), nullable=True
)
not_before = Column(ArrowType)
not_after = Column(ArrowType)
not_after_ix = Index("ix_certificates_not_after", not_after.desc())
date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
signing_algorithm = Column(String(128))
@ -114,34 +147,53 @@ class Certificate(db.Model):
san = Column(String(1024)) # TODO this should be migrated to boolean
rotation = Column(Boolean, default=False)
user_id = Column(Integer, ForeignKey('users.id'))
authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
user_id = Column(Integer, ForeignKey("users.id"))
authority_id = Column(Integer, ForeignKey("authorities.id", ondelete="CASCADE"))
root_authority_id = Column(
Integer, ForeignKey("authorities.id", ondelete="CASCADE")
)
rotation_policy_id = Column(Integer, ForeignKey("rotation_policies.id"))
notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate')
destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate')
sources = relationship('Source', secondary=certificate_source_associations, backref='certificate')
domains = relationship('Domain', secondary=certificate_associations, backref='certificate')
roles = relationship('Role', secondary=roles_certificates, backref='certificate')
replaces = relationship('Certificate',
notifications = relationship(
"Notification",
secondary=certificate_notification_associations,
backref="certificate",
)
destinations = relationship(
"Destination",
secondary=certificate_destination_associations,
backref="certificate",
)
sources = relationship(
"Source", secondary=certificate_source_associations, backref="certificate"
)
domains = relationship(
"Domain", secondary=certificate_associations, backref="certificate"
)
roles = relationship("Role", secondary=roles_certificates, backref="certificate")
replaces = relationship(
"Certificate",
secondary=certificate_replacement_associations,
primaryjoin=id == certificate_replacement_associations.c.certificate_id, # noqa
secondaryjoin=id == certificate_replacement_associations.c.replaced_certificate_id, # noqa
backref='replaced')
secondaryjoin=id
== certificate_replacement_associations.c.replaced_certificate_id, # noqa
backref="replaced",
)
replaced_by_pending = relationship('PendingCertificate',
replaced_by_pending = relationship(
"PendingCertificate",
secondary=pending_cert_replacement_associations,
backref='pending_replace',
viewonly=True)
backref="pending_replace",
viewonly=True,
)
logs = relationship('Log', backref='certificate')
endpoints = relationship('Endpoint', backref='certificate')
logs = relationship("Log", backref="certificate")
endpoints = relationship("Endpoint", backref="certificate")
rotation_policy = relationship("RotationPolicy")
sensitive_fields = ('private_key',)
sensitive_fields = ("private_key",)
def __init__(self, **kwargs):
self.body = kwargs['body'].strip()
self.body = kwargs["body"].strip()
cert = self.parsed_cert
self.issuer = defaults.issuer(cert)
@ -152,40 +204,65 @@ class Certificate(db.Model):
self.serial = defaults.serial(cert)
# when destinations are appended they require a valid name.
if kwargs.get('name'):
self.name = get_or_increase_name(defaults.text_to_slug(kwargs['name']), self.serial)
if kwargs.get("name"):
self.name = get_or_increase_name(
defaults.text_to_slug(kwargs["name"]), self.serial
)
else:
self.name = get_or_increase_name(
defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san), self.serial)
defaults.certificate_name(
self.cn, self.issuer, self.not_before, self.not_after, self.san
),
self.serial,
)
self.owner = kwargs['owner']
self.owner = kwargs["owner"]
if kwargs.get('private_key'):
self.private_key = kwargs['private_key'].strip()
if kwargs.get("private_key"):
self.private_key = kwargs["private_key"].strip()
if kwargs.get('chain'):
self.chain = kwargs['chain'].strip()
if kwargs.get("chain"):
self.chain = kwargs["chain"].strip()
if kwargs.get('csr'):
self.csr = kwargs['csr'].strip()
if kwargs.get("csr"):
self.csr = kwargs["csr"].strip()
self.notify = kwargs.get('notify', True)
self.destinations = kwargs.get('destinations', [])
self.notifications = kwargs.get('notifications', [])
self.description = kwargs.get('description')
self.roles = list(set(kwargs.get('roles', [])))
self.replaces = kwargs.get('replaces', [])
self.rotation = kwargs.get('rotation')
self.rotation_policy = kwargs.get('rotation_policy')
self.notify = kwargs.get("notify", True)
self.destinations = kwargs.get("destinations", [])
self.notifications = kwargs.get("notifications", [])
self.description = kwargs.get("description")
self.roles = list(set(kwargs.get("roles", [])))
self.replaces = kwargs.get("replaces", [])
self.rotation = kwargs.get("rotation")
self.rotation_policy = kwargs.get("rotation_policy")
self.signing_algorithm = defaults.signing_algorithm(cert)
self.bits = defaults.bitstrength(cert)
self.external_id = kwargs.get('external_id')
self.authority_id = kwargs.get('authority_id')
self.dns_provider_id = kwargs.get('dns_provider_id')
self.external_id = kwargs.get("external_id")
self.authority_id = kwargs.get("authority_id")
self.dns_provider_id = kwargs.get("dns_provider_id")
for domain in defaults.domains(cert):
self.domains.append(Domain(name=domain))
# Check integrity before saving anything into the database.
# For user-facing API calls, validation should also be done in schema validators.
self.check_integrity()
def check_integrity(self):
"""
Integrity checks: Does the cert have a valid chain and matching private key?
"""
if self.private_key:
validators.verify_private_key_match(
utils.parse_private_key(self.private_key),
self.parsed_cert,
error_class=AssertionError,
)
if self.chain:
chain = [self.parsed_cert] + utils.parse_cert_chain(self.chain)
validators.verify_cert_chain(chain, error_class=AssertionError)
@cached_property
def parsed_cert(self):
assert self.body, "Certificate body not set"
@ -215,10 +292,16 @@ class Certificate(db.Model):
def location(self):
return defaults.location(self.parsed_cert)
@property
def distinguished_name(self):
return self.parsed_cert.subject.rfc4514_string()
@property
def key_type(self):
if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey):
return 'RSA{key_size}'.format(key_size=self.parsed_cert.public_key().key_size)
return "RSA{key_size}".format(
key_size=self.parsed_cert.public_key().key_size
)
@property
def validity_remaining(self):
@ -243,26 +326,24 @@ class Certificate(db.Model):
@expired.expression
def expired(cls):
return case(
[
(cls.not_after <= arrow.utcnow(), True)
],
else_=False
)
return case([(cls.not_after <= arrow.utcnow(), True)], else_=False)
@hybrid_property
def revoked(self):
if 'revoked' == self.status:
if "revoked" == self.status:
return True
@revoked.expression
def revoked(cls):
return case(
[
(cls.status == 'revoked', True)
],
else_=False
)
return case([(cls.status == "revoked", True)], else_=False)
@hybrid_property
def has_private_key(self):
return self.private_key is not None
@has_private_key.expression
def has_private_key(cls):
return case([(cls.private_key.is_(None), True)], else_=False)
@hybrid_property
def in_rotation_window(self):
@ -285,66 +366,65 @@ class Certificate(db.Model):
:return:
"""
return case(
[
(extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True)
],
else_=False
[(extract("day", cls.not_after - func.now()) <= RotationPolicy.days, True)],
else_=False,
)
@property
def extensions(self):
# setup default values
return_extensions = {
'sub_alt_names': {'names': []}
}
return_extensions = {"sub_alt_names": {"names": []}}
try:
for extension in self.parsed_cert.extensions:
value = extension.value
if isinstance(value, x509.BasicConstraints):
return_extensions['basic_constraints'] = value
return_extensions["basic_constraints"] = value
elif isinstance(value, x509.SubjectAlternativeName):
return_extensions['sub_alt_names']['names'] = value
return_extensions["sub_alt_names"]["names"] = value
elif isinstance(value, x509.ExtendedKeyUsage):
return_extensions['extended_key_usage'] = value
return_extensions["extended_key_usage"] = value
elif isinstance(value, x509.KeyUsage):
return_extensions['key_usage'] = value
return_extensions["key_usage"] = value
elif isinstance(value, x509.SubjectKeyIdentifier):
return_extensions['subject_key_identifier'] = {'include_ski': True}
return_extensions["subject_key_identifier"] = {"include_ski": True}
elif isinstance(value, x509.AuthorityInformationAccess):
return_extensions['certificate_info_access'] = {'include_aia': True}
return_extensions["certificate_info_access"] = {"include_aia": True}
elif isinstance(value, x509.AuthorityKeyIdentifier):
aki = {
'use_key_identifier': False,
'use_authority_cert': False
}
aki = {"use_key_identifier": False, "use_authority_cert": False}
if value.key_identifier:
aki['use_key_identifier'] = True
aki["use_key_identifier"] = True
if value.authority_cert_issuer:
aki['use_authority_cert'] = True
aki["use_authority_cert"] = True
return_extensions['authority_key_identifier'] = aki
return_extensions["authority_key_identifier"] = aki
elif isinstance(value, x509.CRLDistributionPoints):
return_extensions['crl_distribution_points'] = {'include_crl_dp': value}
return_extensions["crl_distribution_points"] = {
"include_crl_dp": value
}
# TODO: Not supporting custom OIDs yet. https://github.com/Netflix/lemur/issues/665
else:
current_app.logger.warning('Custom OIDs not yet supported for clone operation.')
current_app.logger.warning(
"Custom OIDs not yet supported for clone operation."
)
except InvalidCodepoint as e:
sentry.captureException()
current_app.logger.warning('Unable to parse extensions due to underscore in dns name')
current_app.logger.warning(
"Unable to parse extensions due to underscore in dns name"
)
except ValueError as e:
sentry.captureException()
current_app.logger.warning('Unable to parse')
current_app.logger.warning("Unable to parse")
current_app.logger.exception(e)
return return_extensions
@ -353,7 +433,7 @@ class Certificate(db.Model):
return "Certificate(name={name})".format(name=self.name)
@event.listens_for(Certificate.destinations, 'append')
@event.listens_for(Certificate.destinations, "append")
def update_destinations(target, value, initiator):
"""
Attempt to upload certificate to the new destination
@ -367,17 +447,31 @@ def update_destinations(target, value, initiator):
status = FAILURE_METRIC_STATUS
try:
if target.private_key or not destination_plugin.requires_key:
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
destination_plugin.upload(
target.name,
target.body,
target.private_key,
target.chain,
value.options,
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
raise
metrics.send('destination_upload', 'counter', 1,
metric_tags={'status': status, 'certificate': target.name, 'destination': value.label})
metrics.send(
"destination_upload",
"counter",
1,
metric_tags={
"status": status,
"certificate": target.name,
"destination": value.label,
},
)
@event.listens_for(Certificate.replaces, 'append')
@event.listens_for(Certificate.replaces, "append")
def update_replacement(target, value, initiator):
"""
When a certificate is marked as 'replaced' we should not notify.

213
lemur/certificates/schemas.py
View File

@ -6,11 +6,14 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
from flask_restful import inputs
from flask_restful.reqparse import RequestParser
from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.common import validators, missing
from lemur.certificates import utils as cert_utils
from lemur.common import missing, utils, validators
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.constants import CERTIFICATE_KEY_TYPES
@ -38,22 +41,26 @@ from lemur.users.schemas import UserNestedOutputSchema
class CertificateSchema(LemurInputSchema):
owner = fields.Email(required=True)
description = fields.String(missing='', allow_none=True)
description = fields.String(missing="", allow_none=True)
class CertificateCreationSchema(CertificateSchema):
@post_load
def default_notification(self, data):
if not data['notifications']:
data['notifications'] += notification_service.create_default_expiration_notifications(
"DEFAULT_{0}".format(data['owner'].split('@')[0].upper()),
[data['owner']],
if not data["notifications"]:
data[
"notifications"
] += notification_service.create_default_expiration_notifications(
"DEFAULT_{0}".format(data["owner"].split("@")[0].upper()),
[data["owner"]],
)
data['notifications'] += notification_service.create_default_expiration_notifications(
'DEFAULT_SECURITY',
current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'),
current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL_INTERVALS', None)
data[
"notifications"
] += notification_service.create_default_expiration_notifications(
"DEFAULT_SECURITY",
current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL"),
current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL_INTERVALS", None),
)
return data
@ -70,34 +77,56 @@ class CertificateInputSchema(CertificateCreationSchema):
destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True) # deprecated
replacements = fields.Nested(
AssociatedCertificateSchema, missing=[], many=True
) # deprecated
roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
dns_provider = fields.Nested(AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False)
dns_provider = fields.Nested(
AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False
)
csr = fields.String(allow_none=True, validate=validators.csr)
key_type = fields.String(
validate=validate.OneOf(CERTIFICATE_KEY_TYPES),
missing='RSA2048')
validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="RSA2048"
)
notify = fields.Boolean(default=True)
rotation = fields.Boolean()
rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, allow_none=True,
default={'name': 'default'})
rotation_policy = fields.Nested(
AssociatedRotationPolicySchema,
missing={"name": "default"},
allow_none=True,
default={"name": "default"},
)
# certificate body fields
organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
organizational_unit = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
)
organization = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
)
location = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_LOCATION")
)
country = fields.String(
missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
)
state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
extensions = fields.Nested(ExtensionSchema)
@validates_schema
def validate_authority(self, data):
if not data['authority'].active:
raise ValidationError("The authority is inactive.", ['authority'])
if 'authority' not in data:
raise ValidationError("Missing Authority.")
if isinstance(data["authority"], str):
raise ValidationError("Authority not found.")
if not data["authority"].active:
raise ValidationError("The authority is inactive.", ["authority"])
@validates_schema
def validate_dates(self, data):
@ -105,8 +134,19 @@ class CertificateInputSchema(CertificateCreationSchema):
@pre_load
def load_data(self, data):
if data.get('replacements'):
data['replaces'] = data['replacements'] # TODO remove when field is deprecated
if data.get("replacements"):
data["replaces"] = data[
"replacements"
] # TODO remove when field is deprecated
if data.get("csr"):
csr_sans = cert_utils.get_sans_from_csr(data["csr"])
if not data.get("extensions"):
data["extensions"] = {"subAltNames": {"names": []}}
elif not data["extensions"].get("subAltNames"):
data["extensions"]["subAltNames"] = {"names": []}
elif not data["extensions"]["subAltNames"].get("names"):
data["extensions"]["subAltNames"]["names"] = []
data["extensions"]["subAltNames"]["names"] += csr_sans
return missing.convert_validity_years(data)
@ -119,13 +159,17 @@ class CertificateEditInputSchema(CertificateSchema):
destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True) # deprecated
replacements = fields.Nested(
AssociatedCertificateSchema, missing=[], many=True
) # deprecated
roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
@pre_load
def load_data(self, data):
if data.get('replacements'):
data['replaces'] = data['replacements'] # TODO remove when field is deprecated
if data.get("replacements"):
data["replaces"] = data[
"replacements"
] # TODO remove when field is deprecated
return data
@post_load
@ -136,10 +180,15 @@ class CertificateEditInputSchema(CertificateSchema):
:param data:
:return:
"""
if data['owner']:
notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
data['notifications'] += notification_service.create_default_expiration_notifications(notification_name,
[data['owner']])
if data["owner"]:
notification_name = "DEFAULT_{0}".format(
data["owner"].split("@")[0].upper()
)
data[
"notifications"
] += notification_service.create_default_expiration_notifications(
notification_name, [data["owner"]]
)
return data
@ -165,13 +214,13 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
# Note aliasing is the first step in deprecating these fields.
cn = fields.String() # deprecated
common_name = fields.String(attribute='cn')
common_name = fields.String(attribute="cn")
not_after = fields.DateTime() # deprecated
validity_end = ArrowDateTime(attribute='not_after')
validity_end = ArrowDateTime(attribute="not_after")
not_before = fields.DateTime() # deprecated
validity_start = ArrowDateTime(attribute='not_before')
validity_start = ArrowDateTime(attribute="not_before")
issuer = fields.Nested(AuthorityNestedOutputSchema)
@ -202,21 +251,23 @@ class CertificateOutputSchema(LemurOutputSchema):
# Note aliasing is the first step in deprecating these fields.
notify = fields.Boolean()
active = fields.Boolean(attribute='notify')
active = fields.Boolean(attribute="notify")
has_private_key = fields.Boolean()
cn = fields.String()
common_name = fields.String(attribute='cn')
common_name = fields.String(attribute="cn")
distinguished_name = fields.String()
not_after = fields.DateTime()
validity_end = ArrowDateTime(attribute='not_after')
validity_end = ArrowDateTime(attribute="not_after")
not_before = fields.DateTime()
validity_start = ArrowDateTime(attribute='not_before')
validity_start = ArrowDateTime(attribute="not_before")
owner = fields.Email()
san = fields.Boolean()
serial = fields.String()
serial_hex = Hex(attribute='serial')
serial_hex = Hex(attribute="serial")
signing_algorithm = fields.String()
status = fields.String()
@ -233,19 +284,31 @@ class CertificateOutputSchema(LemurOutputSchema):
dns_provider = fields.Nested(DnsProvidersNestedOutputSchema)
roles = fields.Nested(RoleNestedOutputSchema, many=True)
endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
replaced_by = fields.Nested(
CertificateNestedOutputSchema, many=True, attribute="replaced"
)
rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
class CertificateShortOutputSchema(LemurOutputSchema):
id = fields.Integer()
name = fields.String()
owner = fields.Email()
notify = fields.Boolean()
authority = fields.Nested(AuthorityNestedOutputSchema)
issuer = fields.String()
cn = fields.String()
class CertificateUploadInputSchema(CertificateCreationSchema):
name = fields.String()
authority = fields.Nested(AssociatedAuthoritySchema, required=False)
notify = fields.Boolean(missing=True)
external_id = fields.String(missing=None, allow_none=True)
private_key = fields.String(validate=validators.private_key)
body = fields.String(required=True, validate=validators.public_certificate)
chain = fields.String(validate=validators.public_certificate, missing=None,
allow_none=True) # TODO this could be multiple certificates
private_key = fields.String()
body = fields.String(required=True)
chain = fields.String(missing=None, allow_none=True)
csr = fields.String(required=False, allow_none=True, validate=validators.csr)
destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
@ -254,9 +317,44 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
@validates_schema
def keys(self, data):
if data.get('destinations'):
if not data.get('private_key'):
raise ValidationError('Destinations require private key.')
if data.get("destinations"):
if not data.get("private_key"):
raise ValidationError("Destinations require private key.")
@validates_schema
def validate_cert_private_key_chain(self, data):
cert = None
key = None
if data.get("body"):
try:
cert = utils.parse_certificate(data["body"])
except ValueError:
raise ValidationError(
"Public certificate presented is not valid.", field_names=["body"]
)
if data.get("private_key"):
try:
key = utils.parse_private_key(data["private_key"])
except ValueError:
raise ValidationError(
"Private key presented is not valid.", field_names=["private_key"]
)
if cert and key:
# Throws ValidationError
validators.verify_private_key_match(key, cert)
if data.get("chain"):
try:
chain = utils.parse_cert_chain(data["chain"])
except ValueError:
raise ValidationError(
"Invalid certificate in certificate chain.", field_names=["chain"]
)
# Throws ValidationError
validators.verify_cert_chain([cert] + chain)
class CertificateExportInputSchema(LemurInputSchema):
@ -269,8 +367,10 @@ class CertificateNotificationOutputSchema(LemurOutputSchema):
name = fields.String()
owner = fields.Email()
user = fields.Nested(UserNestedOutputSchema)
validity_end = ArrowDateTime(attribute='not_after')
replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
validity_end = ArrowDateTime(attribute="not_after")
replaced_by = fields.Nested(
CertificateNestedOutputSchema, many=True, attribute="replaced"
)
endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
@ -278,9 +378,22 @@ class CertificateRevokeSchema(LemurInputSchema):
comments = fields.String()
certificates_list_request_parser = RequestParser()
certificates_list_request_parser.add_argument("short", type=inputs.boolean, default=False, location="args")
def certificates_list_output_schema_factory():
args = certificates_list_request_parser.parse_args()
if args["short"]:
return certificates_short_output_schema
else:
return certificates_output_schema
certificate_input_schema = CertificateInputSchema()
certificate_output_schema = CertificateOutputSchema()
certificates_output_schema = CertificateOutputSchema(many=True)
certificates_short_output_schema = CertificateShortOutputSchema(many=True)
certificate_upload_input_schema = CertificateUploadInputSchema()
certificate_export_input_schema = CertificateExportInputSchema()
certificate_edit_input_schema = CertificateEditInputSchema()

381
lemur/certificates/service.py
View File

@ -20,17 +20,20 @@ from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.extensions import metrics, sentry, signals
from lemur.models import certificate_associations
from lemur.notifications.models import Notification
from lemur.pending_certificates.models import PendingCertificate
from lemur.plugins.base import plugins
from lemur.roles import service as role_service
from lemur.roles.models import Role
csr_created = signals.signal('csr_created', "CSR generated")
csr_imported = signals.signal('csr_imported', "CSR imported from external source")
certificate_issued = signals.signal('certificate_issued', "Authority issued a certificate")
certificate_imported = signals.signal('certificate_imported', "Certificate imported from external source")
csr_created = signals.signal("csr_created", "CSR generated")
csr_imported = signals.signal("csr_imported", "CSR imported from external source")
certificate_issued = signals.signal(
"certificate_issued", "Authority issued a certificate"
)
certificate_imported = signals.signal(
"certificate_imported", "Certificate imported from external source"
)
def get(cert_id):
@ -50,12 +53,12 @@ def get_by_name(name):
:param name:
:return:
"""
return database.get(Certificate, name, field='name')
return database.get(Certificate, name, field="name")
def get_by_serial(serial):
"""
Retrieves certificate by it's Serial.
Retrieves certificate(s) by serial number.
:param serial:
:return:
"""
@ -65,6 +68,22 @@ def get_by_serial(serial):
return Certificate.query.filter(Certificate.serial == serial).all()
def get_by_attributes(conditions):
"""
Retrieves certificate(s) by conditions given in a hash of given key=>value pairs.
:param serial:
:return:
"""
# Ensure that each of the given conditions corresponds to actual columns
# if not, silently remove it
for attr in conditions.keys():
if attr not in Certificate.__table__.columns:
conditions.pop(attr)
query = database.session_query(Certificate)
return database.find_all(query, Certificate, conditions).all()
def delete(cert_id):
"""
Delete's a certificate.
@ -90,8 +109,12 @@ def get_all_pending_cleaning(source):
:param source:
:return:
"""
return Certificate.query.filter(Certificate.sources.any(id=source.id)) \
.filter(not_(Certificate.endpoints.any())).filter(Certificate.expired).all()
return (
Certificate.query.filter(Certificate.sources.any(id=source.id))
.filter(not_(Certificate.endpoints.any()))
.filter(Certificate.expired)
.all()
)
def get_all_pending_reissue():
@ -104,9 +127,12 @@ def get_all_pending_reissue():
:return:
"""
return Certificate.query.filter(Certificate.rotation == True) \
.filter(not_(Certificate.replaced.any())) \
.filter(Certificate.in_rotation_window == True).all() # noqa
return (
Certificate.query.filter(Certificate.rotation == True)
.filter(not_(Certificate.replaced.any()))
.filter(Certificate.in_rotation_window == True)
.all()
) # noqa
def find_duplicates(cert):
@ -118,10 +144,12 @@ def find_duplicates(cert):
:param cert:
:return:
"""
if cert['chain']:
return Certificate.query.filter_by(body=cert['body'].strip(), chain=cert['chain'].strip()).all()
if cert["chain"]:
return Certificate.query.filter_by(
body=cert["body"].strip(), chain=cert["chain"].strip()
).all()
else:
return Certificate.query.filter_by(body=cert['body'].strip(), chain=None).all()
return Certificate.query.filter_by(body=cert["body"].strip(), chain=None).all()
def export(cert, export_plugin):
@ -133,8 +161,10 @@ def export(cert, export_plugin):
:param cert:
:return:
"""
plugin = plugins.get(export_plugin['slug'])
return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
plugin = plugins.get(export_plugin["slug"])
return plugin.export(
cert.body, cert.chain, cert.private_key, export_plugin["pluginOptions"]
)
def update(cert_id, **kwargs):
@ -153,17 +183,19 @@ def update(cert_id, **kwargs):
def create_certificate_roles(**kwargs):
# create an role for the owner and assign it
owner_role = role_service.get_by_name(kwargs['owner'])
owner_role = role_service.get_by_name(kwargs["owner"])
if not owner_role:
owner_role = role_service.create(
kwargs['owner'],
description="Auto generated role based on owner: {0}".format(kwargs['owner'])
kwargs["owner"],
description="Auto generated role based on owner: {0}".format(
kwargs["owner"]
),
)
# ensure that the authority's owner is also associated with the certificate
if kwargs.get('authority'):
authority_owner_role = role_service.get_by_name(kwargs['authority'].owner)
if kwargs.get("authority"):
authority_owner_role = role_service.get_by_name(kwargs["authority"].owner)
return [owner_role, authority_owner_role]
return [owner_role]
@ -175,16 +207,16 @@ def mint(**kwargs):
Support for multiple authorities is handled by individual plugins.
"""
authority = kwargs['authority']
authority = kwargs["authority"]
issuer = plugins.get(authority.plugin_name)
# allow the CSR to be specified by the user
if not kwargs.get('csr'):
if not kwargs.get("csr"):
csr, private_key = create_csr(**kwargs)
csr_created.send(authority=authority, csr=csr)
else:
csr = str(kwargs.get('csr'))
csr = str(kwargs.get("csr"))
private_key = None
csr_imported.send(authority=authority, csr=csr)
@ -205,8 +237,8 @@ def import_certificate(**kwargs):
:param kwargs:
"""
if not kwargs.get('owner'):
kwargs['owner'] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0]
if not kwargs.get("owner"):
kwargs["owner"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")[0]
return upload(**kwargs)
@ -217,21 +249,16 @@ def upload(**kwargs):
"""
roles = create_certificate_roles(**kwargs)
if kwargs.get('roles'):
kwargs['roles'] += roles
if kwargs.get("roles"):
kwargs["roles"] += roles
else:
kwargs['roles'] = roles
if kwargs.get('private_key'):
private_key = kwargs['private_key']
if not isinstance(private_key, bytes):
kwargs['private_key'] = private_key.encode('utf-8')
kwargs["roles"] = roles
cert = Certificate(**kwargs)
cert.authority = kwargs.get('authority')
cert.authority = kwargs.get("authority")
cert = database.create(cert)
kwargs['creator'].certificates.append(cert)
kwargs["creator"].certificates.append(cert)
cert = database.update(cert)
certificate_imported.send(certificate=cert, authority=cert.authority)
@ -248,39 +275,45 @@ def create(**kwargs):
current_app.logger.error("Exception minting certificate", exc_info=True)
sentry.captureException()
raise
kwargs['body'] = cert_body
kwargs['private_key'] = private_key
kwargs['chain'] = cert_chain
kwargs['external_id'] = external_id
kwargs['csr'] = csr
kwargs["body"] = cert_body
kwargs["private_key"] = private_key
kwargs["chain"] = cert_chain
kwargs["external_id"] = external_id
kwargs["csr"] = csr
roles = create_certificate_roles(**kwargs)
if kwargs.get('roles'):
kwargs['roles'] += roles
if kwargs.get("roles"):
kwargs["roles"] += roles
else:
kwargs['roles'] = roles
kwargs["roles"] = roles
if cert_body:
cert = Certificate(**kwargs)
kwargs['creator'].certificates.append(cert)
kwargs["creator"].certificates.append(cert)
else:
cert = PendingCertificate(**kwargs)
kwargs['creator'].pending_certificates.append(cert)
kwargs["creator"].pending_certificates.append(cert)
cert.authority = kwargs['authority']
cert.authority = kwargs["authority"]
database.commit()
if isinstance(cert, Certificate):
certificate_issued.send(certificate=cert, authority=cert.authority)
metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
metrics.send(
"certificate_issued",
"counter",
1,
metric_tags=dict(owner=cert.owner, issuer=cert.issuer),
)
if isinstance(cert, PendingCertificate):
# We need to refresh the pending certificate to avoid "Instance is not bound to a Session; "
# "attribute refresh operation cannot proceed"
pending_cert = database.session_query(PendingCertificate).get(cert.id)
from lemur.common.celery import fetch_acme_cert
if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False):
fetch_acme_cert.apply_async((pending_cert.id,), countdown=5)
@ -296,85 +329,150 @@ def render(args):
"""
query = database.session_query(Certificate)
time_range = args.pop('time_range')
destination_id = args.pop('destination_id')
notification_id = args.pop('notification_id', None)
show = args.pop('show')
show_expired = args.pop("showExpired")
if show_expired != 1:
one_month_old = arrow.now()\
.shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\
.format("YYYY-MM-DD")
query = query.filter(Certificate.not_after > one_month_old)
time_range = args.pop("time_range")
destination_id = args.pop("destination_id")
notification_id = args.pop("notification_id", None)
show = args.pop("show")
# owner = args.pop('owner')
# creator = args.pop('creator') # TODO we should enabling filtering by owner
filt = args.pop('filter')
filt = args.pop("filter")
if filt:
terms = filt.split(';')
term = '%{0}%'.format(terms[1])
terms = filt.split(";")
term = "%{0}%".format(terms[1])
# Exact matches for quotes. Only applies to name, issuer, and cn
if terms[1].startswith('"') and terms[1].endswith('"'):
term = terms[1][1:-1]
if 'issuer' in terms:
if "issuer" in terms:
# we can't rely on issuer being correct in the cert directly so we combine queries
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike(term)) \
sub_query = (
database.session_query(Authority.id)
.filter(Authority.name.ilike(term))
.subquery()
)
query = query.filter(
or_(
Certificate.issuer.ilike(term),
Certificate.authority_id.in_(sub_query)
Certificate.authority_id.in_(sub_query),
)
)
elif 'destination' in terms:
query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
elif 'notify' in filt:
elif "destination" in terms:
query = query.filter(
Certificate.destinations.any(Destination.id == terms[1])
)
elif "notify" in filt:
query = query.filter(Certificate.notify == truthiness(terms[1]))
elif 'active' in filt:
elif "active" in filt:
query = query.filter(Certificate.active == truthiness(terms[1]))
elif 'cn' in terms:
elif "cn" in terms:
query = query.filter(
or_(
Certificate.cn.ilike(term),
Certificate.domains.any(Domain.name.ilike(term))
Certificate.domains.any(Domain.name.ilike(term)),
)
)
elif 'id' in terms:
elif "id" in terms:
query = query.filter(Certificate.id == cast(terms[1], Integer))
elif 'name' in terms:
query = query.outerjoin(certificate_associations).outerjoin(Domain).filter(
elif "name" in terms:
query = query.filter(
or_(
Certificate.name.ilike(term),
Domain.name.ilike(term),
Certificate.domains.any(Domain.name.ilike(term)),
Certificate.cn.ilike(term),
)
).group_by(Certificate.id)
)
elif "fixedName" in terms:
# only what matches the fixed name directly if a fixedname is provided
query = query.filter(Certificate.name == terms[1])
else:
query = database.filter(query, Certificate, terms)
if show:
sub_query = database.session_query(Role.name).filter(Role.user_id == args['user'].id).subquery()
sub_query = (
database.session_query(Role.name)
.filter(Role.user_id == args["user"].id)
.subquery()
)
query = query.filter(
or_(
Certificate.user_id == args['user'].id,
Certificate.owner.in_(sub_query)
Certificate.user_id == args["user"].id, Certificate.owner.in_(sub_query)
)
)
if destination_id:
query = query.filter(Certificate.destinations.any(Destination.id == destination_id))
query = query.filter(
Certificate.destinations.any(Destination.id == destination_id)
)
if notification_id:
query = query.filter(Certificate.notifications.any(Notification.id == notification_id))
query = query.filter(
Certificate.notifications.any(Notification.id == notification_id)
)
if time_range:
to = arrow.now().replace(weeks=+time_range).format('YYYY-MM-DD')
now = arrow.now().format('YYYY-MM-DD')
query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
to = arrow.now().shift(weeks=+time_range).format("YYYY-MM-DD")
now = arrow.now().format("YYYY-MM-DD")
query = query.filter(Certificate.not_after <= to).filter(
Certificate.not_after >= now
)
if current_app.config.get("ALLOW_CERT_DELETION", False):
query = query.filter(Certificate.deleted == False) # noqa
result = database.sort_and_page(query, Certificate, args)
return result
def query_name(certificate_name, args):
"""
Helper function that queries for a certificate by name
:param args:
:return:
"""
query = database.session_query(Certificate)
query = query.filter(Certificate.name == certificate_name)
result = database.sort_and_page(query, Certificate, args)
return result
def query_common_name(common_name, args):
"""
Helper function that queries for not expired certificates by common name (and owner)
:param common_name:
:param args:
:return:
"""
owner = args.pop("owner")
if not owner:
owner = "%"
# only not expired certificates
current_time = arrow.utcnow()
result = (
Certificate.query.filter(Certificate.cn.ilike(common_name))
.filter(Certificate.owner.ilike(owner))
.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))
.all()
)
return result
def create_csr(**csr_config):
"""
Given a list of domains create the appropriate csr
@ -382,65 +480,77 @@ def create_csr(**csr_config):
:param csr_config:
"""
private_key = generate_private_key(csr_config.get('key_type'))
private_key = generate_private_key(csr_config.get("key_type"))
builder = x509.CertificateSigningRequestBuilder()
name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['common_name'])]
if current_app.config.get('LEMUR_OWNER_EMAIL_IN_SUBJECT', True):
name_list.append(x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config['owner']))
if 'organization' in csr_config and csr_config['organization'].strip():
name_list.append(x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config['organization']))
if 'organizational_unit' in csr_config and csr_config['organizational_unit'].strip():
name_list.append(x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizational_unit']))
if 'country' in csr_config and csr_config['country'].strip():
name_list.append(x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']))
if 'state' in csr_config and csr_config['state'].strip():
name_list.append(x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']))
if 'location' in csr_config and csr_config['location'].strip():
name_list.append(x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']))
name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config["common_name"])]
if current_app.config.get("LEMUR_OWNER_EMAIL_IN_SUBJECT", True):
name_list.append(
x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config["owner"])
)
if "organization" in csr_config and csr_config["organization"].strip():
name_list.append(
x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config["organization"])
)
if (
"organizational_unit" in csr_config
and csr_config["organizational_unit"].strip()
):
name_list.append(
x509.NameAttribute(
x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config["organizational_unit"]
)
)
if "country" in csr_config and csr_config["country"].strip():
name_list.append(
x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config["country"])
)
if "state" in csr_config and csr_config["state"].strip():
name_list.append(
x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config["state"])
)
if "location" in csr_config and csr_config["location"].strip():
name_list.append(
x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config["location"])
)
builder = builder.subject_name(x509.Name(name_list))
extensions = csr_config.get('extensions', {})
critical_extensions = ['basic_constraints', 'sub_alt_names', 'key_usage']
noncritical_extensions = ['extended_key_usage']
extensions = csr_config.get("extensions", {})
critical_extensions = ["basic_constraints", "sub_alt_names", "key_usage"]
noncritical_extensions = ["extended_key_usage"]
for k, v in extensions.items():
if v:
if k in critical_extensions:
current_app.logger.debug('Adding Critical Extension: {0} {1}'.format(k, v))
if k == 'sub_alt_names':
if v['names']:
builder = builder.add_extension(v['names'], critical=True)
current_app.logger.debug(
"Adding Critical Extension: {0} {1}".format(k, v)
)
if k == "sub_alt_names":
if v["names"]:
builder = builder.add_extension(v["names"], critical=True)
else:
builder = builder.add_extension(v, critical=True)
if k in noncritical_extensions:
current_app.logger.debug('Adding Extension: {0} {1}'.format(k, v))
current_app.logger.debug("Adding Extension: {0} {1}".format(k, v))
builder = builder.add_extension(v, critical=False)
ski = extensions.get('subject_key_identifier', {})
if ski.get('include_ski', False):
ski = extensions.get("subject_key_identifier", {})
if ski.get("include_ski", False):
builder = builder.add_extension(
x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
critical=False
critical=False,
)
request = builder.sign(
private_key, hashes.SHA256(), default_backend()
)
request = builder.sign(private_key, hashes.SHA256(), default_backend())
# serialize our private key and CSR
private_key = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL, # would like to use PKCS8 but AWS ELBs don't like it
encryption_algorithm=serialization.NoEncryption()
)
encryption_algorithm=serialization.NoEncryption(),
).decode("utf-8")
if isinstance(private_key, bytes):
private_key = private_key.decode('utf-8')
csr = request.public_bytes(
encoding=serialization.Encoding.PEM
).decode('utf-8')
csr = request.public_bytes(encoding=serialization.Encoding.PEM).decode("utf-8")
return csr, private_key
@ -452,16 +562,19 @@ def stats(**kwargs):
:param kwargs:
:return:
"""
if kwargs.get('metric') == 'not_after':
if kwargs.get("metric") == "not_after":
start = arrow.utcnow()
end = start.replace(weeks=+32)
items = database.db.session.query(Certificate.issuer, func.count(Certificate.id)) \
.group_by(Certificate.issuer) \
.filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
.filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()
end = start.shift(weeks=+32)
items = (
database.db.session.query(Certificate.issuer, func.count(Certificate.id))
.group_by(Certificate.issuer)
.filter(Certificate.not_after <= end.format("YYYY-MM-DD"))
.filter(Certificate.not_after >= start.format("YYYY-MM-DD"))
.all()
)
else:
attr = getattr(Certificate, kwargs.get('metric'))
attr = getattr(Certificate, kwargs.get("metric"))
query = database.db.session.query(attr, func.count(attr))
items = query.group_by(attr).all()
@ -472,7 +585,7 @@ def stats(**kwargs):
keys.append(key)
values.append(count)
return {'labels': keys, 'values': values}
return {"labels": keys, "values": values}
def get_account_number(arn):
@ -519,22 +632,24 @@ def get_certificate_primitives(certificate):
certificate via `create`.
"""
start, end = calculate_reissue_range(certificate.not_before, certificate.not_after)
ser = CertificateInputSchema().load(CertificateOutputSchema().dump(certificate).data)
ser = CertificateInputSchema().load(
CertificateOutputSchema().dump(certificate).data
)
assert not ser.errors, "Error re-serializing certificate: %s" % ser.errors
data = ser.data
# we can't quite tell if we are using a custom name, as this is an automated process (typically)
# we will rely on the Lemur generated name
data.pop('name', None)
data.pop("name", None)
# TODO this can be removed once we migrate away from cn
data['cn'] = data['common_name']
data["cn"] = data["common_name"]
# needed until we move off not_*
data['not_before'] = start
data['not_after'] = end
data['validity_start'] = start
data['validity_end'] = end
data["not_before"] = start
data["not_after"] = end
data["validity_start"] = start
data["validity_end"] = end
return data
@ -552,13 +667,13 @@ def reissue_certificate(certificate, replace=None, user=None):
# We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation.
del primitives["csr"]
if not user:
primitives['creator'] = certificate.user
primitives["creator"] = certificate.user
else:
primitives['creator'] = user
primitives["creator"] = user
if replace:
primitives['replaces'] = [certificate]
primitives["replaces"] = [certificate]
new_cert = create(**primitives)

41
lemur/certificates/utils.py Normal file
View File

@ -0,0 +1,41 @@
"""
Utils to parse certificate data.
.. module: lemur.certificates.hooks
:platform: Unix
:copyright: (c) 2019 by Javier Ramos, see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Javier Ramos <javier.ramos@booking.com>
"""
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from marshmallow.exceptions import ValidationError
def get_sans_from_csr(data):
"""
Fetches SubjectAlternativeNames from CSR.
Works with any kind of SubjectAlternativeName
:param data: PEM-encoded string with CSR
:return: List of LemurAPI-compatible subAltNames
"""
sub_alt_names = []
try:
request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
except Exception:
raise ValidationError("CSR presented is not valid.")
try:
alt_names = request.extensions.get_extension_for_class(
x509.SubjectAlternativeName
)
for alt_name in alt_names.value:
sub_alt_names.append(
{"nameType": type(alt_name).__name__, "value": alt_name.value}
)
except x509.ExtensionNotFound:
pass
return sub_alt_names

54
lemur/certificates/verify.py
View File

@ -29,31 +29,45 @@ def ocsp_verify(cert, cert_path, issuer_chain_path):
:param issuer_chain_path:
:return bool: True if certificate is valid, False otherwise
"""
command = ['openssl', 'x509', '-noout', '-ocsp_uri', '-in', cert_path]
command = ["openssl", "x509", "-noout", "-ocsp_uri", "-in", cert_path]
p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
url, err = p1.communicate()
if not url:
current_app.logger.debug("No OCSP URL in certificate {}".format(cert.serial_number))
current_app.logger.debug(
"No OCSP URL in certificate {}".format(cert.serial_number)
)
return None
p2 = subprocess.Popen(['openssl', 'ocsp', '-issuer', issuer_chain_path,
'-cert', cert_path, "-url", url.strip()],
p2 = subprocess.Popen(
[
"openssl",
"ocsp",
"-issuer",
issuer_chain_path,
"-cert",
cert_path,
"-url",
url.strip(),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stderr=subprocess.PIPE,
)
message, err = p2.communicate()
p_message = message.decode('utf-8')
p_message = message.decode("utf-8")
if 'error' in p_message or 'Error' in p_message:
if "error" in p_message or "Error" in p_message:
raise Exception("Got error when parsing OCSP url")
elif 'revoked' in p_message:
current_app.logger.debug("OCSP reports certificate revoked: {}".format(cert.serial_number))
elif "revoked" in p_message:
current_app.logger.debug(
"OCSP reports certificate revoked: {}".format(cert.serial_number)
)
return False
elif 'good' not in p_message:
elif "good" not in p_message:
raise Exception("Did not receive a valid response")
return True
@ -73,7 +87,9 @@ def crl_verify(cert, cert_path):
x509.OID_CRL_DISTRIBUTION_POINTS
).value
except x509.ExtensionNotFound:
current_app.logger.debug("No CRLDP extension in certificate {}".format(cert.serial_number))
current_app.logger.debug(
"No CRLDP extension in certificate {}".format(cert.serial_number)
)
return None
for p in distribution_points:
@ -92,8 +108,9 @@ def crl_verify(cert, cert_path):
except ConnectionError:
raise Exception("Unable to retrieve CRL: {0}".format(point))
crl_cache[point] = x509.load_der_x509_crl(response.content,
backend=default_backend())
crl_cache[point] = x509.load_der_x509_crl(
response.content, backend=default_backend()
)
else:
current_app.logger.debug("CRL point is cached {}".format(point))
@ -110,8 +127,9 @@ def crl_verify(cert, cert_path):
except x509.ExtensionNotFound:
pass
current_app.logger.debug("CRL reports certificate "
"revoked: {}".format(cert.serial_number))
current_app.logger.debug(
"CRL reports certificate " "revoked: {}".format(cert.serial_number)
)
return False
return True
@ -125,7 +143,7 @@ def verify(cert_path, issuer_chain_path):
:param issuer_chain_path:
:return: True if valid, False otherwise
"""
with open(cert_path, 'rt') as c:
with open(cert_path, "rt") as c:
try:
cert = parse_certificate(c.read())
except ValueError as e:
@ -154,10 +172,10 @@ def verify_string(cert_string, issuer_string):
:return: True if valid, False otherwise
"""
with mktempfile() as cert_tmp:
with open(cert_tmp, 'w') as f:
with open(cert_tmp, "w") as f:
f.write(cert_string)
with mktempfile() as issuer_tmp:
with open(issuer_tmp, 'w') as f:
with open(issuer_tmp, "w") as f:
f.write(issuer_string)
status = verify(cert_tmp, issuer_tmp)
return status

478
lemur/certificates/views.py
View File

@ -8,7 +8,7 @@
import base64
from builtins import str
from flask import Blueprint, make_response, jsonify, g
from flask import Blueprint, make_response, jsonify, g, current_app
from flask_restful import reqparse, Api, inputs
from lemur.common.schema import validate_schema
@ -26,17 +26,224 @@ from lemur.certificates.schemas import (
certificate_upload_input_schema,
certificates_output_schema,
certificate_export_input_schema,
certificate_edit_input_schema
certificate_edit_input_schema,
certificates_list_output_schema_factory,
)
from lemur.roles import service as role_service
from lemur.logs import service as log_service
mod = Blueprint('certificates', __name__)
mod = Blueprint("certificates", __name__)
api = Api(mod)
class CertificatesListValid(AuthenticatedResource):
""" Defines the 'certificates/valid' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(CertificatesListValid, self).__init__()
@validate_schema(None, certificates_output_schema)
def get(self):
"""
.. http:get:: /certificates/valid/<query>
The current list of not-expired certificates for a given common name, and owner
**Example request**:
.. sourcecode:: http
GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com
HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"items": [{
"status": null,
"cn": "*.test.example.net",
"chain": "",
"csr": "-----BEGIN CERTIFICATE REQUEST-----"
"authority": {
"active": true,
"owner": "secure@example.com",
"id": 1,
"description": "verisign test authority",
"name": "verisign"
},
"owner": "joe@example.com",
"serial": "82311058732025924142789179368889309156",
"id": 2288,
"issuer": "SymantecCorporation",
"dateCreated": "2016-06-03T06:09:42.133769+00:00",
"notBefore": "2016-06-03T00:00:00+00:00",
"notAfter": "2018-01-12T23:59:59+00:00",
"destinations": [],
"bits": 2048,
"body": "-----BEGIN CERTIFICATE-----...",
"description": null,
"deleted": null,
"notifications": [{
"id": 1
}],
"signingAlgorithm": "sha256",
"user": {
"username": "jane",
"active": true,
"email": "jane@example.com",
"id": 2
},
"active": true,
"domains": [{
"sensitive": false,
"id": 1090,
"name": "*.test.example.net"
}],
"replaces": [],
"replaced": [],
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
"roles": [{
"id": 464,
"description": "This is a google group based role created by Lemur",
"name": "joe@example.com"
}],
"san": null
}],
"total": 1
}
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args["user"] = g.user
common_name = args["filter"].split(";")[1]
return service.query_common_name(common_name, args)
class CertificatesNameQuery(AuthenticatedResource):
""" Defines the 'certificates/name' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(CertificatesNameQuery, self).__init__()
@validate_schema(None, certificates_output_schema)
def get(self, certificate_name):
"""
.. http:get:: /certificates/name/<query>
The current list of certificates
**Example request**:
.. sourcecode:: http
GET /certificates/name/WILDCARD.test.example.net-SymantecCorporation-20160603-20180112 HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"items": [{
"status": null,
"cn": "*.test.example.net",
"chain": "",
"csr": "-----BEGIN CERTIFICATE REQUEST-----"
"authority": {
"active": true,
"owner": "secure@example.com",
"id": 1,
"description": "verisign test authority",
"name": "verisign"
},
"owner": "joe@example.com",
"serial": "82311058732025924142789179368889309156",
"id": 2288,
"issuer": "SymantecCorporation",
"dateCreated": "2016-06-03T06:09:42.133769+00:00",
"notBefore": "2016-06-03T00:00:00+00:00",
"notAfter": "2018-01-12T23:59:59+00:00",
"destinations": [],
"bits": 2048,
"body": "-----BEGIN CERTIFICATE-----...",
"description": null,
"deleted": null,
"notifications": [{
"id": 1
}],
"signingAlgorithm": "sha256",
"user": {
"username": "jane",
"active": true,
"email": "jane@example.com",
"id": 2
},
"active": true,
"domains": [{
"sensitive": false,
"id": 1090,
"name": "*.test.example.net"
}],
"replaces": [],
"replaced": [],
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
"roles": [{
"id": 464,
"description": "This is a google group based role created by Lemur",
"name": "joe@example.com"
}],
"san": null
}],
"total": 1
}
:query sortBy: field to sort on
:query sortDir: asc or desc
:query page: int. default is 1
:query filter: key value pair format is k;v
:query count: count number. default is 10
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
parser = paginated_parser.copy()
parser.add_argument("timeRange", type=int, dest="time_range", location="args")
parser.add_argument("owner", type=inputs.boolean, location="args")
parser.add_argument("id", type=str, location="args")
parser.add_argument("active", type=inputs.boolean, location="args")
parser.add_argument(
"destinationId", type=int, dest="destination_id", location="args"
)
parser.add_argument("creator", type=str, location="args")
parser.add_argument("show", type=str, location="args")
args = parser.parse_args()
args["user"] = g.user
return service.query_name(certificate_name, args)
class CertificatesList(AuthenticatedResource):
""" Defines the 'certificates' endpoint """
@ -44,7 +251,7 @@ class CertificatesList(AuthenticatedResource):
self.reqparse = reqparse.RequestParser()
super(CertificatesList, self).__init__()
@validate_schema(None, certificates_output_schema)
@validate_schema(None, certificates_list_output_schema_factory)
def get(self):
"""
.. http:get:: /certificates
@ -132,16 +339,19 @@ class CertificatesList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
parser.add_argument('timeRange', type=int, dest='time_range', location='args')
parser.add_argument('owner', type=inputs.boolean, location='args')
parser.add_argument('id', type=str, location='args')
parser.add_argument('active', type=inputs.boolean, location='args')
parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
parser.add_argument('creator', type=str, location='args')
parser.add_argument('show', type=str, location='args')
parser.add_argument("timeRange", type=int, dest="time_range", location="args")
parser.add_argument("owner", type=inputs.boolean, location="args")
parser.add_argument("id", type=str, location="args")
parser.add_argument("active", type=inputs.boolean, location="args")
parser.add_argument(
"destinationId", type=int, dest="destination_id", location="args"
)
parser.add_argument("creator", type=str, location="args")
parser.add_argument("show", type=str, location="args")
parser.add_argument("showExpired", type=int, location="args")
args = parser.parse_args()
args['user'] = g.user
args["user"] = g.user
return service.render(args)
@validate_schema(certificate_input_schema, certificate_output_schema)
@ -259,24 +469,31 @@ class CertificatesList(AuthenticatedResource):
:statuscode 403: unauthenticated
"""
role = role_service.get_by_name(data['authority'].owner)
role = role_service.get_by_name(data["authority"].owner)
# all the authority role members should be allowed
roles = [x.name for x in data['authority'].roles]
roles = [x.name for x in data["authority"].roles]
# allow "owner" roles by team DL
roles.append(role)
authority_permission = AuthorityPermission(data['authority'].id, roles)
authority_permission = AuthorityPermission(data["authority"].id, roles)
if authority_permission.can():
data['creator'] = g.user
data["creator"] = g.user
cert = service.create(**data)
if isinstance(cert, Certificate):
# only log if created, not pending
log_service.create(g.user, 'create_cert', certificate=cert)
log_service.create(g.user, "create_cert", certificate=cert)
return cert
return dict(message="You are not authorized to use the authority: {0}".format(data['authority'].name)), 403
return (
dict(
message="You are not authorized to use the authority: {0}".format(
data["authority"].name
)
),
403,
)
class CertificatesUpload(AuthenticatedResource):
@ -306,6 +523,7 @@ class CertificatesUpload(AuthenticatedResource):
"body": "-----BEGIN CERTIFICATE-----...",
"chain": "-----BEGIN CERTIFICATE-----...",
"privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
"csr": "-----BEGIN CERTIFICATE REQUEST-----..."
"destinations": [],
"notifications": [],
"replacements": [],
@ -378,12 +596,14 @@ class CertificatesUpload(AuthenticatedResource):
:statuscode 200: no error
"""
data['creator'] = g.user
if data.get('destinations'):
if data.get('private_key'):
data["creator"] = g.user
if data.get("destinations"):
if data.get("private_key"):
return service.upload(**data)
else:
raise Exception("Private key must be provided in order to upload certificate to AWS")
raise Exception(
"Private key must be provided in order to upload certificate to AWS"
)
return service.upload(**data)
@ -395,10 +615,12 @@ class CertificatesStats(AuthenticatedResource):
super(CertificatesStats, self).__init__()
def get(self):
self.reqparse.add_argument('metric', type=str, location='args')
self.reqparse.add_argument('range', default=32, type=int, location='args')
self.reqparse.add_argument('destinationId', dest='destination_id', location='args')
self.reqparse.add_argument('active', type=str, default='true', location='args')
self.reqparse.add_argument("metric", type=str, location="args")
self.reqparse.add_argument("range", default=32, type=int, location="args")
self.reqparse.add_argument(
"destinationId", dest="destination_id", location="args"
)
self.reqparse.add_argument("active", type=str, default="true", location="args")
args = self.reqparse.parse_args()
@ -450,12 +672,12 @@ class CertificatePrivateKey(AuthenticatedResource):
permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
if not permission.can():
return dict(message='You are not authorized to view this key'), 403
return dict(message="You are not authorized to view this key"), 403
log_service.create(g.current_user, 'key_view', certificate=cert)
log_service.create(g.current_user, "key_view", certificate=cert)
response = make_response(jsonify(key=cert.private_key), 200)
response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
response.headers['pragma'] = 'no-cache'
response.headers["cache-control"] = "private, max-age=0, no-cache, no-store"
response.headers["pragma"] = "no-cache"
return response
@ -645,21 +867,79 @@ class Certificates(AuthenticatedResource):
permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
if not permission.can():
return dict(message='You are not authorized to update this certificate'), 403
return (
dict(message="You are not authorized to update this certificate"),
403,
)
for destination in data['destinations']:
for destination in data["destinations"]:
if destination.plugin.requires_key:
if not cert.private_key:
return dict(
message='Unable to add destination: {0}. Certificate does not have required private key.'.format(
return (
dict(
message="Unable to add destination: {0}. Certificate does not have required private key.".format(
destination.label
)
), 400
),
400,
)
cert = service.update(certificate_id, **data)
log_service.create(g.current_user, 'update_cert', certificate=cert)
log_service.create(g.current_user, "update_cert", certificate=cert)
return cert
def delete(self, certificate_id, data=None):
"""
.. http:delete:: /certificates/1
Delete a certificate
**Example request**:
.. sourcecode:: http
DELETE /certificates/1 HTTP/1.1
Host: example.com
**Example response**:
.. sourcecode:: http
HTTP/1.1 204 OK
:reqheader Authorization: OAuth token to authenticate
:statuscode 204: no error
:statuscode 403: unauthenticated
:statuscode 404: certificate not found
:statuscode 405: certificate deletion is disabled
"""
if not current_app.config.get("ALLOW_CERT_DELETION", False):
return dict(message="Certificate deletion is disabled"), 405
cert = service.get(certificate_id)
if not cert:
return dict(message="Cannot find specified certificate"), 404
if cert.deleted:
return dict(message="Certificate is already deleted"), 412
# allow creators
if g.current_user != cert.user:
owner_role = role_service.get_by_name(cert.owner)
permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
if not permission.can():
return (
dict(message="You are not authorized to delete this certificate"),
403,
)
service.update(certificate_id, deleted=True)
log_service.create(g.current_user, "delete_cert", certificate=cert)
return "Certificate deleted", 204
class NotificationCertificatesList(AuthenticatedResource):
""" Defines the 'certificates' endpoint """
@ -758,17 +1038,19 @@ class NotificationCertificatesList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
parser.add_argument('timeRange', type=int, dest='time_range', location='args')
parser.add_argument('owner', type=inputs.boolean, location='args')
parser.add_argument('id', type=str, location='args')
parser.add_argument('active', type=inputs.boolean, location='args')
parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
parser.add_argument('creator', type=str, location='args')
parser.add_argument('show', type=str, location='args')
parser.add_argument("timeRange", type=int, dest="time_range", location="args")
parser.add_argument("owner", type=inputs.boolean, location="args")
parser.add_argument("id", type=str, location="args")
parser.add_argument("active", type=inputs.boolean, location="args")
parser.add_argument(
"destinationId", type=int, dest="destination_id", location="args"
)
parser.add_argument("creator", type=str, location="args")
parser.add_argument("show", type=str, location="args")
args = parser.parse_args()
args['notification_id'] = notification_id
args['user'] = g.current_user
args["notification_id"] = notification_id
args["user"] = g.current_user
return service.render(args)
@ -941,30 +1223,48 @@ class CertificateExport(AuthenticatedResource):
if not cert:
return dict(message="Cannot find specified certificate"), 404
plugin = data['plugin']['plugin_object']
plugin = data["plugin"]["plugin_object"]
if plugin.requires_key:
if not cert.private_key:
return dict(
message='Unable to export certificate, plugin: {0} requires a private key but no key was found.'.format(
plugin.slug)), 400
return (
dict(
message="Unable to export certificate, plugin: {0} requires a private key but no key was found.".format(
plugin.slug
)
),
400,
)
else:
# allow creators
if g.current_user != cert.user:
owner_role = role_service.get_by_name(cert.owner)
permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
permission = CertificatePermission(
owner_role, [x.name for x in cert.roles]
)
if not permission.can():
return dict(message='You are not authorized to export this certificate.'), 403
return (
dict(
message="You are not authorized to export this certificate."
),
403,
)
options = data['plugin']['plugin_options']
options = data["plugin"]["plugin_options"]
log_service.create(g.current_user, 'key_view', certificate=cert)
extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
log_service.create(g.current_user, "key_view", certificate=cert)
extension, passphrase, data = plugin.export(
cert.body, cert.chain, cert.private_key, options
)
# we take a hit in message size when b64 encoding
return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data).decode('utf-8'))
return dict(
extension=extension,
passphrase=passphrase,
data=base64.b64encode(data).decode("utf-8"),
)
class CertificateRevoke(AuthenticatedResource):
@ -1015,28 +1315,66 @@ class CertificateRevoke(AuthenticatedResource):
permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
if not permission.can():
return dict(message='You are not authorized to revoke this certificate.'), 403
return (
dict(message="You are not authorized to revoke this certificate."),
403,
)
if not cert.external_id:
return dict(message='Cannot revoke certificate. No external id found.'), 400
return dict(message="Cannot revoke certificate. No external id found."), 400
if cert.endpoints:
return dict(message='Cannot revoke certificate. Endpoints are deployed with the given certificate.'), 403
return (
dict(
message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
),
403,
)
plugin = plugins.get(cert.authority.plugin_name)
plugin.revoke_certificate(cert, data)
log_service.create(g.current_user, 'revoke_cert', certificate=cert)
log_service.create(g.current_user, "revoke_cert", certificate=cert)
return dict(id=cert.id)
api.add_resource(CertificateRevoke, '/certificates/<int:certificate_id>/revoke', endpoint='revokeCertificate')
api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
api.add_resource(Certificates, '/certificates/<int:certificate_id>', endpoint='certificate')
api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificateStats')
api.add_resource(CertificatesUpload, '/certificates/upload', endpoint='certificateUpload')
api.add_resource(CertificatePrivateKey, '/certificates/<int:certificate_id>/key', endpoint='privateKeyCertificates')
api.add_resource(CertificateExport, '/certificates/<int:certificate_id>/export', endpoint='exportCertificate')
api.add_resource(NotificationCertificatesList, '/notifications/<int:notification_id>/certificates',
endpoint='notificationCertificates')
api.add_resource(CertificatesReplacementsList, '/certificates/<int:certificate_id>/replacements',
endpoint='replacements')
api.add_resource(
CertificateRevoke,
"/certificates/<int:certificate_id>/revoke",
endpoint="revokeCertificate",
)
api.add_resource(
CertificatesNameQuery,
"/certificates/name/<string:certificate_name>",
endpoint="certificatesNameQuery",
)
api.add_resource(CertificatesList, "/certificates", endpoint="certificates")
api.add_resource(
CertificatesListValid, "/certificates/valid", endpoint="certificatesListValid"
)
api.add_resource(
Certificates, "/certificates/<int:certificate_id>", endpoint="certificate"
)
api.add_resource(CertificatesStats, "/certificates/stats", endpoint="certificateStats")
api.add_resource(
CertificatesUpload, "/certificates/upload", endpoint="certificateUpload"
)
api.add_resource(
CertificatePrivateKey,
"/certificates/<int:certificate_id>/key",
endpoint="privateKeyCertificates",
)
api.add_resource(
CertificateExport,
"/certificates/<int:certificate_id>/export",
endpoint="exportCertificate",
)
api.add_resource(
NotificationCertificatesList,
"/notifications/<int:notification_id>/certificates",
endpoint="notificationCertificates",
)
api.add_resource(
CertificatesReplacementsList,
"/certificates/<int:certificate_id>/replacements",
endpoint="replacements",
)

546
lemur/common/celery.py
View File

@ -9,27 +9,43 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
"""
import copy
import sys
import time
from datetime import datetime, timezone, timedelta
from celery import Celery
from celery.exceptions import SoftTimeLimitExceeded
from flask import current_app
from lemur.authorities.service import get as get_authority
from lemur.common.redis import RedisHandler
from lemur.destinations import service as destinations_service
from lemur.extensions import metrics, sentry
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.sources.cli import clean, sync, validate_sources
from lemur.sources.service import add_aws_destination_to_sources
from lemur.certificates import cli as cli_certificate
from lemur.dns_providers import cli as cli_dns_providers
from lemur.notifications import cli as cli_notification
from lemur.endpoints import cli as cli_endpoints
if current_app:
flask_app = current_app
else:
flask_app = create_app()
red = RedisHandler().redis()
def make_celery(app):
celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
broker=app.config.get('CELERY_BROKER_URL'))
celery = Celery(
app.import_name,
backend=app.config.get("CELERY_RESULT_BACKEND"),
broker=app.config.get("CELERY_BROKER_URL"),
)
celery.conf.update(app.config)
TaskBase = celery.Task
@ -47,7 +63,63 @@ def make_celery(app):
celery = make_celery(flask_app)
def is_task_active(fun, task_id, args):
from celery.task.control import inspect
if not args:
args = '()' # empty args
i = inspect()
active_tasks = i.active()
for _, tasks in active_tasks.items():
for task in tasks:
if task.get("id") == task_id:
continue
if task.get("name") == fun and task.get("args") == str(args):
return True
return False
@celery.task()
def report_celery_last_success_metrics():
"""
For each celery task, this will determine the number of seconds since it has last been successful.
Celery tasks should be emitting redis stats with a deterministic key (In our case, `f"{task}.last_success"`.
report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful.
Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted
from this function.
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "recurrent task",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_time = int(time.time())
schedule = current_app.config.get('CELERYBEAT_SCHEDULE')
for _, t in schedule.items():
task = t.get("task")
last_success = int(red.get(f"{task}.last_success") or 0)
metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success)
red.set(
f"{function}.last_success", int(time.time())
) # Alert if this metric is not seen
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=600)
def fetch_acme_cert(id):
"""
Attempt to get the full certificate for the pending certificate listed.
@ -55,11 +127,25 @@ def fetch_acme_cert(id):
Args:
id: an id of a PendingCertificate
"""
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
function = f"{__name__}.{sys._getframe().f_code.co_name}"
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
"message": "Resolving pending certificate {}".format(id)
"function": function,
"message": "Resolving pending certificate {}".format(id),
"task_id": task_id,
"id": id,
}
current_app.logger.debug(log_data)
if task_id and is_task_active(log_data["function"], task_id, (id,)):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
pending_certs = pending_certificate_service.get_pending_certs([id])
new = 0
failed = 0
@ -69,7 +155,7 @@ def fetch_acme_cert(id):
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
if cert_authority.plugin_name == "acme-issuer":
acme_certs.append(cert)
else:
wrong_issuer += 1
@ -82,20 +168,22 @@ def fetch_acme_cert(id):
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if not pending_cert:
log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
log_data[
"message"
] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
current_app.logger.error(log_data)
continue
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate as resolved
final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved=True
final_cert = pending_certificate_service.create_certificate(
pending_cert, real_cert, pending_cert.user
)
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved_cert_id=final_cert.id
cert.get("pending_cert").id, resolved_cert_id=final_cert.id
)
pending_certificate_service.update(
cert.get("pending_cert").id, resolved=True
)
# add metrics to metrics extension
new += 1
@ -109,17 +197,17 @@ def fetch_acme_cert(id):
if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
send_pending_failure_notification(
pending_cert, notify_owner=pending_cert.notify
)
# Mark the pending cert as resolved
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved=True
cert.get("pending_cert").id, resolved=True
)
else:
pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update(
cert.get("pending_cert").id,
status=str(cert.get("last_error"))
cert.get("pending_cert").id, status=str(cert.get("last_error"))
)
# Add failed pending cert task back to queue
fetch_acme_cert.delay(id)
@ -129,31 +217,44 @@ def fetch_acme_cert(id):
log_data["failed"] = failed
log_data["wrong_issuer"] = wrong_issuer
current_app.logger.debug(log_data)
metrics.send(f"{function}.resolved", 'gauge', new)
metrics.send(f"{function}.failed", 'gauge', failed)
metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer)
print(
"[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
new=new,
failed=failed,
wrong_issuer=wrong_issuer
new=new, failed=failed, wrong_issuer=wrong_issuer
)
)
red.set(f'{function}.last_success', int(time.time()))
@celery.task()
def fetch_all_pending_acme_certs():
"""Instantiate celery workers to resolve all pending Acme certificates"""
pending_certs = pending_certificate_service.get_unresolved_pending_certs()
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
"message": "Starting job."
"function": function,
"message": "Starting job.",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
pending_certs = pending_certificate_service.get_unresolved_pending_certs()
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
if cert_authority.plugin_name == "acme-issuer":
if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
log_data["message"] = "Triggering job for cert {}".format(cert.name)
log_data["cert_name"] = cert.name
@ -161,23 +262,42 @@ def fetch_all_pending_acme_certs():
current_app.logger.debug(log_data)
fetch_acme_cert.delay(cert.id)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task()
def remove_old_acme_certs():
"""Prune old pending acme certificates from the database"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
"function": function,
"message": "Starting job.",
"task_id": task_id,
}
pending_certs = pending_certificate_service.get_pending_certs('all')
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
pending_certs = pending_certificate_service.get_pending_certs("all")
# Delete pending certs more than a week old
for cert in pending_certs:
if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
log_data['pending_cert_id'] = cert.id
log_data['pending_cert_name'] = cert.name
log_data['message'] = "Deleting pending certificate"
log_data["pending_cert_id"] = cert.id
log_data["pending_cert_name"] = cert.name
log_data["message"] = "Deleting pending certificate"
current_app.logger.debug(log_data)
pending_certificate_service.delete(cert.id)
pending_certificate_service.delete(cert)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task()
@ -186,13 +306,33 @@ def clean_all_sources():
This function will clean unused certificates from sources. This is a destructive operation and should only
be ran periodically. This function triggers one celery task per source.
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "Creating celery task to clean source",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
sources = validate_sources("all")
for source in sources:
current_app.logger.debug("Creating celery task to clean source {}".format(source.label))
log_data["source"] = source.label
current_app.logger.debug(log_data)
clean_source.delay(source.label)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task()
@celery.task(soft_time_limit=600)
def clean_source(source):
"""
This celery task will clean the specified source. This is a destructive operation that will delete unused
@ -201,8 +341,31 @@ def clean_source(source):
:param source:
:return:
"""
current_app.logger.debug("Cleaning source {}".format(source))
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "Cleaning source",
"source": source,
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, (source,)):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
clean([source], True)
except SoftTimeLimitExceeded:
log_data["message"] = "Clean source: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
@celery.task()
@ -210,13 +373,33 @@ def sync_all_sources():
"""
This function will sync certificates from all sources. This function triggers one celery task per source.
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "creating celery task to sync source",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
sources = validate_sources("all")
for source in sources:
current_app.logger.debug("Creating celery task to sync source {}".format(source.label))
log_data["source"] = source.label
current_app.logger.debug(log_data)
sync_source.delay(source.label)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task()
@celery.task(soft_time_limit=7200)
def sync_source(source):
"""
This celery task will sync the specified source.
@ -224,5 +407,296 @@ def sync_source(source):
:param source:
:return:
"""
current_app.logger.debug("Syncing source {}".format(source))
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "Syncing source",
"source": source,
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, (source,)):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
sync([source])
metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
except SoftTimeLimitExceeded:
log_data["message"] = "Error syncing source: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source})
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data)
metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
red.set(f'{function}.last_success', int(time.time()))
@celery.task()
def sync_source_destination():
"""
This celery task will sync destination and source, to make sure all new destinations are also present as source.
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reveals the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "syncing AWS destinations and sources",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
for dst in destinations_service.get_all():
if add_aws_destination_to_sources(dst):
log_data["message"] = "new source added"
log_data["source"] = dst.label
current_app.logger.debug(log_data)
log_data["message"] = "completed Syncing AWS destinations and sources"
current_app.logger.debug(log_data)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=3600)
def certificate_reissue():
"""
This celery task reissues certificates which are pending reissue
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "reissuing certificates",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_certificate.reissue(None, True)
except SoftTimeLimitExceeded:
log_data["message"] = "Certificate reissue: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
log_data["message"] = "reissuance completed"
current_app.logger.debug(log_data)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=3600)
def certificate_rotate():
"""
This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "rotating certificates",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_certificate.rotate(None, None, None, None, True)
except SoftTimeLimitExceeded:
log_data["message"] = "Certificate rotate: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
log_data["message"] = "rotation completed"
current_app.logger.debug(log_data)
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=3600)
def endpoints_expire():
"""
This celery task removes all endpoints that have not been recently updated
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "endpoints expire",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_endpoints.expire(2) # Time in hours
except SoftTimeLimitExceeded:
log_data["message"] = "endpoint expire: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=600)
def get_all_zones():
"""
This celery syncs all zones from the available dns providers
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "refresh all zones from available DNS providers",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_dns_providers.get_all_zones()
except SoftTimeLimitExceeded:
log_data["message"] = "get all zones: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=3600)
def check_revoked():
"""
This celery task attempts to check if any certs are expired
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "check if any certificates are revoked revoked",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_certificate.check_revoked()
except SoftTimeLimitExceeded:
log_data["message"] = "Checking revoked: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)
@celery.task(soft_time_limit=3600)
def notify_expirations():
"""
This celery task notifies about expiring certs
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "notify for cert expiration",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
except SoftTimeLimitExceeded:
log_data["message"] = "Notify expiring Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
red.set(f'{function}.last_success', int(time.time()))
metrics.send(f"{function}.success", 'counter', 1)

104
lemur/common/defaults.py
View File

@ -3,22 +3,29 @@ import unicodedata
from cryptography import x509
from flask import current_app
from lemur.common.utils import is_selfsigned
from lemur.extensions import sentry
from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
def text_to_slug(value):
"""Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters."""
def text_to_slug(value, joiner="-"):
"""
Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.
A series of non-alphanumeric characters is replaced with the joiner character.
"""
# Strip all character accents: decompose Unicode characters and then drop combining chars.
value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c))
value = "".join(
c for c in unicodedata.normalize("NFKD", value) if not unicodedata.combining(c)
)
# Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash.
# Except, keep 'xn--' used in IDNA domain names as is.
value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', '-', value)
# Replace all remaining non-alphanumeric characters with joiner string. Multiple characters get collapsed into a
# single joiner. Except, keep 'xn--' used in IDNA domain names as is.
value = re.sub(r"[^A-Za-z0-9.]+(?<!xn--)", joiner, value)
# '-' in the beginning or end of string looks ugly.
return value.strip('-')
return value.strip(joiner)
def certificate_name(common_name, issuer, not_before, not_after, san):
@ -43,12 +50,12 @@ def certificate_name(common_name, issuer, not_before, not_after, san):
temp = t.format(
subject=common_name,
issuer=issuer.replace(' ', ''),
not_before=not_before.strftime('%Y%m%d'),
not_after=not_after.strftime('%Y%m%d')
issuer=issuer.replace(" ", ""),
not_before=not_before.strftime("%Y%m%d"),
not_after=not_after.strftime("%Y%m%d"),
)
temp = temp.replace('*', "WILDCARD")
temp = temp.replace("*", "WILDCARD")
return text_to_slug(temp)
@ -64,9 +71,9 @@ def common_name(cert):
:return: Common name or None
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_COMMON_NAME
)[0].value.strip()
return cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[
0
].value.strip()
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get common name! {0}".format(e))
@ -79,9 +86,9 @@ def organization(cert):
:return:
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_ORGANIZATION_NAME
)[0].value.strip()
return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[
0
].value.strip()
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get organization! {0}".format(e))
@ -94,9 +101,9 @@ def organizational_unit(cert):
:return:
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_ORGANIZATIONAL_UNIT_NAME
)[0].value.strip()
return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)[
0
].value.strip()
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get organizational unit! {0}".format(e))
@ -109,9 +116,9 @@ def country(cert):
:return:
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_COUNTRY_NAME
)[0].value.strip()
return cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)[
0
].value.strip()
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get country! {0}".format(e))
@ -124,9 +131,9 @@ def state(cert):
:return:
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_STATE_OR_PROVINCE_NAME
)[0].value.strip()
return cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)[
0
].value.strip()
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get state! {0}".format(e))
@ -139,9 +146,9 @@ def location(cert):
:return:
"""
try:
return cert.subject.get_attributes_for_oid(
x509.OID_LOCALITY_NAME
)[0].value.strip()
return cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)[
0
].value.strip()
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get location! {0}".format(e))
@ -219,29 +226,34 @@ def bitstrength(cert):
return cert.public_key().key_size
except AttributeError:
sentry.captureException()
current_app.logger.debug('Unable to get bitstrength.')
current_app.logger.debug("Unable to get bitstrength.")
def issuer(cert):
"""
Gets a sane issuer name from a given certificate.
Gets a sane issuer slug from a given certificate, stripping non-alphanumeric characters.
:param cert:
:return: Issuer
For self-signed certificates, the special value '<selfsigned>' is returned.
If issuer cannot be determined, '<unknown>' is returned.
:param cert: Parsed certificate object
:return: Issuer slug
"""
delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
try:
# Try organization name or fall back to CN
issuer = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME))
issuer = str(issuer[0].value)
for c in delchars:
issuer = issuer.replace(c, "")
return issuer
except Exception as e:
sentry.captureException()
current_app.logger.error("Unable to get issuer! {0}".format(e))
return "Unknown"
# If certificate is self-signed, we return a special value -- there really is no distinct "issuer" for it
if is_selfsigned(cert):
return "<selfsigned>"
# Try Common Name or fall back to Organization name
attrs = cert.issuer.get_attributes_for_oid(
x509.OID_COMMON_NAME
) or cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
if not attrs:
current_app.logger.error(
"Unable to get issuer! Cert serial {:x}".format(cert.serial_number)
)
return "<unknown>"
return text_to_slug(attrs[0].value, "")
def not_before(cert):

272
lemur/common/fields.py
View File

@ -25,6 +25,7 @@ class Hex(Field):
"""
A hex formatted string.
"""
def _serialize(self, value, attr, obj):
if value:
value = hex(int(value))[2:].upper()
@ -48,25 +49,25 @@ class ArrowDateTime(Field):
"""
DATEFORMAT_SERIALIZATION_FUNCS = {
'iso': utils.isoformat,
'iso8601': utils.isoformat,
'rfc': utils.rfcformat,
'rfc822': utils.rfcformat,
"iso": utils.isoformat,
"iso8601": utils.isoformat,
"rfc": utils.rfcformat,
"rfc822": utils.rfcformat,
}
DATEFORMAT_DESERIALIZATION_FUNCS = {
'iso': utils.from_iso,
'iso8601': utils.from_iso,
'rfc': utils.from_rfc,
'rfc822': utils.from_rfc,
"iso": utils.from_iso,
"iso8601": utils.from_iso,
"rfc": utils.from_rfc,
"rfc822": utils.from_rfc,
}
DEFAULT_FORMAT = 'iso'
DEFAULT_FORMAT = "iso"
localtime = False
default_error_messages = {
'invalid': 'Not a valid datetime.',
'format': '"{input}" cannot be formatted as a datetime.',
"invalid": "Not a valid datetime.",
"format": '"{input}" cannot be formatted as a datetime.',
}
def __init__(self, format=None, **kwargs):
@ -89,34 +90,36 @@ class ArrowDateTime(Field):
try:
return format_func(value, localtime=self.localtime)
except (AttributeError, ValueError) as err:
self.fail('format', input=value)
self.fail("format", input=value)
else:
return value.strftime(self.dateformat)
def _deserialize(self, value, attr, data):
if not value: # Falsy values, e.g. '', None, [] are not valid
raise self.fail('invalid')
raise self.fail("invalid")
self.dateformat = self.dateformat or self.DEFAULT_FORMAT
func = self.DATEFORMAT_DESERIALIZATION_FUNCS.get(self.dateformat)
if func:
try:
return arrow.get(func(value))
except (TypeError, AttributeError, ValueError):
raise self.fail('invalid')
raise self.fail("invalid")
elif self.dateformat:
try:
return dt.datetime.strptime(value, self.dateformat)
except (TypeError, AttributeError, ValueError):
raise self.fail('invalid')
raise self.fail("invalid")
elif utils.dateutil_available:
try:
return arrow.get(utils.from_datestring(value))
except TypeError:
raise self.fail('invalid')
raise self.fail("invalid")
else:
warnings.warn('It is recommended that you install python-dateutil '
'for improved datetime deserialization.')
raise self.fail('invalid')
warnings.warn(
"It is recommended that you install python-dateutil "
"for improved datetime deserialization."
)
raise self.fail("invalid")
class KeyUsageExtension(Field):
@ -131,73 +134,75 @@ class KeyUsageExtension(Field):
def _serialize(self, value, attr, obj):
return {
'useDigitalSignature': value.digital_signature,
'useNonRepudiation': value.content_commitment,
'useKeyEncipherment': value.key_encipherment,
'useDataEncipherment': value.data_encipherment,
'useKeyAgreement': value.key_agreement,
'useKeyCertSign': value.key_cert_sign,
'useCRLSign': value.crl_sign,
'useEncipherOnly': value._encipher_only,
'useDecipherOnly': value._decipher_only
"useDigitalSignature": value.digital_signature,
"useNonRepudiation": value.content_commitment,
"useKeyEncipherment": value.key_encipherment,
"useDataEncipherment": value.data_encipherment,
"useKeyAgreement": value.key_agreement,
"useKeyCertSign": value.key_cert_sign,
"useCRLSign": value.crl_sign,
"useEncipherOnly": value._encipher_only,
"useDecipherOnly": value._decipher_only,
}
def _deserialize(self, value, attr, data):
keyusages = {
'digital_signature': False,
'content_commitment': False,
'key_encipherment': False,
'data_encipherment': False,
'key_agreement': False,
'key_cert_sign': False,
'crl_sign': False,
'encipher_only': False,
'decipher_only': False
"digital_signature": False,
"content_commitment": False,
"key_encipherment": False,
"data_encipherment": False,
"key_agreement": False,
"key_cert_sign": False,
"crl_sign": False,
"encipher_only": False,
"decipher_only": False,
}
for k, v in value.items():
if k == 'useDigitalSignature':
keyusages['digital_signature'] = v
if k == "useDigitalSignature":
keyusages["digital_signature"] = v
elif k == 'useNonRepudiation':
keyusages['content_commitment'] = v
elif k == "useNonRepudiation":
keyusages["content_commitment"] = v
elif k == 'useKeyEncipherment':
keyusages['key_encipherment'] = v
elif k == "useKeyEncipherment":
keyusages["key_encipherment"] = v
elif k == 'useDataEncipherment':
keyusages['data_encipherment'] = v
elif k == "useDataEncipherment":
keyusages["data_encipherment"] = v
elif k == 'useKeyCertSign':
keyusages['key_cert_sign'] = v
elif k == "useKeyCertSign":
keyusages["key_cert_sign"] = v
elif k == 'useCRLSign':
keyusages['crl_sign'] = v
elif k == "useCRLSign":
keyusages["crl_sign"] = v
elif k == 'useKeyAgreement':
keyusages['key_agreement'] = v
elif k == "useKeyAgreement":
keyusages["key_agreement"] = v
elif k == 'useEncipherOnly' and v:
keyusages['encipher_only'] = True
keyusages['key_agreement'] = True
elif k == "useEncipherOnly" and v:
keyusages["encipher_only"] = True
keyusages["key_agreement"] = True
elif k == 'useDecipherOnly' and v:
keyusages['decipher_only'] = True
keyusages['key_agreement'] = True
elif k == "useDecipherOnly" and v:
keyusages["decipher_only"] = True
keyusages["key_agreement"] = True
if keyusages['encipher_only'] and keyusages['decipher_only']:
raise ValidationError('A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages.')
if keyusages["encipher_only"] and keyusages["decipher_only"]:
raise ValidationError(
"A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages."
)
return x509.KeyUsage(
digital_signature=keyusages['digital_signature'],
content_commitment=keyusages['content_commitment'],
key_encipherment=keyusages['key_encipherment'],
data_encipherment=keyusages['data_encipherment'],
key_agreement=keyusages['key_agreement'],
key_cert_sign=keyusages['key_cert_sign'],
crl_sign=keyusages['crl_sign'],
encipher_only=keyusages['encipher_only'],
decipher_only=keyusages['decipher_only']
digital_signature=keyusages["digital_signature"],
content_commitment=keyusages["content_commitment"],
key_encipherment=keyusages["key_encipherment"],
data_encipherment=keyusages["data_encipherment"],
key_agreement=keyusages["key_agreement"],
key_cert_sign=keyusages["key_cert_sign"],
crl_sign=keyusages["crl_sign"],
encipher_only=keyusages["encipher_only"],
decipher_only=keyusages["decipher_only"],
)
@ -216,69 +221,77 @@ class ExtendedKeyUsageExtension(Field):
usage_list = {}
for usage in usages:
if usage == x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH:
usage_list['useClientAuthentication'] = True
usage_list["useClientAuthentication"] = True
elif usage == x509.oid.ExtendedKeyUsageOID.SERVER_AUTH:
usage_list['useServerAuthentication'] = True
usage_list["useServerAuthentication"] = True
elif usage == x509.oid.ExtendedKeyUsageOID.CODE_SIGNING:
usage_list['useCodeSigning'] = True
usage_list["useCodeSigning"] = True
elif usage == x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION:
usage_list['useEmailProtection'] = True
usage_list["useEmailProtection"] = True
elif usage == x509.oid.ExtendedKeyUsageOID.TIME_STAMPING:
usage_list['useTimestamping'] = True
usage_list["useTimestamping"] = True
elif usage == x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING:
usage_list['useOCSPSigning'] = True
usage_list["useOCSPSigning"] = True
elif usage.dotted_string == '1.3.6.1.5.5.7.3.14':
usage_list['useEapOverLAN'] = True
elif usage.dotted_string == "1.3.6.1.5.5.7.3.14":
usage_list["useEapOverLAN"] = True
elif usage.dotted_string == '1.3.6.1.5.5.7.3.13':
usage_list['useEapOverPPP'] = True
elif usage.dotted_string == "1.3.6.1.5.5.7.3.13":
usage_list["useEapOverPPP"] = True
elif usage.dotted_string == '1.3.6.1.4.1.311.20.2.2':
usage_list['useSmartCardLogon'] = True
elif usage.dotted_string == "1.3.6.1.4.1.311.20.2.2":
usage_list["useSmartCardLogon"] = True
else:
current_app.logger.warning('Unable to serialize ExtendedKeyUsage with OID: {usage}'.format(usage=usage.dotted_string))
current_app.logger.warning(
"Unable to serialize ExtendedKeyUsage with OID: {usage}".format(
usage=usage.dotted_string
)
)
return usage_list
def _deserialize(self, value, attr, data):
usage_oids = []
for k, v in value.items():
if k == 'useClientAuthentication' and v:
if k == "useClientAuthentication" and v:
usage_oids.append(x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH)
elif k == 'useServerAuthentication' and v:
elif k == "useServerAuthentication" and v:
usage_oids.append(x509.oid.ExtendedKeyUsageOID.SERVER_AUTH)
elif k == 'useCodeSigning' and v:
elif k == "useCodeSigning" and v:
usage_oids.append(x509.oid.ExtendedKeyUsageOID.CODE_SIGNING)
elif k == 'useEmailProtection' and v:
elif k == "useEmailProtection" and v:
usage_oids.append(x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION)
elif k == 'useTimestamping' and v:
elif k == "useTimestamping" and v:
usage_oids.append(x509.oid.ExtendedKeyUsageOID.TIME_STAMPING)
elif k == 'useOCSPSigning' and v:
elif k == "useOCSPSigning" and v:
usage_oids.append(x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING)
elif k == 'useEapOverLAN' and v:
elif k == "useEapOverLAN" and v:
usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.14"))
elif k == 'useEapOverPPP' and v:
elif k == "useEapOverPPP" and v:
usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.13"))
elif k == 'useSmartCardLogon' and v:
elif k == "useSmartCardLogon" and v:
usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.4.1.311.20.2.2"))
else:
current_app.logger.warning('Unable to deserialize ExtendedKeyUsage with name: {key}'.format(key=k))
current_app.logger.warning(
"Unable to deserialize ExtendedKeyUsage with name: {key}".format(
key=k
)
)
return x509.ExtendedKeyUsage(usage_oids)
@ -294,15 +307,17 @@ class BasicConstraintsExtension(Field):
"""
def _serialize(self, value, attr, obj):
return {'ca': value.ca, 'path_length': value.path_length}
return {"ca": value.ca, "path_length": value.path_length}
def _deserialize(self, value, attr, data):
ca = value.get('ca', False)
path_length = value.get('path_length', None)
ca = value.get("ca", False)
path_length = value.get("path_length", None)
if ca:
if not isinstance(path_length, (type(None), int)):
raise ValidationError('A CA certificate path_length (for BasicConstraints) must be None or an integer.')
raise ValidationError(
"A CA certificate path_length (for BasicConstraints) must be None or an integer."
)
return x509.BasicConstraints(ca=True, path_length=path_length)
else:
return x509.BasicConstraints(ca=False, path_length=None)
@ -317,6 +332,7 @@ class SubjectAlternativeNameExtension(Field):
:param kwargs: The same keyword arguments that :class:`Field` receives.
"""
def _serialize(self, value, attr, obj):
general_names = []
name_type = None
@ -326,53 +342,59 @@ class SubjectAlternativeNameExtension(Field):
value = name.value
if isinstance(name, x509.DNSName):
name_type = 'DNSName'
name_type = "DNSName"
elif isinstance(name, x509.IPAddress):
if isinstance(value, ipaddress.IPv4Network):
name_type = 'IPNetwork'
name_type = "IPNetwork"
else:
name_type = 'IPAddress'
name_type = "IPAddress"
value = str(value)
elif isinstance(name, x509.UniformResourceIdentifier):
name_type = 'uniformResourceIdentifier'
name_type = "uniformResourceIdentifier"
elif isinstance(name, x509.DirectoryName):
name_type = 'directoryName'
name_type = "directoryName"
elif isinstance(name, x509.RFC822Name):
name_type = 'rfc822Name'
name_type = "rfc822Name"
elif isinstance(name, x509.RegisteredID):
name_type = 'registeredID'
name_type = "registeredID"
value = value.dotted_string
else:
current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name))
current_app.logger.warning(
"Unknown SubAltName type: {name}".format(name=name)
)
continue
general_names.append({'nameType': name_type, 'value': value})
general_names.append({"nameType": name_type, "value": value})
return general_names
def _deserialize(self, value, attr, data):
general_names = []
for name in value:
if name['nameType'] == 'DNSName':
validators.sensitive_domain(name['value'])
general_names.append(x509.DNSName(name['value']))
if name["nameType"] == "DNSName":
validators.sensitive_domain(name["value"])
general_names.append(x509.DNSName(name["value"]))
elif name['nameType'] == 'IPAddress':
general_names.append(x509.IPAddress(ipaddress.ip_address(name['value'])))
elif name["nameType"] == "IPAddress":
general_names.append(
x509.IPAddress(ipaddress.ip_address(name["value"]))
)
elif name['nameType'] == 'IPNetwork':
general_names.append(x509.IPAddress(ipaddress.ip_network(name['value'])))
elif name["nameType"] == "IPNetwork":
general_names.append(
x509.IPAddress(ipaddress.ip_network(name["value"]))
)
elif name['nameType'] == 'uniformResourceIdentifier':
general_names.append(x509.UniformResourceIdentifier(name['value']))
elif name["nameType"] == "uniformResourceIdentifier":
general_names.append(x509.UniformResourceIdentifier(name["value"]))
elif name['nameType'] == 'directoryName':
elif name["nameType"] == "directoryName":
# TODO: Need to parse a string in name['value'] like:
# 'CN=Common Name, O=Org Name, OU=OrgUnit Name, C=US, ST=ST, L=City/emailAddress=person@example.com'
# or
@ -390,26 +412,32 @@ class SubjectAlternativeNameExtension(Field):
# general_names.append(x509.DirectoryName(x509.Name(BLAH))))
pass
elif name['nameType'] == 'rfc822Name':
general_names.append(x509.RFC822Name(name['value']))
elif name["nameType"] == "rfc822Name":
general_names.append(x509.RFC822Name(name["value"]))
elif name['nameType'] == 'registeredID':
general_names.append(x509.RegisteredID(x509.ObjectIdentifier(name['value'])))
elif name["nameType"] == "registeredID":
general_names.append(
x509.RegisteredID(x509.ObjectIdentifier(name["value"]))
)
elif name['nameType'] == 'otherName':
elif name["nameType"] == "otherName":
# This has two inputs (type and value), so it doesn't fit the mold of the rest of these GeneralName entities.
# general_names.append(x509.OtherName(name['type'], bytes(name['value']), 'utf-8'))
pass
elif name['nameType'] == 'x400Address':
elif name["nameType"] == "x400Address":
# The Python Cryptography library doesn't support x400Address types (yet?)
pass
elif name['nameType'] == 'EDIPartyName':
elif name["nameType"] == "EDIPartyName":
# The Python Cryptography library doesn't support EDIPartyName types (yet?)
pass
else:
current_app.logger.warning('Unable to deserialize SubAltName with type: {name_type}'.format(name_type=name['nameType']))
current_app.logger.warning(
"Unable to deserialize SubAltName with type: {name_type}".format(
name_type=name["nameType"]
)
)
return x509.SubjectAlternativeName(general_names)

10
lemur/common/health.py
View File

@ -10,20 +10,20 @@ from flask import Blueprint
from lemur.database import db
from lemur.extensions import sentry
mod = Blueprint('healthCheck', __name__)
mod = Blueprint("healthCheck", __name__)
@mod.route('/healthcheck')
@mod.route("/healthcheck")
def health():
try:
if healthcheck(db):
return 'ok'
return "ok"
except Exception:
sentry.captureException()
return 'db check failed'
return "db check failed"
def healthcheck(db):
with db.engine.connect() as connection:
connection.execute('SELECT 1;')
connection.execute("SELECT 1;")
return True

10
lemur/common/managers.py
View File

@ -52,7 +52,7 @@ class InstanceManager(object):
results = []
for cls_path in class_list:
module_name, class_name = cls_path.rsplit('.', 1)
module_name, class_name = cls_path.rsplit(".", 1)
try:
module = __import__(module_name, {}, {}, class_name)
cls = getattr(module, class_name)
@ -62,10 +62,14 @@ class InstanceManager(object):
results.append(cls)
except InvalidConfiguration as e:
current_app.logger.warning("Plugin '{0}' may not work correctly. {1}".format(class_name, e))
current_app.logger.warning(
"Plugin '{0}' may not work correctly. {1}".format(class_name, e)
)
except Exception as e:
current_app.logger.exception("Unable to import {0}. Reason: {1}".format(cls_path, e))
current_app.logger.exception(
"Unable to import {0}. Reason: {1}".format(cls_path, e)
)
continue
self.cache = results

13
lemur/common/missing.py
View File

@ -11,14 +11,15 @@ def convert_validity_years(data):
:param data:
:return:
"""
if data.get('validity_years'):
if data.get("validity_years"):
now = arrow.utcnow()
data['validity_start'] = now.isoformat()
data["validity_start"] = now.isoformat()
end = now.replace(years=+int(data['validity_years']))
if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
end = now.shift(years=+int(data["validity_years"]))
if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True):
if is_weekend(end):
end = end.replace(days=-2)
end = end.shift(days=-2)
data['validity_end'] = end.isoformat()
data["validity_end"] = end.isoformat()
return data

52
lemur/common/redis.py Normal file
View File

@ -0,0 +1,52 @@
"""
Helper Class for Redis
"""
import redis
import sys
from flask import current_app
from lemur.extensions import sentry
from lemur.factory import create_app
if current_app:
flask_app = current_app
else:
flask_app = create_app()
class RedisHandler:
def __init__(self, host=flask_app.config.get('REDIS_HOST', 'localhost'),
port=flask_app.config.get('REDIS_PORT', 6379),
db=flask_app.config.get('REDIS_DB', 0)):
self.host = host
self.port = port
self.db = db
def redis(self, db=0):
# The decode_responses flag here directs the client to convert the responses from Redis into Python strings
# using the default encoding utf-8. This is client specific.
function = f"{__name__}.{sys._getframe().f_code.co_name}"
try:
red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True)
red.set("test", 0)
except redis.ConnectionError:
log_data = {
"function": function,
"message": "Redis Connection error",
"host": self.host,
"port": self.port
}
current_app.logger.error(log_data)
sentry.captureException()
return red
def redis_get(key, default=None):
red = RedisHandler().redis()
try:
v = red.get(key)
except redis.exceptions.ConnectionError:
v = None
if not v:
return default
return v

65
lemur/common/schema.py
View File

@ -22,27 +22,26 @@ class LemurSchema(Schema):
"""
Base schema from which all grouper schema's inherit
"""
__envelope__ = True
def under(self, data, many=None):
items = []
if many:
for i in data:
items.append(
{underscore(key): value for key, value in i.items()}
)
items.append({underscore(key): value for key, value in i.items()})
return items
return {
underscore(key): value
for key, value in data.items()
}
return {underscore(key): value for key, value in data.items()}
def camel(self, data, many=None):
items = []
if many:
for i in data:
items.append(
{camelize(key, uppercase_first_letter=False): value for key, value in i.items()}
{
camelize(key, uppercase_first_letter=False): value
for key, value in i.items()
}
)
return items
return {
@ -52,16 +51,16 @@ class LemurSchema(Schema):
def wrap_with_envelope(self, data, many):
if many:
if 'total' in self.context.keys():
return dict(total=self.context['total'], items=data)
if "total" in self.context.keys():
return dict(total=self.context["total"], items=data)
return data
class LemurInputSchema(LemurSchema):
@pre_load(pass_many=True)
def preprocess(self, data, many):
if isinstance(data, dict) and data.get('owner'):
data['owner'] = data['owner'].lower()
if isinstance(data, dict) and data.get("owner"):
data["owner"] = data["owner"].lower()
return self.under(data, many=many)
@ -74,17 +73,17 @@ class LemurOutputSchema(LemurSchema):
def unwrap_envelope(self, data, many):
if many:
if data['items']:
if data["items"]:
if isinstance(data, InstrumentedList) or isinstance(data, list):
self.context['total'] = len(data)
self.context["total"] = len(data)
return data
else:
self.context['total'] = data['total']
self.context["total"] = data["total"]
else:
self.context['total'] = 0
data = {'items': []}
self.context["total"] = 0
data = {"items": []}
return data['items']
return data["items"]
return data
@ -110,11 +109,11 @@ def format_errors(messages):
def wrap_errors(messages):
errors = dict(message='Validation Error.')
if messages.get('_schema'):
errors['reasons'] = {'Schema': {'rule': messages['_schema']}}
errors = dict(message="Validation Error.")
if messages.get("_schema"):
errors["reasons"] = {"Schema": {"rule": messages["_schema"]}}
else:
errors['reasons'] = format_errors(messages)
errors["reasons"] = format_errors(messages)
return errors
@ -123,19 +122,19 @@ def unwrap_pagination(data, output_schema):
return data
if isinstance(data, dict):
if 'total' in data.keys():
if data.get('total') == 0:
if "total" in data.keys():
if data.get("total") == 0:
return data
marshaled_data = {'total': data['total']}
marshaled_data['items'] = output_schema.dump(data['items'], many=True).data
marshaled_data = {"total": data["total"]}
marshaled_data["items"] = output_schema.dump(data["items"], many=True).data
return marshaled_data
return output_schema.dump(data).data
elif isinstance(data, list):
marshaled_data = {'total': len(data)}
marshaled_data['items'] = output_schema.dump(data, many=True).data
marshaled_data = {"total": len(data)}
marshaled_data["items"] = output_schema.dump(data, many=True).data
return marshaled_data
return output_schema.dump(data).data
@ -155,7 +154,7 @@ def validate_schema(input_schema, output_schema):
if errors:
return wrap_errors(errors), 400
kwargs['data'] = data
kwargs["data"] = data
try:
resp = f(*args, **kwargs)
@ -170,7 +169,13 @@ def validate_schema(input_schema, output_schema):
if not resp:
return dict(message="No data found"), 404
return unwrap_pagination(resp, output_schema), 200
if callable(output_schema):
output_schema_to_use = output_schema()
else:
output_schema_to_use = output_schema
return unwrap_pagination(resp, output_schema_to_use), 200
return decorated_function
return decorator

180
lemur/common/utils.py
View File

@ -7,12 +7,16 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import random
import re
import string
import sqlalchemy
from cryptography import x509
from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa, ec
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from flask_restful.reqparse import RequestParser
from sqlalchemy import and_, func
@ -21,21 +25,22 @@ from lemur.exceptions import InvalidConfiguration
paginated_parser = RequestParser()
paginated_parser.add_argument('count', type=int, default=10, location='args')
paginated_parser.add_argument('page', type=int, default=1, location='args')
paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
paginated_parser.add_argument('filter', type=str, location='args')
paginated_parser.add_argument("count", type=int, default=10, location="args")
paginated_parser.add_argument("page", type=int, default=1, location="args")
paginated_parser.add_argument("sortDir", type=str, dest="sort_dir", location="args")
paginated_parser.add_argument("sortBy", type=str, dest="sort_by", location="args")
paginated_parser.add_argument("filter", type=str, location="args")
paginated_parser.add_argument("owner", type=str, location="args")
def get_psuedo_random_string():
"""
Create a random and strongish challenge.
"""
challenge = ''.join(random.choice(string.ascii_uppercase) for x in range(6)) # noqa
challenge += ''.join(random.choice("~!@#$%^&*()_+") for x in range(6)) # noqa
challenge += ''.join(random.choice(string.ascii_lowercase) for x in range(6))
challenge += ''.join(random.choice(string.digits) for x in range(6)) # noqa
challenge = "".join(random.choice(string.ascii_uppercase) for x in range(6)) # noqa
challenge += "".join(random.choice("~!@#$%^&*()_+") for x in range(6)) # noqa
challenge += "".join(random.choice(string.ascii_lowercase) for x in range(6))
challenge += "".join(random.choice(string.digits) for x in range(6)) # noqa
return challenge
@ -46,10 +51,46 @@ def parse_certificate(body):
:param body:
:return:
"""
if isinstance(body, str):
body = body.encode('utf-8')
assert isinstance(body, str)
return x509.load_pem_x509_certificate(body, default_backend())
return x509.load_pem_x509_certificate(body.encode("utf-8"), default_backend())
def parse_private_key(private_key):
"""
Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm).
Raises ValueError for an invalid string. Raises AssertionError when passed value is not str-type.
:param private_key: String containing PEM private key
"""
assert isinstance(private_key, str)
return load_pem_private_key(
private_key.encode("utf8"), password=None, backend=default_backend()
)
def split_pem(data):
"""
Split a string of several PEM payloads to a list of strings.
:param data: String
:return: List of strings
"""
return re.split("\n(?=-----BEGIN )", data)
def parse_cert_chain(pem_chain):
"""
Helper function to split and parse a series of PEM certificates.
:param pem_chain: string
:return: List of parsed certificates
"""
if pem_chain is None:
return []
return [parse_certificate(cert) for cert in split_pem(pem_chain) if cert]
def parse_csr(csr):
@ -59,17 +100,17 @@ def parse_csr(csr):
:param csr:
:return:
"""
if isinstance(csr, str):
csr = csr.encode('utf-8')
assert isinstance(csr, str)
return x509.load_pem_x509_csr(csr, default_backend())
return x509.load_pem_x509_csr(csr.encode("utf-8"), default_backend())
def get_authority_key(body):
"""Returns the authority key for a given certificate in hex format"""
parsed_cert = parse_certificate(body)
authority_key = parsed_cert.extensions.get_extension_for_class(
x509.AuthorityKeyIdentifier).value.key_identifier
x509.AuthorityKeyIdentifier
).value.key_identifier
return authority_key.hex()
@ -89,20 +130,17 @@ def generate_private_key(key_type):
_CURVE_TYPES = {
"ECCPRIME192V1": ec.SECP192R1(),
"ECCPRIME256V1": ec.SECP256R1(),
"ECCSECP192R1": ec.SECP192R1(),
"ECCSECP224R1": ec.SECP224R1(),
"ECCSECP256R1": ec.SECP256R1(),
"ECCSECP384R1": ec.SECP384R1(),
"ECCSECP521R1": ec.SECP521R1(),
"ECCSECP256K1": ec.SECP256K1(),
"ECCSECT163K1": ec.SECT163K1(),
"ECCSECT233K1": ec.SECT233K1(),
"ECCSECT283K1": ec.SECT283K1(),
"ECCSECT409K1": ec.SECT409K1(),
"ECCSECT571K1": ec.SECT571K1(),
"ECCSECT163R2": ec.SECT163R2(),
"ECCSECT233R1": ec.SECT233R1(),
"ECCSECT283R1": ec.SECT283R1(),
@ -111,25 +149,74 @@ def generate_private_key(key_type):
}
if key_type not in CERTIFICATE_KEY_TYPES:
raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format(
key_type=key_type,
choices=",".join(CERTIFICATE_KEY_TYPES)
))
raise Exception(
"Invalid key type: {key_type}. Supported key types: {choices}".format(
key_type=key_type, choices=",".join(CERTIFICATE_KEY_TYPES)
)
)
if 'RSA' in key_type:
if "RSA" in key_type:
key_size = int(key_type[3:])
return rsa.generate_private_key(
public_exponent=65537,
key_size=key_size,
backend=default_backend()
public_exponent=65537, key_size=key_size, backend=default_backend()
)
elif 'ECC' in key_type:
elif "ECC" in key_type:
return ec.generate_private_key(
curve=_CURVE_TYPES[key_type],
backend=default_backend()
curve=_CURVE_TYPES[key_type], backend=default_backend()
)
def check_cert_signature(cert, issuer_public_key):
"""
Check a certificate's signature against an issuer public key.
Before EC validation, make sure we support the algorithm, otherwise raise UnsupportedAlgorithm
On success, returns None; on failure, raises UnsupportedAlgorithm or InvalidSignature.
"""
if isinstance(issuer_public_key, rsa.RSAPublicKey):
# RSA requires padding, just to make life difficult for us poor developers :(
if cert.signature_algorithm_oid == x509.SignatureAlgorithmOID.RSASSA_PSS:
# In 2005, IETF devised a more secure padding scheme to replace PKCS #1 v1.5. To make sure that
# nobody can easily support or use it, they mandated lots of complicated parameters, unlike any
# other X.509 signature scheme.
# https://tools.ietf.org/html/rfc4056
raise UnsupportedAlgorithm("RSASSA-PSS not supported")
else:
padder = padding.PKCS1v15()
issuer_public_key.verify(
cert.signature,
cert.tbs_certificate_bytes,
padder,
cert.signature_hash_algorithm,
)
elif isinstance(issuer_public_key, ec.EllipticCurvePublicKey) and isinstance(
ec.ECDSA(cert.signature_hash_algorithm), ec.ECDSA
):
issuer_public_key.verify(
cert.signature,
cert.tbs_certificate_bytes,
ec.ECDSA(cert.signature_hash_algorithm),
)
else:
raise UnsupportedAlgorithm(
"Unsupported Algorithm '{var}'.".format(
var=cert.signature_algorithm_oid._name
)
)
def is_selfsigned(cert):
"""
Returns True if the certificate is self-signed.
Returns False for failed verification or unsupported signing algorithm.
"""
try:
check_cert_signature(cert, cert.public_key())
# If verification was successful, it's self-signed.
return True
except InvalidSignature:
return False
def is_weekend(date):
"""
Determines if a given date is on a weekend.
@ -150,7 +237,9 @@ def validate_conf(app, required_vars):
"""
for var in required_vars:
if var not in app.config:
raise InvalidConfiguration("Required variable '{var}' is not set in Lemur's conf.".format(var=var))
raise InvalidConfiguration(
"Required variable '{var}' is not set in Lemur's conf.".format(var=var)
)
# https://bitbucket.org/zzzeek/sqlalchemy/wiki/UsageRecipes/WindowedRangeQuery
@ -169,18 +258,15 @@ def column_windows(session, column, windowsize):
be computed.
"""
def int_for_range(start_id, end_id):
if end_id:
return and_(
column >= start_id,
column < end_id
)
return and_(column >= start_id, column < end_id)
else:
return column >= start_id
q = session.query(
column,
func.row_number().over(order_by=column).label('rownum')
column, func.row_number().over(order_by=column).label("rownum")
).from_self(column)
if windowsize > 1:
@ -200,9 +286,7 @@ def column_windows(session, column, windowsize):
def windowed_query(q, column, windowsize):
""""Break a Query into windows on a given column."""
for whereclause in column_windows(
q.session,
column, windowsize):
for whereclause in column_windows(q.session, column, windowsize):
for row in q.filter(whereclause).order_by(column):
yield row
@ -210,4 +294,16 @@ def windowed_query(q, column, windowsize):
def truthiness(s):
"""If input string resembles something truthy then return True, else False."""
return s.lower() in ('true', 'yes', 'on', 't', '1')
return s.lower() in ("true", "yes", "on", "t", "1")
def find_matching_certificates_by_hash(cert, matching_certs):
"""Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
determine if any of the certificate hashes match and return the matches."""
matching = []
for c in matching_certs:
if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(
hashes.SHA256()
):
matching.append(c)
return matching

183
lemur/common/validators.py
View File

@ -1,45 +1,14 @@
import re
from cryptography import x509
from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.x509 import NameOID
from flask import current_app
from marshmallow.exceptions import ValidationError
from lemur.auth.permissions import SensitiveDomainPermission
from lemur.common.utils import parse_certificate, is_weekend
from lemur.domains import service as domain_service
def public_certificate(body):
"""
Determines if specified string is valid public certificate.
:param body:
:return:
"""
try:
parse_certificate(body)
except Exception as e:
current_app.logger.exception(e)
raise ValidationError('Public certificate presented is not valid.')
def private_key(key):
"""
User to validate that a given string is a RSA private key
:param key:
:return: :raise ValueError:
"""
try:
if isinstance(key, bytes):
serialization.load_pem_private_key(key, None, backend=default_backend())
else:
serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
except Exception:
raise ValidationError('Private key presented is not valid.')
from lemur.common.utils import check_cert_signature, is_weekend
def common_name(value):
@ -47,7 +16,7 @@ def common_name(value):
# Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
# certificates). As a simple heuristic, we assume that human-readable names always include a space.
# However, to avoid confusion for humans, we also don't count spaces at the beginning or end of the string.
if ' ' not in value.strip():
if " " not in value.strip():
return sensitive_domain(value)
@ -61,14 +30,21 @@ def sensitive_domain(domain):
# User has permission, no need to check anything
return
whitelist = current_app.config.get('LEMUR_WHITELISTED_DOMAINS', [])
whitelist = current_app.config.get("LEMUR_WHITELISTED_DOMAINS", [])
if whitelist and not any(re.match(pattern, domain) for pattern in whitelist):
raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
'Contact an administrator to issue the certificate.'.format(domain))
raise ValidationError(
"Domain {0} does not match whitelisted domain patterns. "
"Contact an administrator to issue the certificate.".format(domain)
)
if any(d.sensitive for d in domain_service.get_by_name(domain)):
raise ValidationError('Domain {0} has been marked as sensitive. '
'Contact an administrator to issue the certificate.'.format(domain))
# Avoid circular import.
from lemur.domains import service as domain_service
if domain_service.is_domain_sensitive(domain):
raise ValidationError(
"Domain {0} has been marked as sensitive. "
"Contact an administrator to issue the certificate.".format(domain)
)
def encoding(oid_encoding):
@ -77,9 +53,13 @@ def encoding(oid_encoding):
:param oid_encoding:
:return:
"""
valid_types = ['b64asn1', 'string', 'ia5string']
valid_types = ["b64asn1", "string", "ia5string"]
if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]:
raise ValidationError('Invalid Oid Encoding: {0} choose from {1}'.format(oid_encoding, ",".join(valid_types)))
raise ValidationError(
"Invalid Oid Encoding: {0} choose from {1}".format(
oid_encoding, ",".join(valid_types)
)
)
def sub_alt_type(alt_type):
@ -88,10 +68,23 @@ def sub_alt_type(alt_type):
:param alt_type:
:return:
"""
valid_types = ['DNSName', 'IPAddress', 'uniFormResourceIdentifier', 'directoryName', 'rfc822Name', 'registrationID',
'otherName', 'x400Address', 'EDIPartyName']
valid_types = [
"DNSName",
"IPAddress",
"uniFormResourceIdentifier",
"directoryName",
"rfc822Name",
"registrationID",
"otherName",
"x400Address",
"EDIPartyName",
]
if alt_type.lower() not in [a_type.lower() for a_type in valid_types]:
raise ValidationError('Invalid SubAltName Type: {0} choose from {1}'.format(type, ",".join(valid_types)))
raise ValidationError(
"Invalid SubAltName Type: {0} choose from {1}".format(
type, ",".join(valid_types)
)
)
def csr(data):
@ -101,16 +94,18 @@ def csr(data):
:return:
"""
try:
request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
except Exception:
raise ValidationError('CSR presented is not valid.')
raise ValidationError("CSR presented is not valid.")
# Validate common name and SubjectAltNames
for name in request.subject.get_attributes_for_oid(NameOID.COMMON_NAME):
common_name(name.value)
try:
alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
alt_names = request.extensions.get_extension_for_class(
x509.SubjectAlternativeName
)
for name in alt_names.value.get_values_for_type(x509.DNSName):
sensitive_domain(name)
@ -119,25 +114,87 @@ def csr(data):
def dates(data):
if not data.get('validity_start') and data.get('validity_end'):
raise ValidationError('If validity start is specified so must validity end.')
if not data.get("validity_start") and data.get("validity_end"):
raise ValidationError("If validity start is specified so must validity end.")
if not data.get('validity_end') and data.get('validity_start'):
raise ValidationError('If validity end is specified so must validity start.')
if not data.get("validity_end") and data.get("validity_start"):
raise ValidationError("If validity end is specified so must validity start.")
if data.get('validity_start') and data.get('validity_end'):
if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
if is_weekend(data.get('validity_end')):
raise ValidationError('Validity end must not land on a weekend.')
if data.get("validity_start") and data.get("validity_end"):
if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True):
if is_weekend(data.get("validity_end")):
raise ValidationError("Validity end must not land on a weekend.")
if not data['validity_start'] < data['validity_end']:
raise ValidationError('Validity start must be before validity end.')
if not data["validity_start"] < data["validity_end"]:
raise ValidationError("Validity start must be before validity end.")
if data.get('authority'):
if data.get('validity_start').date() < data['authority'].authority_certificate.not_before.date():
raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
if data.get("authority"):
if (
data.get("validity_start").date()
< data["authority"].authority_certificate.not_before.date()
):
raise ValidationError(
"Validity start must not be before {0}".format(
data["authority"].authority_certificate.not_before
)
)
if data.get('validity_end').date() > data['authority'].authority_certificate.not_after.date():
raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
if (
data.get("validity_end").date()
> data["authority"].authority_certificate.not_after.date()
):
raise ValidationError(
"Validity end must not be after {0}".format(
data["authority"].authority_certificate.not_after
)
)
return data
def verify_private_key_match(key, cert, error_class=ValidationError):
"""
Checks that the supplied private key matches the certificate.
:param cert: Parsed certificate
:param key: Parsed private key
:param error_class: Exception class to raise on error
"""
if key.public_key().public_numbers() != cert.public_key().public_numbers():
raise error_class("Private key does not match certificate.")
def verify_cert_chain(certs, error_class=ValidationError):
"""
Verifies that the certificates in the chain are correct.
We don't bother with full cert validation but just check that certs in the chain are signed by the next, to avoid
basic human errors -- such as pasting the wrong certificate.
:param certs: List of parsed certificates, use parse_cert_chain()
:param error_class: Exception class to raise on error
"""
cert = certs[0]
for issuer in certs[1:]:
# Use the current cert's public key to verify the previous signature.
# "certificate validation is a complex problem that involves much more than just signature checks"
try:
check_cert_signature(cert, issuer.public_key())
except InvalidSignature:
# Avoid circular import.
from lemur.common import defaults
raise error_class(
"Incorrect chain certificate(s) provided: '%s' is not signed by '%s'"
% (
defaults.common_name(cert) or "Unknown",
defaults.common_name(issuer),
)
)
except UnsupportedAlgorithm as err:
current_app.logger.warning("Skipping chain validation: %s", err)
# Next loop will validate that *this issuer* cert is signed by the next chain cert.
cert = issuer

44
lemur/constants.py
View File

@ -7,28 +7,28 @@ SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"
DEFAULT_NAMING_TEMPLATE = "{subject}-{issuer}-{not_before}-{not_after}"
NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
SUCCESS_METRIC_STATUS = 'success'
FAILURE_METRIC_STATUS = 'failure'
SUCCESS_METRIC_STATUS = "success"
FAILURE_METRIC_STATUS = "failure"
CERTIFICATE_KEY_TYPES = [
'RSA2048',
'RSA4096',
'ECCPRIME192V1',
'ECCPRIME256V1',
'ECCSECP192R1',
'ECCSECP224R1',
'ECCSECP256R1',
'ECCSECP384R1',
'ECCSECP521R1',
'ECCSECP256K1',
'ECCSECT163K1',
'ECCSECT233K1',
'ECCSECT283K1',
'ECCSECT409K1',
'ECCSECT571K1',
'ECCSECT163R2',
'ECCSECT233R1',
'ECCSECT283R1',
'ECCSECT409R1',
'ECCSECT571R2'
"RSA2048",
"RSA4096",
"ECCPRIME192V1",
"ECCPRIME256V1",
"ECCSECP192R1",
"ECCSECP224R1",
"ECCSECP256R1",
"ECCSECP384R1",
"ECCSECP521R1",
"ECCSECP256K1",
"ECCSECT163K1",
"ECCSECT233K1",
"ECCSECT283K1",
"ECCSECT409K1",
"ECCSECT571K1",
"ECCSECT163R2",
"ECCSECT233R1",
"ECCSECT283R1",
"ECCSECT409R1",
"ECCSECT571R2",
]

36
lemur/database.py
View File

@ -43,7 +43,7 @@ def session_query(model):
:param model: sqlalchemy model
:return: query object for model
"""
return model.query if hasattr(model, 'query') else db.session.query(model)
return model.query if hasattr(model, "query") else db.session.query(model)
def create_query(model, kwargs):
@ -77,7 +77,7 @@ def add(model):
def get_model_column(model, field):
if field in getattr(model, 'sensitive_fields', ()):
if field in getattr(model, "sensitive_fields", ()):
raise AttrNotFound(field)
column = model.__table__.columns._data.get(field, None)
if column is None:
@ -100,7 +100,7 @@ def find_all(query, model, kwargs):
kwargs = filter_none(kwargs)
for attr, value in kwargs.items():
if not isinstance(value, list):
value = value.split(',')
value = value.split(",")
conditions.append(get_model_column(model, attr).in_(value))
@ -200,7 +200,7 @@ def filter(query, model, terms):
:return:
"""
column = get_model_column(model, underscore(terms[0]))
return query.filter(column.ilike('%{}%'.format(terms[1])))
return query.filter(column.ilike("%{}%".format(terms[1])))
def sort(query, model, field, direction):
@ -214,7 +214,7 @@ def sort(query, model, field, direction):
:param direction:
"""
column = get_model_column(model, underscore(field))
return query.order_by(column.desc() if direction == 'desc' else column.asc())
return query.order_by(column.desc() if direction == "desc" else column.asc())
def paginate(query, page, count):
@ -247,10 +247,10 @@ def update_list(model, model_attr, item_model, items):
for i in items:
for item in getattr(model, model_attr):
if item.id == i['id']:
if item.id == i["id"]:
break
else:
getattr(model, model_attr).append(get(item_model, i['id']))
getattr(model, model_attr).append(get(item_model, i["id"]))
return model
@ -276,9 +276,9 @@ def get_count(q):
disable_group_by = False
if len(q._entities) > 1:
# currently support only one entity
raise Exception('only one entity is supported for get_count, got: %s' % q)
raise Exception("only one entity is supported for get_count, got: %s" % q)
entity = q._entities[0]
if hasattr(entity, 'column'):
if hasattr(entity, "column"):
# _ColumnEntity has column attr - on case: query(Model.column)...
col = entity.column
if q._group_by and q._distinct:
@ -295,7 +295,11 @@ def get_count(q):
count_func = func.count()
if q._group_by and not disable_group_by:
count_func = count_func.over(None)
count_q = q.options(lazyload('*')).statement.with_only_columns([count_func]).order_by(None)
count_q = (
q.options(lazyload("*"))
.statement.with_only_columns([count_func])
.order_by(None)
)
if disable_group_by:
count_q = count_q.group_by(None)
count = q.session.execute(count_q).scalar()
@ -311,13 +315,13 @@ def sort_and_page(query, model, args):
:param args:
:return:
"""
sort_by = args.pop('sort_by')
sort_dir = args.pop('sort_dir')
page = args.pop('page')
count = args.pop('count')
sort_by = args.pop("sort_by")
sort_dir = args.pop("sort_dir")
page = args.pop("page")
count = args.pop("count")
if args.get('user'):
user = args.pop('user')
if args.get("user"):
user = args.pop("user")
query = find_all(query, model, args)

1
lemur/default.conf.py
View File

@ -1,6 +1,7 @@
# This is just Python which means you can inherit and tweak settings
import os
_basedir = os.path.abspath(os.path.dirname(__file__))
THREADS_PER_PAGE = 8

23
lemur/defaults/views.py
View File

@ -13,12 +13,13 @@ from lemur.auth.service import AuthenticatedResource
from lemur.defaults.schemas import default_output_schema
mod = Blueprint('default', __name__)
mod = Blueprint("default", __name__)
api = Api(mod)
class LemurDefaults(AuthenticatedResource):
""" Defines the 'defaults' endpoint """
def __init__(self):
super(LemurDefaults)
@ -59,17 +60,21 @@ class LemurDefaults(AuthenticatedResource):
:statuscode 403: unauthenticated
"""
default_authority = get_by_name(current_app.config.get('LEMUR_DEFAULT_AUTHORITY'))
default_authority = get_by_name(
current_app.config.get("LEMUR_DEFAULT_AUTHORITY")
)
return dict(
country=current_app.config.get('LEMUR_DEFAULT_COUNTRY'),
state=current_app.config.get('LEMUR_DEFAULT_STATE'),
location=current_app.config.get('LEMUR_DEFAULT_LOCATION'),
organization=current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'),
organizational_unit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'),
issuer_plugin=current_app.config.get('LEMUR_DEFAULT_ISSUER_PLUGIN'),
country=current_app.config.get("LEMUR_DEFAULT_COUNTRY"),
state=current_app.config.get("LEMUR_DEFAULT_STATE"),
location=current_app.config.get("LEMUR_DEFAULT_LOCATION"),
organization=current_app.config.get("LEMUR_DEFAULT_ORGANIZATION"),
organizational_unit=current_app.config.get(
"LEMUR_DEFAULT_ORGANIZATIONAL_UNIT"
),
issuer_plugin=current_app.config.get("LEMUR_DEFAULT_ISSUER_PLUGIN"),
authority=default_authority,
)
api.add_resource(LemurDefaults, '/defaults', endpoint='default')
api.add_resource(LemurDefaults, "/defaults", endpoint="default")

2
lemur/destinations/models.py
View File

@ -13,7 +13,7 @@ from lemur.plugins.base import plugins
class Destination(db.Model):
__tablename__ = 'destinations'
__tablename__ = "destinations"
id = Column(Integer, primary_key=True)
label = Column(String(32))
options = Column(JSONType)

2
lemur/destinations/schemas.py
View File

@ -30,7 +30,7 @@ class DestinationOutputSchema(LemurOutputSchema):
@post_dump
def fill_object(self, data):
if data:
data['plugin']['pluginOptions'] = data['options']
data["plugin"]["pluginOptions"] = data["options"]
return data

42
lemur/destinations/service.py
View File

@ -6,11 +6,13 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import func
from flask import current_app
from lemur import database
from lemur.models import certificate_destination_associations
from lemur.destinations.models import Destination
from lemur.certificates.models import Certificate
from lemur.sources.service import add_aws_destination_to_sources
def create(label, plugin_name, options, description=None):
@ -24,10 +26,18 @@ def create(label, plugin_name, options, description=None):
"""
# remove any sub-plugin objects before try to save the json options
for option in options:
if 'plugin' in option['type']:
del option['value']['plugin_object']
if "plugin" in option["type"]:
del option["value"]["plugin_object"]
destination = Destination(
label=label, options=options, plugin_name=plugin_name, description=description
)
current_app.logger.info("Destination: %s created", label)
# add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
if add_aws_destination_to_sources(destination):
current_app.logger.info("Source: %s created", label)
destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
return database.create(destination)
@ -77,7 +87,7 @@ def get_by_label(label):
:param label:
:return:
"""
return database.get(Destination, label, field='label')
return database.get(Destination, label, field="label")
def get_all():
@ -91,17 +101,19 @@ def get_all():
def render(args):
filt = args.pop('filter')
certificate_id = args.pop('certificate_id', None)
filt = args.pop("filter")
certificate_id = args.pop("certificate_id", None)
if certificate_id:
query = database.session_query(Destination).join(Certificate, Destination.certificate)
query = database.session_query(Destination).join(
Certificate, Destination.certificate
)
query = query.filter(Certificate.id == certificate_id)
else:
query = database.session_query(Destination)
if filt:
terms = filt.split(';')
terms = filt.split(";")
query = database.filter(query, Destination, terms)
return database.sort_and_page(query, Destination, args)
@ -114,9 +126,15 @@ def stats(**kwargs):
:param kwargs:
:return:
"""
items = database.db.session.query(Destination.label, func.count(certificate_destination_associations.c.certificate_id))\
.join(certificate_destination_associations)\
.group_by(Destination.label).all()
items = (
database.db.session.query(
Destination.label,
func.count(certificate_destination_associations.c.certificate_id),
)
.join(certificate_destination_associations)
.group_by(Destination.label)
.all()
)
keys = []
values = []
@ -124,4 +142,4 @@ def stats(**kwargs):
keys.append(key)
values.append(count)
return {'labels': keys, 'values': values}
return {"labels": keys, "values": values}

46
lemur/destinations/views.py
View File

@ -15,15 +15,20 @@ from lemur.auth.permissions import admin_permission
from lemur.common.utils import paginated_parser
from lemur.common.schema import validate_schema
from lemur.destinations.schemas import destinations_output_schema, destination_input_schema, destination_output_schema
from lemur.destinations.schemas import (
destinations_output_schema,
destination_input_schema,
destination_output_schema,
)
mod = Blueprint('destinations', __name__)
mod = Blueprint("destinations", __name__)
api = Api(mod)
class DestinationsList(AuthenticatedResource):
""" Defines the 'destinations' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(DestinationsList, self).__init__()
@ -176,7 +181,12 @@ class DestinationsList(AuthenticatedResource):
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
"""
return service.create(data['label'], data['plugin']['slug'], data['plugin']['plugin_options'], data['description'])
return service.create(
data["label"],
data["plugin"]["slug"],
data["plugin"]["plugin_options"],
data["description"],
)
class Destinations(AuthenticatedResource):
@ -325,16 +335,22 @@ class Destinations(AuthenticatedResource):
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
"""
return service.update(destination_id, data['label'], data['plugin']['plugin_options'], data['description'])
return service.update(
destination_id,
data["label"],
data["plugin"]["plugin_options"],
data["description"],
)
@admin_permission.require(http_exception=403)
def delete(self, destination_id):
service.delete(destination_id)
return {'result': True}
return {"result": True}
class CertificateDestinations(AuthenticatedResource):
""" Defines the 'certificate/<int:certificate_id/destinations'' endpoint """
def __init__(self):
super(CertificateDestinations, self).__init__()
@ -401,25 +417,31 @@ class CertificateDestinations(AuthenticatedResource):
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args['certificate_id'] = certificate_id
args["certificate_id"] = certificate_id
return service.render(args)
class DestinationsStats(AuthenticatedResource):
""" Defines the 'certificates' stats endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(DestinationsStats, self).__init__()
def get(self):
self.reqparse.add_argument('metric', type=str, location='args')
self.reqparse.add_argument("metric", type=str, location="args")
args = self.reqparse.parse_args()
items = service.stats(**args)
return dict(items=items, total=len(items))
api.add_resource(DestinationsList, '/destinations', endpoint='destinations')
api.add_resource(Destinations, '/destinations/<int:destination_id>', endpoint='destination')
api.add_resource(CertificateDestinations, '/certificates/<int:certificate_id>/destinations',
endpoint='certificateDestinations')
api.add_resource(DestinationsStats, '/destinations/stats', endpoint='destinationStats')
api.add_resource(DestinationsList, "/destinations", endpoint="destinations")
api.add_resource(
Destinations, "/destinations/<int:destination_id>", endpoint="destination"
)
api.add_resource(
CertificateDestinations,
"/certificates/<int:certificate_id>/destinations",
endpoint="certificateDestinations",
)
api.add_resource(DestinationsStats, "/destinations/stats", endpoint="destinationStats")

6
lemur/dns_providers/cli.py
View File

@ -5,7 +5,9 @@ from lemur.dns_providers.service import get_all_dns_providers, set_domains
from lemur.extensions import metrics
from lemur.plugins.base import plugins
manager = Manager(usage="Iterates through all DNS providers and sets DNS zones in the database.")
manager = Manager(
usage="Iterates through all DNS providers and sets DNS zones in the database."
)
@manager.command
@ -27,5 +29,5 @@ def get_all_zones():
status = SUCCESS_METRIC_STATUS
metrics.send('get_all_zones', 'counter', 1, metric_tags={'status': status})
metrics.send("get_all_zones", "counter", 1, metric_tags={"status": status})
print("[+] Done with dns provider zone lookup and configuration.")

17
lemur/dns_providers/models.py
View File

@ -9,22 +9,23 @@ from lemur.utils import Vault
class DnsProvider(db.Model):
__tablename__ = 'dns_providers'
id = Column(
Integer(),
primary_key=True,
)
__tablename__ = "dns_providers"
id = Column(Integer(), primary_key=True)
name = Column(String(length=256), unique=True, nullable=True)
description = Column(Text(), nullable=True)
provider_type = Column(String(length=256), nullable=True)
credentials = Column(Vault, nullable=True)
api_endpoint = Column(String(length=256), nullable=True)
date_created = Column(ArrowType(), server_default=text('now()'), nullable=False)
date_created = Column(ArrowType(), server_default=text("now()"), nullable=False)
status = Column(String(length=128), nullable=True)
options = Column(JSON, nullable=True)
domains = Column(JSON, nullable=True)
certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id',
lazy='dynamic')
certificates = relationship(
"Certificate",
backref="dns_provider",
foreign_keys="Certificate.dns_provider_id",
lazy="dynamic",
)
def __init__(self, name, description, provider_type, credentials):
self.name = name

63
lemur/dns_providers/service.py
View File

@ -49,7 +49,9 @@ def get_friendly(dns_provider_id):
}
if dns_provider.provider_type == "route53":
dns_provider_friendly["account_id"] = json.loads(dns_provider.credentials).get("account_id")
dns_provider_friendly["account_id"] = json.loads(dns_provider.credentials).get(
"account_id"
)
return dns_provider_friendly
@ -64,40 +66,41 @@ def delete(dns_provider_id):
def get_types():
provider_config = current_app.config.get(
'ACME_DNS_PROVIDER_TYPES',
{"items": [
"ACME_DNS_PROVIDER_TYPES",
{
'name': 'route53',
'requirements': [
"items": [
{
'name': 'account_id',
'type': 'int',
'required': True,
'helpMessage': 'AWS Account number'
"name": "route53",
"requirements": [
{
"name": "account_id",
"type": "int",
"required": True,
"helpMessage": "AWS Account number",
}
],
},
{
"name": "cloudflare",
"requirements": [
{
"name": "email",
"type": "str",
"required": True,
"helpMessage": "Cloudflare Email",
},
{
"name": "key",
"type": "str",
"required": True,
"helpMessage": "Cloudflare Key",
},
],
},
{"name": "dyn"},
{"name": "ultradns"},
]
},
{
'name': 'cloudflare',
'requirements': [
{
'name': 'email',
'type': 'str',
'required': True,
'helpMessage': 'Cloudflare Email'
},
{
'name': 'key',
'type': 'str',
'required': True,
'helpMessage': 'Cloudflare Key'
},
]
},
{
'name': 'dyn',
},
]}
)
if not provider_config:
raise Exception("No DNS Provider configuration specified.")

27
lemur/dns_providers/views.py
View File

@ -13,9 +13,12 @@ from lemur.auth.service import AuthenticatedResource
from lemur.common.schema import validate_schema
from lemur.common.utils import paginated_parser
from lemur.dns_providers import service
from lemur.dns_providers.schemas import dns_provider_output_schema, dns_provider_input_schema
from lemur.dns_providers.schemas import (
dns_provider_output_schema,
dns_provider_input_schema,
)
mod = Blueprint('dns_providers', __name__)
mod = Blueprint("dns_providers", __name__)
api = Api(mod)
@ -71,12 +74,12 @@ class DnsProvidersList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
parser.add_argument('dns_provider_id', type=int, location='args')
parser.add_argument('name', type=str, location='args')
parser.add_argument('type', type=str, location='args')
parser.add_argument("dns_provider_id", type=int, location="args")
parser.add_argument("name", type=str, location="args")
parser.add_argument("type", type=str, location="args")
args = parser.parse_args()
args['user'] = g.user
args["user"] = g.user
return service.render(args)
@validate_schema(dns_provider_input_schema, None)
@ -152,7 +155,7 @@ class DnsProviders(AuthenticatedResource):
@admin_permission.require(http_exception=403)
def delete(self, dns_provider_id):
service.delete(dns_provider_id)
return {'result': True}
return {"result": True}
class DnsProviderOptions(AuthenticatedResource):
@ -166,6 +169,10 @@ class DnsProviderOptions(AuthenticatedResource):
return service.get_types()
api.add_resource(DnsProvidersList, '/dns_providers', endpoint='dns_providers')
api.add_resource(DnsProviders, '/dns_providers/<int:dns_provider_id>', endpoint='dns_provider')
api.add_resource(DnsProviderOptions, '/dns_provider_options', endpoint='dns_provider_options')
api.add_resource(DnsProvidersList, "/dns_providers", endpoint="dns_providers")
api.add_resource(
DnsProviders, "/dns_providers/<int:dns_provider_id>", endpoint="dns_provider"
)
api.add_resource(
DnsProviderOptions, "/dns_provider_options", endpoint="dns_provider_options"
)

9
lemur/domains/models.py
View File

@ -13,11 +13,14 @@ from lemur.database import db
class Domain(db.Model):
__tablename__ = 'domains'
__tablename__ = "domains"
__table_args__ = (
Index('ix_domains_name_gin', "name",
Index(
"ix_domains_name_gin",
"name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
postgresql_using="gin",
),
)
id = Column(Integer, primary_key=True)
name = Column(String(256), index=True)

25
lemur/domains/service.py
View File

@ -6,10 +6,11 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from lemur.domains.models import Domain
from lemur.certificates.models import Certificate
from sqlalchemy import and_
from lemur import database
from lemur.certificates.models import Certificate
from lemur.domains.models import Domain
def get(domain_id):
@ -42,6 +43,20 @@ def get_by_name(name):
return database.get_all(Domain, name, field="name").all()
def is_domain_sensitive(name):
"""
Return True if domain is marked sensitive
:param name:
:return:
"""
query = database.session_query(Domain)
query = query.filter(and_(Domain.sensitive, Domain.name == name))
return database.find_all(query, Domain, {}).all()
def create(name, sensitive):
"""
Create a new domain
@ -77,11 +92,11 @@ def render(args):
:return:
"""
query = database.session_query(Domain)
filt = args.pop('filter')
certificate_id = args.pop('certificate_id', None)
filt = args.pop("filter")
certificate_id = args.pop("certificate_id", None)
if filt:
terms = filt.split(';')
terms = filt.split(";")
query = database.filter(query, Domain, terms)
if certificate_id:

28
lemur/domains/views.py
View File

@ -17,14 +17,19 @@ from lemur.auth.permissions import SensitiveDomainPermission
from lemur.common.schema import validate_schema
from lemur.common.utils import paginated_parser
from lemur.domains.schemas import domain_input_schema, domain_output_schema, domains_output_schema
from lemur.domains.schemas import (
domain_input_schema,
domain_output_schema,
domains_output_schema,
)
mod = Blueprint('domains', __name__)
mod = Blueprint("domains", __name__)
api = Api(mod)
class DomainsList(AuthenticatedResource):
""" Defines the 'domains' endpoint """
def __init__(self):
super(DomainsList, self).__init__()
@ -123,7 +128,7 @@ class DomainsList(AuthenticatedResource):
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
return service.create(data['name'], data['sensitive'])
return service.create(data["name"], data["sensitive"])
class Domains(AuthenticatedResource):
@ -205,13 +210,14 @@ class Domains(AuthenticatedResource):
:statuscode 403: unauthenticated
"""
if SensitiveDomainPermission().can():
return service.update(domain_id, data['name'], data['sensitive'])
return service.update(domain_id, data["name"], data["sensitive"])
return dict(message='You are not authorized to modify this domain'), 403
return dict(message="You are not authorized to modify this domain"), 403
class CertificateDomains(AuthenticatedResource):
""" Defines the 'domains' endpoint """
def __init__(self):
super(CertificateDomains, self).__init__()
@ -265,10 +271,14 @@ class CertificateDomains(AuthenticatedResource):
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args['certificate_id'] = certificate_id
args["certificate_id"] = certificate_id
return service.render(args)
api.add_resource(DomainsList, '/domains', endpoint='domains')
api.add_resource(Domains, '/domains/<int:domain_id>', endpoint='domain')
api.add_resource(CertificateDomains, '/certificates/<int:certificate_id>/domains', endpoint='certificateDomains')
api.add_resource(DomainsList, "/domains", endpoint="domains")
api.add_resource(Domains, "/domains/<int:domain_id>", endpoint="domain")
api.add_resource(
CertificateDomains,
"/certificates/<int:certificate_id>/domains",
endpoint="certificateDomains",
)

21
lemur/endpoints/cli.py
View File

@ -21,7 +21,14 @@ from lemur.endpoints.models import Endpoint
manager = Manager(usage="Handles all endpoint related tasks.")
@manager.option('-ttl', '--time-to-live', type=int, dest='ttl', default=2, help='Time in hours, which endpoint has not been refreshed to remove the endpoint.')
@manager.option(
"-ttl",
"--time-to-live",
type=int,
dest="ttl",
default=2,
help="Time in hours, which endpoint has not been refreshed to remove the endpoint.",
)
def expire(ttl):
"""
Removed all endpoints that have not been recently updated.
@ -31,12 +38,18 @@ def expire(ttl):
try:
now = arrow.utcnow()
expiration = now - timedelta(hours=ttl)
endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration)
endpoints = database.session_query(Endpoint).filter(
cast(Endpoint.last_updated, ArrowType) <= expiration
)
for endpoint in endpoints:
print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated))
print(
"[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(
name=endpoint.name, last_updated=endpoint.last_updated
)
)
database.delete(endpoint)
metrics.send('endpoint_expired', 'counter', 1)
metrics.send("endpoint_expired", "counter", 1)
print("[+] Finished expiration.")
except Exception as e:

60
lemur/endpoints/models.py
View File

@ -20,15 +20,11 @@ from lemur.database import db
from lemur.models import policies_ciphers
BAD_CIPHERS = [
'Protocol-SSLv3',
'Protocol-SSLv2',
'Protocol-TLSv1'
]
BAD_CIPHERS = ["Protocol-SSLv3", "Protocol-SSLv2", "Protocol-TLSv1"]
class Cipher(db.Model):
__tablename__ = 'ciphers'
__tablename__ = "ciphers"
id = Column(Integer, primary_key=True)
name = Column(String(128), nullable=False)
@ -38,23 +34,18 @@ class Cipher(db.Model):
@deprecated.expression
def deprecated(cls):
return case(
[
(cls.name in BAD_CIPHERS, True)
],
else_=False
)
return case([(cls.name in BAD_CIPHERS, True)], else_=False)
class Policy(db.Model):
___tablename__ = 'policies'
___tablename__ = "policies"
id = Column(Integer, primary_key=True)
name = Column(String(128), nullable=True)
ciphers = relationship('Cipher', secondary=policies_ciphers, backref='policy')
ciphers = relationship("Cipher", secondary=policies_ciphers, backref="policy")
class Endpoint(db.Model):
__tablename__ = 'endpoints'
__tablename__ = "endpoints"
id = Column(Integer, primary_key=True)
owner = Column(String(128))
name = Column(String(128))
@ -62,16 +53,18 @@ class Endpoint(db.Model):
type = Column(String(128))
active = Column(Boolean, default=True)
port = Column(Integer)
policy_id = Column(Integer, ForeignKey('policy.id'))
policy = relationship('Policy', backref='endpoint')
certificate_id = Column(Integer, ForeignKey('certificates.id'))
source_id = Column(Integer, ForeignKey('sources.id'))
policy_id = Column(Integer, ForeignKey("policy.id"))
policy = relationship("Policy", backref="endpoint")
certificate_id = Column(Integer, ForeignKey("certificates.id"))
source_id = Column(Integer, ForeignKey("sources.id"))
sensitive = Column(Boolean, default=False)
source = relationship('Source', back_populates='endpoints')
source = relationship("Source", back_populates="endpoints")
last_updated = Column(ArrowType, default=arrow.utcnow, nullable=False)
date_created = Column(ArrowType, default=arrow.utcnow, onupdate=arrow.utcnow, nullable=False)
date_created = Column(
ArrowType, default=arrow.utcnow, onupdate=arrow.utcnow, nullable=False
)
replaced = association_proxy('certificate', 'replaced')
replaced = association_proxy("certificate", "replaced")
@property
def issues(self):
@ -79,13 +72,30 @@ class Endpoint(db.Model):
for cipher in self.policy.ciphers:
if cipher.deprecated:
issues.append({'name': 'deprecated cipher', 'value': '{0} has been deprecated consider removing it.'.format(cipher.name)})
issues.append(
{
"name": "deprecated cipher",
"value": "{0} has been deprecated consider removing it.".format(
cipher.name
),
}
)
if self.certificate.expired:
issues.append({'name': 'expired certificate', 'value': 'There is an expired certificate attached to this endpoint consider replacing it.'})
issues.append(
{
"name": "expired certificate",
"value": "There is an expired certificate attached to this endpoint consider replacing it.",
}
)
if self.certificate.revoked:
issues.append({'name': 'revoked', 'value': 'There is a revoked certificate attached to this endpoint consider replacing it.'})
issues.append(
{
"name": "revoked",
"value": "There is a revoked certificate attached to this endpoint consider replacing it.",
}
)
return issues

48
lemur/endpoints/service.py
View File

@ -46,7 +46,7 @@ def get_by_name(name):
:param name:
:return:
"""
return database.get(Endpoint, name, field='name')
return database.get(Endpoint, name, field="name")
def get_by_dnsname(dnsname):
@ -56,7 +56,7 @@ def get_by_dnsname(dnsname):
:param dnsname:
:return:
"""
return database.get(Endpoint, dnsname, field='dnsname')
return database.get(Endpoint, dnsname, field="dnsname")
def get_by_dnsname_and_port(dnsname, port):
@ -66,7 +66,11 @@ def get_by_dnsname_and_port(dnsname, port):
:param port:
:return:
"""
return Endpoint.query.filter(Endpoint.dnsname == dnsname).filter(Endpoint.port == port).scalar()
return (
Endpoint.query.filter(Endpoint.dnsname == dnsname)
.filter(Endpoint.port == port)
.scalar()
)
def get_by_source(source_label):
@ -95,12 +99,14 @@ def create(**kwargs):
"""
endpoint = Endpoint(**kwargs)
database.create(endpoint)
metrics.send('endpoint_added', 'counter', 1, metric_tags={'source': endpoint.source.label})
metrics.send(
"endpoint_added", "counter", 1, metric_tags={"source": endpoint.source.label}
)
return endpoint
def get_or_create_policy(**kwargs):
policy = database.get(Policy, kwargs['name'], field='name')
policy = database.get(Policy, kwargs["name"], field="name")
if not policy:
policy = Policy(**kwargs)
@ -110,7 +116,7 @@ def get_or_create_policy(**kwargs):
def get_or_create_cipher(**kwargs):
cipher = database.get(Cipher, kwargs['name'], field='name')
cipher = database.get(Cipher, kwargs["name"], field="name")
if not cipher:
cipher = Cipher(**kwargs)
@ -122,11 +128,13 @@ def get_or_create_cipher(**kwargs):
def update(endpoint_id, **kwargs):
endpoint = database.get(Endpoint, endpoint_id)
endpoint.policy = kwargs['policy']
endpoint.certificate = kwargs['certificate']
endpoint.source = kwargs['source']
endpoint.policy = kwargs["policy"]
endpoint.certificate = kwargs["certificate"]
endpoint.source = kwargs["source"]
endpoint.last_updated = arrow.utcnow()
metrics.send('endpoint_updated', 'counter', 1, metric_tags={'source': endpoint.source.label})
metrics.send(
"endpoint_updated", "counter", 1, metric_tags={"source": endpoint.source.label}
)
database.update(endpoint)
return endpoint
@ -138,19 +146,17 @@ def render(args):
:return:
"""
query = database.session_query(Endpoint)
filt = args.pop('filter')
filt = args.pop("filter")
if filt:
terms = filt.split(';')
if 'active' in filt: # this is really weird but strcmp seems to not work here??
terms = filt.split(";")
if "active" in filt: # this is really weird but strcmp seems to not work here??
query = query.filter(Endpoint.active == truthiness(terms[1]))
elif 'port' in filt:
if terms[1] != 'null': # ng-table adds 'null' if a number is removed
elif "port" in filt:
if terms[1] != "null": # ng-table adds 'null' if a number is removed
query = query.filter(Endpoint.port == terms[1])
elif 'ciphers' in filt:
query = query.filter(
Cipher.name == terms[1]
)
elif "ciphers" in filt:
query = query.filter(Cipher.name == terms[1])
else:
query = database.filter(query, Endpoint, terms)
@ -164,7 +170,7 @@ def stats(**kwargs):
:param kwargs:
:return:
"""
attr = getattr(Endpoint, kwargs.get('metric'))
attr = getattr(Endpoint, kwargs.get("metric"))
query = database.db.session.query(attr, func.count(attr))
items = query.group_by(attr).all()
@ -175,4 +181,4 @@ def stats(**kwargs):
keys.append(key)
values.append(count)
return {'labels': keys, 'values': values}
return {"labels": keys, "values": values}

9
lemur/endpoints/views.py
View File

@ -16,12 +16,13 @@ from lemur.endpoints import service
from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema
mod = Blueprint('endpoints', __name__)
mod = Blueprint("endpoints", __name__)
api = Api(mod)
class EndpointsList(AuthenticatedResource):
""" Defines the 'endpoints' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(EndpointsList, self).__init__()
@ -63,7 +64,7 @@ class EndpointsList(AuthenticatedResource):
"""
parser = paginated_parser.copy()
args = parser.parse_args()
args['user'] = g.current_user
args["user"] = g.current_user
return service.render(args)
@ -103,5 +104,5 @@ class Endpoints(AuthenticatedResource):
return service.get(endpoint_id)
api.add_resource(EndpointsList, '/endpoints', endpoint='endpoints')
api.add_resource(Endpoints, '/endpoints/<int:endpoint_id>', endpoint='endpoint')
api.add_resource(EndpointsList, "/endpoints", endpoint="endpoints")
api.add_resource(Endpoints, "/endpoints/<int:endpoint_id>", endpoint="endpoint")

4
lemur/exceptions.py
View File

@ -21,7 +21,9 @@ class DuplicateError(LemurException):
class InvalidListener(LemurException):
def __str__(self):
return repr("Invalid listener, ensure you select a certificate if you are using a secure protocol")
return repr(
"Invalid listener, ensure you select a certificate if you are using a secure protocol"
)
class AttrNotFound(LemurException):

8
lemur/extensions.py
View File

@ -15,25 +15,33 @@ class SQLAlchemy(SA):
db = SQLAlchemy()
from flask_migrate import Migrate
migrate = Migrate()
from flask_bcrypt import Bcrypt
bcrypt = Bcrypt()
from flask_principal import Principal
principal = Principal(use_sessions=False)
from flask_mail import Mail
smtp_mail = Mail()
from lemur.metrics import Metrics
metrics = Metrics()
from raven.contrib.flask import Sentry
sentry = Sentry()
from blinker import Namespace
signals = Namespace()
from flask_cors import CORS
cors = CORS()

96
lemur/factory.py
View File

@ -13,20 +13,21 @@ import os
import imp
import errno
import pkg_resources
import socket
from logging import Formatter, StreamHandler
from logging.handlers import RotatingFileHandler
from flask import Flask
from flask_replicated import FlaskReplicated
import logmatic
from lemur.certificates.hooks import activate_debug_dump
from lemur.common.health import mod as health
from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors
DEFAULT_BLUEPRINTS = (
health,
)
DEFAULT_BLUEPRINTS = (health,)
API_VERSION = 1
@ -53,6 +54,7 @@ def create_app(app_name=None, blueprints=None, config=None):
configure_blueprints(app, blueprints)
configure_extensions(app)
configure_logging(app)
configure_database(app)
install_plugins(app)
@app.teardown_appcontext
@ -71,16 +73,17 @@ def from_file(file_path, silent=False):
:param file_path:
:param silent:
"""
d = imp.new_module('config')
d = imp.new_module("config")
d.__file__ = file_path
try:
with open(file_path) as config_file:
exec(compile(config_file.read(), # nosec: config file safe
file_path, 'exec'), d.__dict__)
exec( # nosec: config file safe
compile(config_file.read(), file_path, "exec"), d.__dict__
)
except IOError as e:
if silent and e.errno in (errno.ENOENT, errno.EISDIR):
return False
e.strerror = 'Unable to load configuration file (%s)' % e.strerror
e.strerror = "Unable to load configuration file (%s)" % e.strerror
raise
return d
@ -94,8 +97,8 @@ def configure_app(app, config=None):
:return:
"""
# respect the config first
if config and config != 'None':
app.config['CONFIG_PATH'] = config
if config and config != "None":
app.config["CONFIG_PATH"] = config
app.config.from_object(from_file(config))
else:
try:
@ -103,12 +106,21 @@ def configure_app(app, config=None):
except RuntimeError:
# look in default paths
if os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
app.config.from_object(
from_file(os.path.expanduser("~/.lemur/lemur.conf.py"))
)
else:
app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
app.config.from_object(
from_file(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"default.conf.py",
)
)
)
# we don't use this
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
def configure_extensions(app):
@ -125,9 +137,15 @@ def configure_extensions(app):
metrics.init_app(app)
sentry.init_app(app)
if app.config['CORS']:
app.config['CORS_HEADERS'] = 'Content-Type'
cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True)
if app.config["CORS"]:
app.config["CORS_HEADERS"] = "Content-Type"
cors.init_app(
app,
resources=r"/api/*",
headers="Content-Type",
origin="*",
supports_credentials=True,
)
def configure_blueprints(app, blueprints):
@ -142,28 +160,41 @@ def configure_blueprints(app, blueprints):
app.register_blueprint(blueprint, url_prefix="/api/{0}".format(API_VERSION))
def configure_database(app):
if app.config.get("SQLALCHEMY_ENABLE_FLASK_REPLICATED"):
FlaskReplicated(app)
def configure_logging(app):
"""
Sets up application wide logging.
:param app:
"""
handler = RotatingFileHandler(app.config.get('LOG_FILE', 'lemur.log'), maxBytes=10000000, backupCount=100)
handler = RotatingFileHandler(
app.config.get("LOG_FILE", "lemur.log"), maxBytes=10000000, backupCount=100
)
handler.setFormatter(Formatter(
'%(asctime)s %(levelname)s: %(message)s '
'[in %(pathname)s:%(lineno)d]'
))
handler.setFormatter(
Formatter(
"%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
)
)
handler.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
app.logger.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
if app.config.get("LOG_JSON", False):
handler.setFormatter(
logmatic.JsonFormatter(extra={"hostname": socket.gethostname()})
)
handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG"))
app.logger.setLevel(app.config.get("LOG_LEVEL", "DEBUG"))
app.logger.addHandler(handler)
stream_handler = StreamHandler()
stream_handler.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
stream_handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG"))
app.logger.addHandler(stream_handler)
if app.config.get('DEBUG_DUMP', False):
if app.config.get("DEBUG_DUMP", False):
activate_debug_dump()
@ -176,17 +207,21 @@ def install_plugins(app):
"""
from lemur.plugins import plugins
from lemur.plugins.base import register
# entry_points={
# 'lemur.plugins': [
# 'verisign = lemur_verisign.plugin:VerisignPlugin'
# ],
# },
for ep in pkg_resources.iter_entry_points('lemur.plugins'):
for ep in pkg_resources.iter_entry_points("lemur.plugins"):
try:
plugin = ep.load()
except Exception:
import traceback
app.logger.error("Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()))
app.logger.error(
"Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc())
)
else:
register(plugin)
@ -196,6 +231,9 @@ def install_plugins(app):
try:
plugins.get(slug)
except KeyError:
raise Exception("Unable to location notification plugin: {slug}. Ensure that "
"LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin."
.format(slug=slug))
raise Exception(
"Unable to location notification plugin: {slug}. Ensure that "
"LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin.".format(
slug=slug
)
)

18
lemur/logs/models.py
View File

@ -15,9 +15,19 @@ from lemur.database import db
class Log(db.Model):
__tablename__ = 'logs'
__tablename__ = "logs"
id = Column(Integer, primary_key=True)
certificate_id = Column(Integer, ForeignKey('certificates.id'))
log_type = Column(Enum('key_view', 'create_cert', 'update_cert', 'revoke_cert', name='log_type'), nullable=False)
certificate_id = Column(Integer, ForeignKey("certificates.id"))
log_type = Column(
Enum(
"key_view",
"create_cert",
"update_cert",
"revoke_cert",
"delete_cert",
name="log_type",
),
nullable=False,
)
logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False)
user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)

24
lemur/logs/service.py
View File

@ -24,7 +24,11 @@ def create(user, type, certificate=None):
:param certificate:
:return:
"""
current_app.logger.info("[lemur-audit] action: {0}, user: {1}, certificate: {2}.".format(type, user.email, certificate.name))
current_app.logger.info(
"[lemur-audit] action: {0}, user: {1}, certificate: {2}.".format(
type, user.email, certificate.name
)
)
view = Log(user_id=user.id, log_type=type, certificate_id=certificate.id)
database.add(view)
database.commit()
@ -50,20 +54,22 @@ def render(args):
"""
query = database.session_query(Log)
filt = args.pop('filter')
filt = args.pop("filter")
if filt:
terms = filt.split(';')
terms = filt.split(";")
if 'certificate.name' in terms:
sub_query = database.session_query(Certificate.id)\
.filter(Certificate.name.ilike('%{0}%'.format(terms[1])))
if "certificate.name" in terms:
sub_query = database.session_query(Certificate.id).filter(
Certificate.name.ilike("%{0}%".format(terms[1]))
)
query = query.filter(Log.certificate_id.in_(sub_query))
elif 'user.email' in terms:
sub_query = database.session_query(User.id)\
.filter(User.email.ilike('%{0}%'.format(terms[1])))
elif "user.email" in terms:
sub_query = database.session_query(User.id).filter(
User.email.ilike("%{0}%".format(terms[1]))
)
query = query.filter(Log.user_id.in_(sub_query))

9
lemur/logs/views.py
View File

@ -17,12 +17,13 @@ from lemur.logs.schemas import logs_output_schema
from lemur.logs import service
mod = Blueprint('logs', __name__)
mod = Blueprint("logs", __name__)
api = Api(mod)
class LogsList(AuthenticatedResource):
""" Defines the 'logs' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(LogsList, self).__init__()
@ -65,10 +66,10 @@ class LogsList(AuthenticatedResource):
:statuscode 200: no error
"""
parser = paginated_parser.copy()
parser.add_argument('owner', type=str, location='args')
parser.add_argument('id', type=str, location='args')
parser.add_argument("owner", type=str, location="args")
parser.add_argument("id", type=str, location="args")
args = parser.parse_args()
return service.render(args)
api.add_resource(LogsList, '/logs', endpoint='logs')
api.add_resource(LogsList, "/logs", endpoint="logs")

181
lemur/manage.py
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python
from __future__ import unicode_literals # at top of module
import os
@ -49,25 +50,27 @@ from lemur.policies.models import RotationPolicy # noqa
from lemur.pending_certificates.models import PendingCertificate # noqa
from lemur.dns_providers.models import DnsProvider # noqa
from sqlalchemy.sql import text
manager = Manager(create_app)
manager.add_option('-c', '--config', dest='config')
manager.add_option("-c", "--config", dest="config_path", required=False)
migrate = Migrate(create_app)
REQUIRED_VARIABLES = [
'LEMUR_SECURITY_TEAM_EMAIL',
'LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',
'LEMUR_DEFAULT_ORGANIZATION',
'LEMUR_DEFAULT_LOCATION',
'LEMUR_DEFAULT_COUNTRY',
'LEMUR_DEFAULT_STATE',
'SQLALCHEMY_DATABASE_URI'
"LEMUR_SECURITY_TEAM_EMAIL",
"LEMUR_DEFAULT_ORGANIZATIONAL_UNIT",
"LEMUR_DEFAULT_ORGANIZATION",
"LEMUR_DEFAULT_LOCATION",
"LEMUR_DEFAULT_COUNTRY",
"LEMUR_DEFAULT_STATE",
"SQLALCHEMY_DATABASE_URI",
]
KEY_LENGTH = 40
DEFAULT_CONFIG_PATH = '~/.lemur/lemur.conf.py'
DEFAULT_SETTINGS = 'lemur.conf.server'
SETTINGS_ENVVAR = 'LEMUR_CONF'
DEFAULT_CONFIG_PATH = "~/.lemur/lemur.conf.py"
DEFAULT_SETTINGS = "lemur.conf.server"
SETTINGS_ENVVAR = "LEMUR_CONF"
CONFIG_TEMPLATE = """
# This is just Python which means you can inherit and tweak settings
@ -142,8 +145,9 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
@MigrateCommand.command
def create():
database.db.engine.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
database.db.create_all()
stamp(revision='head')
stamp(revision="head")
@MigrateCommand.command
@ -171,9 +175,9 @@ def generate_settings():
output = CONFIG_TEMPLATE.format(
# we use Fernet.generate_key to make sure that the key length is
# compatible with Fernet
encryption_key=Fernet.generate_key().decode('utf-8'),
secret_token=base64.b64encode(os.urandom(KEY_LENGTH)).decode('utf-8'),
flask_secret_key=base64.b64encode(os.urandom(KEY_LENGTH)).decode('utf-8'),
encryption_key=Fernet.generate_key().decode("utf-8"),
secret_token=base64.b64encode(os.urandom(KEY_LENGTH)).decode("utf-8"),
flask_secret_key=base64.b64encode(os.urandom(KEY_LENGTH)).decode("utf-8"),
)
return output
@ -187,39 +191,44 @@ class InitializeApp(Command):
Additionally a Lemur user will be created as a default user
and be used when certificates are discovered by Lemur.
"""
option_list = (
Option('-p', '--password', dest='password'),
)
option_list = (Option("-p", "--password", dest="password"),)
def run(self, password):
create()
user = user_service.get_by_username("lemur")
admin_role = role_service.get_by_name('admin')
admin_role = role_service.get_by_name("admin")
if admin_role:
sys.stdout.write("[-] Admin role already created, skipping...!\n")
else:
# we create an admin role
admin_role = role_service.create('admin', description='This is the Lemur administrator role.')
admin_role = role_service.create(
"admin", description="This is the Lemur administrator role."
)
sys.stdout.write("[+] Created 'admin' role\n")
operator_role = role_service.get_by_name('operator')
operator_role = role_service.get_by_name("operator")
if operator_role:
sys.stdout.write("[-] Operator role already created, skipping...!\n")
else:
# we create an operator role
operator_role = role_service.create('operator', description='This is the Lemur operator role.')
operator_role = role_service.create(
"operator", description="This is the Lemur operator role."
)
sys.stdout.write("[+] Created 'operator' role\n")
read_only_role = role_service.get_by_name('read-only')
read_only_role = role_service.get_by_name("read-only")
if read_only_role:
sys.stdout.write("[-] Read only role already created, skipping...!\n")
else:
# we create an read only role
read_only_role = role_service.create('read-only', description='This is the Lemur read only role.')
read_only_role = role_service.create(
"read-only", description="This is the Lemur read only role."
)
sys.stdout.write("[+] Created 'read-only' role\n")
if not user:
@ -232,34 +241,54 @@ class InitializeApp(Command):
sys.stderr.write("[!] Passwords do not match!\n")
sys.exit(1)
user_service.create("lemur", password, 'lemur@nobody.com', True, None, [admin_role])
sys.stdout.write("[+] Created the user 'lemur' and granted it the 'admin' role!\n")
user_service.create(
"lemur", password, "lemur@nobody.com", True, None, [admin_role]
)
sys.stdout.write(
"[+] Created the user 'lemur' and granted it the 'admin' role!\n"
)
else:
sys.stdout.write("[-] Default user has already been created, skipping...!\n")
sys.stdout.write(
"[-] Default user has already been created, skipping...!\n"
)
intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [])
intervals = current_app.config.get(
"LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", []
)
sys.stdout.write(
"[!] Creating {num} notifications for {intervals} days as specified by LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS\n".format(
num=len(intervals),
intervals=",".join([str(x) for x in intervals])
num=len(intervals), intervals=",".join([str(x) for x in intervals])
)
)
recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
recipients = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
sys.stdout.write("[+] Creating expiration email notifications!\n")
sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(recipients))
notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients)
sys.stdout.write(
"[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(
recipients
)
)
notification_service.create_default_expiration_notifications(
"DEFAULT_SECURITY", recipients=recipients
)
_DEFAULT_ROTATION_INTERVAL = 'default'
default_rotation_interval = policy_service.get_by_name(_DEFAULT_ROTATION_INTERVAL)
_DEFAULT_ROTATION_INTERVAL = "default"
default_rotation_interval = policy_service.get_by_name(
_DEFAULT_ROTATION_INTERVAL
)
if default_rotation_interval:
sys.stdout.write("[-] Default rotation interval policy already created, skipping...!\n")
sys.stdout.write(
"[-] Default rotation interval policy already created, skipping...!\n"
)
else:
days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30)
sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
days=days))
sys.stdout.write(
"[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
days=days
)
)
policy_service.create(days=days, name=_DEFAULT_ROTATION_INTERVAL)
sys.stdout.write("[/] Done!\n")
@ -269,14 +298,16 @@ class CreateUser(Command):
"""
This command allows for the creation of a new user within Lemur.
"""
option_list = (
Option('-u', '--username', dest='username', required=True),
Option('-e', '--email', dest='email', required=True),
Option('-a', '--active', dest='active', default=True),
Option('-r', '--roles', dest='roles', action='append', default=[])
Option("-u", "--username", dest="username", required=True),
Option("-e", "--email", dest="email", required=True),
Option("-a", "--active", dest="active", default=True),
Option("-r", "--roles", dest="roles", action="append", default=[]),
Option("-p", "--password", dest="password", default=None),
)
def run(self, username, email, active, roles):
def run(self, username, email, active, roles, password):
role_objs = []
for r in roles:
role_obj = role_service.get_by_name(r)
@ -286,14 +317,16 @@ class CreateUser(Command):
sys.stderr.write("[!] Cannot find role {0}\n".format(r))
sys.exit(1)
if not password:
password1 = prompt_pass("Password")
password2 = prompt_pass("Confirm Password")
password = password1
if password1 != password2:
sys.stderr.write("[!] Passwords do not match!\n")
sys.exit(1)
user_service.create(username, password1, email, active, None, role_objs)
user_service.create(username, password, email, active, None, role_objs)
sys.stdout.write("[+] Created new user: {0}\n".format(username))
@ -301,9 +334,8 @@ class ResetPassword(Command):
"""
This command allows you to reset a user's password.
"""
option_list = (
Option('-u', '--username', dest='username', required=True),
)
option_list = (Option("-u", "--username", dest="username", required=True),)
def run(self, username):
user = user_service.get_by_username(username)
@ -329,10 +361,11 @@ class CreateRole(Command):
"""
This command allows for the creation of a new role within Lemur
"""
option_list = (
Option('-n', '--name', dest='name', required=True),
Option('-u', '--users', dest='users', default=[]),
Option('-d', '--description', dest='description', required=True)
Option("-n", "--name", dest="name", required=True),
Option("-u", "--users", dest="users", default=[]),
Option("-d", "--description", dest="description", required=True),
)
def run(self, name, users, description):
@ -363,7 +396,8 @@ class LemurServer(Command):
Will start gunicorn with 4 workers bound to 127.0.0.0:8002
"""
description = 'Run the app within Gunicorn'
description = "Run the app within Gunicorn"
def get_options(self):
settings = make_settings()
@ -371,8 +405,10 @@ class LemurServer(Command):
for setting, klass in settings.items():
if klass.cli:
if klass.action:
if klass.action == 'store_const':
options.append(Option(*klass.cli, const=klass.const, action=klass.action))
if klass.action == "store_const":
options.append(
Option(*klass.cli, const=klass.const, action=klass.action)
)
else:
options.append(Option(*klass.cli, action=klass.action))
else:
@ -388,7 +424,9 @@ class LemurServer(Command):
# run startup tasks on an app like object
validate_conf(current_app, REQUIRED_VARIABLES)
app.app_uri = 'lemur:create_app(config="{0}")'.format(current_app.config.get('CONFIG_PATH'))
app.app_uri = 'lemur:create_app(config_path="{0}")'.format(
current_app.config.get("CONFIG_PATH")
)
return app.run()
@ -408,7 +446,7 @@ def create_config(config_path=None):
os.makedirs(dir)
config = generate_settings()
with open(config_path, 'w') as f:
with open(config_path, "w") as f:
f.write(config)
sys.stdout.write("[+] Created a new configuration file {0}\n".format(config_path))
@ -430,7 +468,7 @@ def lock(path=None):
:param: path
"""
if not path:
path = os.path.expanduser('~/.lemur/keys')
path = os.path.expanduser("~/.lemur/keys")
dest_dir = os.path.join(path, "encrypted")
sys.stdout.write("[!] Generating a new key...\n")
@ -441,15 +479,17 @@ def lock(path=None):
sys.stdout.write("[+] Creating encryption directory: {0}\n".format(dest_dir))
os.makedirs(dest_dir)
for root, dirs, files in os.walk(os.path.join(path, 'decrypted')):
for root, dirs, files in os.walk(os.path.join(path, "decrypted")):
for f in files:
source = os.path.join(root, f)
dest = os.path.join(dest_dir, f + ".enc")
with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
with open(source, "rb") as in_file, open(dest, "wb") as out_file:
f = Fernet(key)
data = f.encrypt(in_file.read())
out_file.write(data)
sys.stdout.write("[+] Writing file: {0} Source: {1}\n".format(dest, source))
sys.stdout.write(
"[+] Writing file: {0} Source: {1}\n".format(dest, source)
)
sys.stdout.write("[+] Keys have been encrypted with key {0}\n".format(key))
@ -469,7 +509,7 @@ def unlock(path=None):
key = prompt_pass("[!] Please enter the encryption password")
if not path:
path = os.path.expanduser('~/.lemur/keys')
path = os.path.expanduser("~/.lemur/keys")
dest_dir = os.path.join(path, "decrypted")
source_dir = os.path.join(path, "encrypted")
@ -482,11 +522,13 @@ def unlock(path=None):
for f in files:
source = os.path.join(source_dir, f)
dest = os.path.join(dest_dir, ".".join(f.split(".")[:-1]))
with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
with open(source, "rb") as in_file, open(dest, "wb") as out_file:
f = Fernet(key)
data = f.decrypt(in_file.read())
out_file.write(data)
sys.stdout.write("[+] Writing file: {0} Source: {1}\n".format(dest, source))
sys.stdout.write(
"[+] Writing file: {0} Source: {1}\n".format(dest, source)
)
sys.stdout.write("[+] Keys have been unencrypted!\n")
@ -499,15 +541,16 @@ def publish_verisign_units():
:return:
"""
from lemur.plugins import plugins
v = plugins.get('verisign-issuer')
v = plugins.get("verisign-issuer")
units = v.get_available_units()
metrics = {}
for item in units:
if item['@type'] in metrics.keys():
metrics[item['@type']] += int(item['@remaining'])
if item["@type"] in metrics.keys():
metrics[item["@type"]] += int(item["@remaining"])
else:
metrics.update({item['@type']: int(item['@remaining'])})
metrics.update({item["@type"]: int(item["@remaining"])})
for name, value in metrics.items():
metric = [
@ -516,16 +559,16 @@ def publish_verisign_units():
"type": "GAUGE",
"name": "Symantec {0} Unit Count".format(name),
"tags": {},
"value": value
"value": value,
}
]
requests.post('http://localhost:8078/metrics', data=json.dumps(metric))
requests.post("http://localhost:8078/metrics", data=json.dumps(metric))
def main():
manager.add_command("start", LemurServer())
manager.add_command("runserver", Server(host='127.0.0.1', threaded=True))
manager.add_command("runserver", Server(host="127.0.0.1", threaded=True))
manager.add_command("clean", Clean())
manager.add_command("show_urls", ShowUrls())
manager.add_command("db", MigrateCommand)

8
lemur/metrics.py
View File

@ -11,6 +11,7 @@ class Metrics(object):
"""
:param app: The Flask application object. Defaults to None.
"""
_providers = []
def __init__(self, app=None):
@ -22,11 +23,14 @@ class Metrics(object):
:param app: The Flask application object.
"""
self._providers = app.config.get('METRIC_PROVIDERS', [])
self._providers = app.config.get("METRIC_PROVIDERS", [])
def send(self, metric_name, metric_type, metric_value, *args, **kwargs):
for provider in self._providers:
current_app.logger.debug(
"Sending metric '{metric}' to the {provider} provider.".format(metric=metric_name, provider=provider))
"Sending metric '{metric}' to the {provider} provider.".format(
metric=metric_name, provider=provider
)
)
p = plugins.get(provider)
p.submit(metric_name, metric_type, metric_value, *args, **kwargs)

23
lemur/migrations/env.py
View File

@ -19,8 +19,11 @@ fileConfig(config.config_file_name)
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from flask import current_app
config.set_main_option('sqlalchemy.url', current_app.config.get('SQLALCHEMY_DATABASE_URI'))
target_metadata = current_app.extensions['migrate'].db.metadata
config.set_main_option(
"sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
)
target_metadata = current_app.extensions["migrate"].db.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
@ -54,14 +57,18 @@ def run_migrations_online():
and associate a connection with the context.
"""
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
engine = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
connection = engine.connect()
context.configure(connection=connection,
context.configure(
connection=connection,
target_metadata=target_metadata,
**current_app.extensions['migrate'].configure_args)
**current_app.extensions["migrate"].configure_args
)
try:
with context.begin_transaction():
@ -69,8 +76,8 @@ def run_migrations_online():
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

23
lemur/migrations/versions/131ec6accff5_.py
View File

@ -7,8 +7,8 @@ Create Date: 2016-12-07 17:29:42.049986
"""
# revision identifiers, used by Alembic.
revision = '131ec6accff5'
down_revision = 'e3691fc396e9'
revision = "131ec6accff5"
down_revision = "e3691fc396e9"
from alembic import op
import sqlalchemy as sa
@ -16,13 +16,24 @@ import sqlalchemy as sa
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('certificates', sa.Column('rotation', sa.Boolean(), nullable=False, server_default=sa.false()))
op.add_column('endpoints', sa.Column('last_updated', sa.DateTime(), server_default=sa.text('now()'), nullable=False))
op.add_column(
"certificates",
sa.Column("rotation", sa.Boolean(), nullable=False, server_default=sa.false()),
)
op.add_column(
"endpoints",
sa.Column(
"last_updated",
sa.DateTime(),
server_default=sa.text("now()"),
nullable=False,
),
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('endpoints', 'last_updated')
op.drop_column('certificates', 'rotation')
op.drop_column("endpoints", "last_updated")
op.drop_column("certificates", "rotation")
# ### end Alembic commands ###

12
lemur/migrations/versions/1ae8e3104db8_.py
View File

@ -7,15 +7,19 @@ Create Date: 2017-07-13 12:32:09.162800
"""
# revision identifiers, used by Alembic.
revision = '1ae8e3104db8'
down_revision = 'a02a678ddc25'
revision = "1ae8e3104db8"
down_revision = "a02a678ddc25"
from alembic import op
def upgrade():
op.sync_enum_values('public', 'log_type', ['key_view'], ['create_cert', 'key_view', 'update_cert'])
op.sync_enum_values(
"public", "log_type", ["key_view"], ["create_cert", "key_view", "update_cert"]
)
def downgrade():
op.sync_enum_values('public', 'log_type', ['create_cert', 'key_view', 'update_cert'], ['key_view'])
op.sync_enum_values(
"public", "log_type", ["create_cert", "key_view", "update_cert"], ["key_view"]
)

10
lemur/migrations/versions/1db4f82bc780_.py
View File

@ -7,8 +7,8 @@ Create Date: 2018-08-03 12:56:44.565230
"""
# revision identifiers, used by Alembic.
revision = '1db4f82bc780'
down_revision = '3adfdd6598df'
revision = "1db4f82bc780"
down_revision = "3adfdd6598df"
import logging
@ -20,12 +20,14 @@ log = logging.getLogger(__name__)
def upgrade():
connection = op.get_bind()
result = connection.execute("""\
result = connection.execute(
"""\
UPDATE certificates
SET rotation_policy_id=(SELECT id FROM rotation_policies WHERE name='default')
WHERE rotation_policy_id IS NULL
RETURNING id
""")
"""
)
log.info("Filled rotation_policy for %d certificates" % result.rowcount)

84
lemur/migrations/versions/29d8c8455c86_.py
View File

@ -7,8 +7,8 @@ Create Date: 2016-06-28 16:05:25.720213
"""
# revision identifiers, used by Alembic.
revision = '29d8c8455c86'
down_revision = '3307381f3b88'
revision = "29d8c8455c86"
down_revision = "3307381f3b88"
from alembic import op
import sqlalchemy as sa
@ -17,46 +17,60 @@ from sqlalchemy.dialects import postgresql
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('ciphers',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=128), nullable=False),
sa.PrimaryKeyConstraint('id')
op.create_table(
"ciphers",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(length=128), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_table('policy',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=128), nullable=True),
sa.PrimaryKeyConstraint('id')
op.create_table(
"policy",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(length=128), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_table('policies_ciphers',
sa.Column('cipher_id', sa.Integer(), nullable=True),
sa.Column('policy_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['cipher_id'], ['ciphers.id'], ),
sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], )
op.create_table(
"policies_ciphers",
sa.Column("cipher_id", sa.Integer(), nullable=True),
sa.Column("policy_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(["cipher_id"], ["ciphers.id"]),
sa.ForeignKeyConstraint(["policy_id"], ["policy.id"]),
)
op.create_index('policies_ciphers_ix', 'policies_ciphers', ['cipher_id', 'policy_id'], unique=False)
op.create_table('endpoints',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('owner', sa.String(length=128), nullable=True),
sa.Column('name', sa.String(length=128), nullable=True),
sa.Column('dnsname', sa.String(length=256), nullable=True),
sa.Column('type', sa.String(length=128), nullable=True),
sa.Column('active', sa.Boolean(), nullable=True),
sa.Column('port', sa.Integer(), nullable=True),
sa.Column('date_created', sa.DateTime(), server_default=sa.text(u'now()'), nullable=False),
sa.Column('policy_id', sa.Integer(), nullable=True),
sa.Column('certificate_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], ),
sa.PrimaryKeyConstraint('id')
op.create_index(
"policies_ciphers_ix",
"policies_ciphers",
["cipher_id", "policy_id"],
unique=False,
)
op.create_table(
"endpoints",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("owner", sa.String(length=128), nullable=True),
sa.Column("name", sa.String(length=128), nullable=True),
sa.Column("dnsname", sa.String(length=256), nullable=True),
sa.Column("type", sa.String(length=128), nullable=True),
sa.Column("active", sa.Boolean(), nullable=True),
sa.Column("port", sa.Integer(), nullable=True),
sa.Column(
"date_created",
sa.DateTime(),
server_default=sa.text(u"now()"),
nullable=False,
),
sa.Column("policy_id", sa.Integer(), nullable=True),
sa.Column("certificate_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(["certificate_id"], ["certificates.id"]),
sa.ForeignKeyConstraint(["policy_id"], ["policy.id"]),
sa.PrimaryKeyConstraint("id"),
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('endpoints')
op.drop_index('policies_ciphers_ix', table_name='policies_ciphers')
op.drop_table('policies_ciphers')
op.drop_table('policy')
op.drop_table('ciphers')
op.drop_table("endpoints")
op.drop_index("policies_ciphers_ix", table_name="policies_ciphers")
op.drop_table("policies_ciphers")
op.drop_table("policy")
op.drop_table("ciphers")
### end Alembic commands ###

23
lemur/migrations/versions/318b66568358_.py Normal file
View File

@ -0,0 +1,23 @@
""" Set 'deleted' flag from null to false on all certificates once
Revision ID: 318b66568358
Revises: 9f79024fe67b
Create Date: 2019-02-05 15:42:25.477587
"""
# revision identifiers, used by Alembic.
revision = "318b66568358"
down_revision = "9f79024fe67b"
from alembic import op
def upgrade():
connection = op.get_bind()
# Delete duplicate entries
connection.execute("UPDATE certificates SET deleted = false WHERE deleted IS NULL")
def downgrade():
pass

160
lemur/migrations/versions/3307381f3b88_.py
View File

@ -12,8 +12,8 @@ Create Date: 2016-05-20 17:33:04.360687
"""
# revision identifiers, used by Alembic.
revision = '3307381f3b88'
down_revision = '412b22cb656a'
revision = "3307381f3b88"
down_revision = "412b22cb656a"
from alembic import op
import sqlalchemy as sa
@ -23,109 +23,165 @@ from sqlalchemy.dialects import postgresql
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.alter_column('authorities', 'owner',
existing_type=sa.VARCHAR(length=128),
nullable=True)
op.drop_column('authorities', 'not_after')
op.drop_column('authorities', 'bits')
op.drop_column('authorities', 'cn')
op.drop_column('authorities', 'not_before')
op.add_column('certificates', sa.Column('root_authority_id', sa.Integer(), nullable=True))
op.alter_column('certificates', 'body',
existing_type=sa.TEXT(),
nullable=False)
op.alter_column('certificates', 'owner',
existing_type=sa.VARCHAR(length=128),
nullable=True)
op.drop_constraint(u'certificates_authority_id_fkey', 'certificates', type_='foreignkey')
op.create_foreign_key(None, 'certificates', 'authorities', ['authority_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(None, 'certificates', 'authorities', ['root_authority_id'], ['id'], ondelete='CASCADE')
op.alter_column(
"authorities", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
)
op.drop_column("authorities", "not_after")
op.drop_column("authorities", "bits")
op.drop_column("authorities", "cn")
op.drop_column("authorities", "not_before")
op.add_column(
"certificates", sa.Column("root_authority_id", sa.Integer(), nullable=True)
)
op.alter_column("certificates", "body", existing_type=sa.TEXT(), nullable=False)
op.alter_column(
"certificates", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
)
op.drop_constraint(
u"certificates_authority_id_fkey", "certificates", type_="foreignkey"
)
op.create_foreign_key(
None,
"certificates",
"authorities",
["authority_id"],
["id"],
ondelete="CASCADE",
)
op.create_foreign_key(
None,
"certificates",
"authorities",
["root_authority_id"],
["id"],
ondelete="CASCADE",
)
### end Alembic commands ###
# link existing certificate to their authority certificates
conn = op.get_bind()
for id, body, owner in conn.execute(text('select id, body, owner from authorities')):
for id, body, owner in conn.execute(
text("select id, body, owner from authorities")
):
if not owner:
owner = "lemur@nobody"
# look up certificate by body, if duplications are found, pick one
stmt = text('select id from certificates where body=:body')
stmt = text("select id from certificates where body=:body")
stmt = stmt.bindparams(body=body)
root_certificate = conn.execute(stmt).fetchone()
if root_certificate:
stmt = text('update certificates set root_authority_id=:root_authority_id where id=:id')
stmt = text(
"update certificates set root_authority_id=:root_authority_id where id=:id"
)
stmt = stmt.bindparams(root_authority_id=id, id=root_certificate[0])
op.execute(stmt)
# link owner roles to their authorities
stmt = text('select id from roles where name=:name')
stmt = text("select id from roles where name=:name")
stmt = stmt.bindparams(name=owner)
owner_role = conn.execute(stmt).fetchone()
if not owner_role:
stmt = text('insert into roles (name, description) values (:name, :description)')
stmt = stmt.bindparams(name=owner, description='Lemur generated role or existing owner.')
stmt = text(
"insert into roles (name, description) values (:name, :description)"
)
stmt = stmt.bindparams(
name=owner, description="Lemur generated role or existing owner."
)
op.execute(stmt)
stmt = text('select id from roles where name=:name')
stmt = text("select id from roles where name=:name")
stmt = stmt.bindparams(name=owner)
owner_role = conn.execute(stmt).fetchone()
stmt = text('select * from roles_authorities where role_id=:role_id and authority_id=:authority_id')
stmt = text(
"select * from roles_authorities where role_id=:role_id and authority_id=:authority_id"
)
stmt = stmt.bindparams(role_id=owner_role[0], authority_id=id)
exists = conn.execute(stmt).fetchone()
if not exists:
stmt = text('insert into roles_authorities (role_id, authority_id) values (:role_id, :authority_id)')
stmt = text(
"insert into roles_authorities (role_id, authority_id) values (:role_id, :authority_id)"
)
stmt = stmt.bindparams(role_id=owner_role[0], authority_id=id)
op.execute(stmt)
# link owner roles to their certificates
for id, owner in conn.execute(text('select id, owner from certificates')):
for id, owner in conn.execute(text("select id, owner from certificates")):
if not owner:
owner = "lemur@nobody"
stmt = text('select id from roles where name=:name')
stmt = text("select id from roles where name=:name")
stmt = stmt.bindparams(name=owner)
owner_role = conn.execute(stmt).fetchone()
if not owner_role:
stmt = text('insert into roles (name, description) values (:name, :description)')
stmt = stmt.bindparams(name=owner, description='Lemur generated role or existing owner.')
stmt = text(
"insert into roles (name, description) values (:name, :description)"
)
stmt = stmt.bindparams(
name=owner, description="Lemur generated role or existing owner."
)
op.execute(stmt)
# link owner roles to their authorities
stmt = text('select id from roles where name=:name')
stmt = text("select id from roles where name=:name")
stmt = stmt.bindparams(name=owner)
owner_role = conn.execute(stmt).fetchone()
stmt = text('select * from roles_certificates where role_id=:role_id and certificate_id=:certificate_id')
stmt = text(
"select * from roles_certificates where role_id=:role_id and certificate_id=:certificate_id"
)
stmt = stmt.bindparams(role_id=owner_role[0], certificate_id=id)
exists = conn.execute(stmt).fetchone()
if not exists:
stmt = text('insert into roles_certificates (role_id, certificate_id) values (:role_id, :certificate_id)')
stmt = text(
"insert into roles_certificates (role_id, certificate_id) values (:role_id, :certificate_id)"
)
stmt = stmt.bindparams(role_id=owner_role[0], certificate_id=id)
op.execute(stmt)
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, 'certificates', type_='foreignkey')
op.drop_constraint(None, 'certificates', type_='foreignkey')
op.create_foreign_key(u'certificates_authority_id_fkey', 'certificates', 'authorities', ['authority_id'], ['id'])
op.alter_column('certificates', 'owner',
existing_type=sa.VARCHAR(length=128),
nullable=True)
op.alter_column('certificates', 'body',
existing_type=sa.TEXT(),
nullable=True)
op.drop_column('certificates', 'root_authority_id')
op.add_column('authorities', sa.Column('not_before', postgresql.TIMESTAMP(), autoincrement=False, nullable=True))
op.add_column('authorities', sa.Column('cn', sa.VARCHAR(length=128), autoincrement=False, nullable=True))
op.add_column('authorities', sa.Column('bits', sa.INTEGER(), autoincrement=False, nullable=True))
op.add_column('authorities', sa.Column('not_after', postgresql.TIMESTAMP(), autoincrement=False, nullable=True))
op.alter_column('authorities', 'owner',
existing_type=sa.VARCHAR(length=128),
nullable=True)
op.drop_constraint(None, "certificates", type_="foreignkey")
op.drop_constraint(None, "certificates", type_="foreignkey")
op.create_foreign_key(
u"certificates_authority_id_fkey",
"certificates",
"authorities",
["authority_id"],
["id"],
)
op.alter_column(
"certificates", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
)
op.alter_column("certificates", "body", existing_type=sa.TEXT(), nullable=True)
op.drop_column("certificates", "root_authority_id")
op.add_column(
"authorities",
sa.Column(
"not_before", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
),
)
op.add_column(
"authorities",
sa.Column("cn", sa.VARCHAR(length=128), autoincrement=False, nullable=True),
)
op.add_column(
"authorities",
sa.Column("bits", sa.INTEGER(), autoincrement=False, nullable=True),
)
op.add_column(
"authorities",
sa.Column(
"not_after", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
),
)
op.alter_column(
"authorities", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
)
### end Alembic commands ###

20
lemur/migrations/versions/33de094da890_.py
View File

@ -7,25 +7,31 @@ Create Date: 2015-11-30 15:40:19.827272
"""
# revision identifiers, used by Alembic.
revision = '33de094da890'
revision = "33de094da890"
down_revision = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('certificate_replacement_associations',
sa.Column('replaced_certificate_id', sa.Integer(), nullable=True),
sa.Column('certificate_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['replaced_certificate_id'], ['certificates.id'], ondelete='cascade')
op.create_table(
"certificate_replacement_associations",
sa.Column("replaced_certificate_id", sa.Integer(), nullable=True),
sa.Column("certificate_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["certificate_id"], ["certificates.id"], ondelete="cascade"
),
sa.ForeignKeyConstraint(
["replaced_certificate_id"], ["certificates.id"], ondelete="cascade"
),
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('certificate_replacement_associations')
op.drop_table("certificate_replacement_associations")
### end Alembic commands ###

122
lemur/migrations/versions/3adfdd6598df_.py
View File

@ -7,8 +7,8 @@ Create Date: 2018-04-10 13:25:47.007556
"""
# revision identifiers, used by Alembic.
revision = '3adfdd6598df'
down_revision = '556ceb3e3c3e'
revision = "3adfdd6598df"
down_revision = "556ceb3e3c3e"
import sqlalchemy as sa
from alembic import op
@ -22,84 +22,90 @@ def upgrade():
# create provider table
print("Creating dns_providers table")
op.create_table(
'dns_providers',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=256), nullable=True),
sa.Column('description', sa.String(length=1024), nullable=True),
sa.Column('provider_type', sa.String(length=256), nullable=True),
sa.Column('credentials', Vault(), nullable=True),
sa.Column('api_endpoint', sa.String(length=256), nullable=True),
sa.Column('date_created', ArrowType(), server_default=sa.text('now()'), nullable=False),
sa.Column('status', sa.String(length=128), nullable=True),
sa.Column('options', JSON),
sa.Column('domains', sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
"dns_providers",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("name", sa.String(length=256), nullable=True),
sa.Column("description", sa.String(length=1024), nullable=True),
sa.Column("provider_type", sa.String(length=256), nullable=True),
sa.Column("credentials", Vault(), nullable=True),
sa.Column("api_endpoint", sa.String(length=256), nullable=True),
sa.Column(
"date_created", ArrowType(), server_default=sa.text("now()"), nullable=False
),
sa.Column("status", sa.String(length=128), nullable=True),
sa.Column("options", JSON),
sa.Column("domains", sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("name"),
)
print("Adding dns_provider_id column to certificates")
op.add_column('certificates', sa.Column('dns_provider_id', sa.Integer(), nullable=True))
op.add_column(
"certificates", sa.Column("dns_provider_id", sa.Integer(), nullable=True)
)
print("Adding dns_provider_id column to pending_certs")
op.add_column('pending_certs', sa.Column('dns_provider_id', sa.Integer(), nullable=True))
op.add_column(
"pending_certs", sa.Column("dns_provider_id", sa.Integer(), nullable=True)
)
print("Adding options column to pending_certs")
op.add_column('pending_certs', sa.Column('options', JSON))
op.add_column("pending_certs", sa.Column("options", JSON))
print("Creating pending_dns_authorizations table")
op.create_table(
'pending_dns_authorizations',
sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
sa.Column('account_number', sa.String(length=128), nullable=True),
sa.Column('domains', JSON, nullable=True),
sa.Column('dns_provider_type', sa.String(length=128), nullable=True),
sa.Column('options', JSON, nullable=True),
"pending_dns_authorizations",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("account_number", sa.String(length=128), nullable=True),
sa.Column("domains", JSON, nullable=True),
sa.Column("dns_provider_type", sa.String(length=128), nullable=True),
sa.Column("options", JSON, nullable=True),
)
print("Creating certificates_dns_providers_fk foreign key")
op.create_foreign_key('certificates_dns_providers_fk', 'certificates', 'dns_providers', ['dns_provider_id'], ['id'],
ondelete='cascade')
op.create_foreign_key(
"certificates_dns_providers_fk",
"certificates",
"dns_providers",
["dns_provider_id"],
["id"],
ondelete="cascade",
)
print("Altering column types in the api_keys table")
op.alter_column('api_keys', 'issued_at',
existing_type=sa.BIGINT(),
nullable=True)
op.alter_column('api_keys', 'revoked',
existing_type=sa.BOOLEAN(),
nullable=True)
op.alter_column('api_keys', 'ttl',
existing_type=sa.BIGINT(),
nullable=True)
op.alter_column('api_keys', 'user_id',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column("api_keys", "issued_at", existing_type=sa.BIGINT(), nullable=True)
op.alter_column("api_keys", "revoked", existing_type=sa.BOOLEAN(), nullable=True)
op.alter_column("api_keys", "ttl", existing_type=sa.BIGINT(), nullable=True)
op.alter_column("api_keys", "user_id", existing_type=sa.INTEGER(), nullable=True)
print("Creating dns_providers_id foreign key on pending_certs table")
op.create_foreign_key(None, 'pending_certs', 'dns_providers', ['dns_provider_id'], ['id'], ondelete='CASCADE')
op.create_foreign_key(
None,
"pending_certs",
"dns_providers",
["dns_provider_id"],
["id"],
ondelete="CASCADE",
)
def downgrade():
print("Removing dns_providers_id foreign key on pending_certs table")
op.drop_constraint(None, 'pending_certs', type_='foreignkey')
op.drop_constraint(None, "pending_certs", type_="foreignkey")
print("Reverting column types in the api_keys table")
op.alter_column('api_keys', 'user_id',
existing_type=sa.INTEGER(),
nullable=False)
op.alter_column('api_keys', 'ttl',
existing_type=sa.BIGINT(),
nullable=False)
op.alter_column('api_keys', 'revoked',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('api_keys', 'issued_at',
existing_type=sa.BIGINT(),
nullable=False)
op.alter_column("api_keys", "user_id", existing_type=sa.INTEGER(), nullable=False)
op.alter_column("api_keys", "ttl", existing_type=sa.BIGINT(), nullable=False)
op.alter_column("api_keys", "revoked", existing_type=sa.BOOLEAN(), nullable=False)
op.alter_column("api_keys", "issued_at", existing_type=sa.BIGINT(), nullable=False)
print("Reverting certificates_dns_providers_fk foreign key")
op.drop_constraint('certificates_dns_providers_fk', 'certificates', type_='foreignkey')
op.drop_constraint(
"certificates_dns_providers_fk", "certificates", type_="foreignkey"
)
print("Dropping pending_dns_authorizations table")
op.drop_table('pending_dns_authorizations')
op.drop_table("pending_dns_authorizations")
print("Undoing modifications to pending_certs table")
op.drop_column('pending_certs', 'options')
op.drop_column('pending_certs', 'dns_provider_id')
op.drop_column("pending_certs", "options")
op.drop_column("pending_certs", "dns_provider_id")
print("Undoing modifications to certificates table")
op.drop_column('certificates', 'dns_provider_id')
op.drop_column("certificates", "dns_provider_id")
print("Deleting dns_providers table")
op.drop_table('dns_providers')
op.drop_table("dns_providers")

119
lemur/migrations/versions/412b22cb656a_.py
View File

@ -7,8 +7,8 @@ Create Date: 2016-05-17 17:37:41.210232
"""
# revision identifiers, used by Alembic.
revision = '412b22cb656a'
down_revision = '4c50b903d1ae'
revision = "412b22cb656a"
down_revision = "4c50b903d1ae"
from alembic import op
import sqlalchemy as sa
@ -17,47 +17,102 @@ from sqlalchemy.sql import text
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('roles_authorities',
sa.Column('authority_id', sa.Integer(), nullable=True),
sa.Column('role_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['authority_id'], ['authorities.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
op.create_table(
"roles_authorities",
sa.Column("authority_id", sa.Integer(), nullable=True),
sa.Column("role_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(["authority_id"], ["authorities.id"]),
sa.ForeignKeyConstraint(["role_id"], ["roles.id"]),
)
op.create_index('roles_authorities_ix', 'roles_authorities', ['authority_id', 'role_id'], unique=True)
op.create_table('roles_certificates',
sa.Column('certificate_id', sa.Integer(), nullable=True),
sa.Column('role_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
op.create_index(
"roles_authorities_ix",
"roles_authorities",
["authority_id", "role_id"],
unique=True,
)
op.create_table(
"roles_certificates",
sa.Column("certificate_id", sa.Integer(), nullable=True),
sa.Column("role_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(["certificate_id"], ["certificates.id"]),
sa.ForeignKeyConstraint(["role_id"], ["roles.id"]),
)
op.create_index(
"roles_certificates_ix",
"roles_certificates",
["certificate_id", "role_id"],
unique=True,
)
op.create_index(
"certificate_associations_ix",
"certificate_associations",
["domain_id", "certificate_id"],
unique=True,
)
op.create_index(
"certificate_destination_associations_ix",
"certificate_destination_associations",
["destination_id", "certificate_id"],
unique=True,
)
op.create_index(
"certificate_notification_associations_ix",
"certificate_notification_associations",
["notification_id", "certificate_id"],
unique=True,
)
op.create_index(
"certificate_replacement_associations_ix",
"certificate_replacement_associations",
["certificate_id", "certificate_id"],
unique=True,
)
op.create_index(
"certificate_source_associations_ix",
"certificate_source_associations",
["source_id", "certificate_id"],
unique=True,
)
op.create_index(
"roles_users_ix", "roles_users", ["user_id", "role_id"], unique=True
)
op.create_index('roles_certificates_ix', 'roles_certificates', ['certificate_id', 'role_id'], unique=True)
op.create_index('certificate_associations_ix', 'certificate_associations', ['domain_id', 'certificate_id'], unique=True)
op.create_index('certificate_destination_associations_ix', 'certificate_destination_associations', ['destination_id', 'certificate_id'], unique=True)
op.create_index('certificate_notification_associations_ix', 'certificate_notification_associations', ['notification_id', 'certificate_id'], unique=True)
op.create_index('certificate_replacement_associations_ix', 'certificate_replacement_associations', ['certificate_id', 'certificate_id'], unique=True)
op.create_index('certificate_source_associations_ix', 'certificate_source_associations', ['source_id', 'certificate_id'], unique=True)
op.create_index('roles_users_ix', 'roles_users', ['user_id', 'role_id'], unique=True)
### end Alembic commands ###
# migrate existing authority_id relationship to many_to_many
conn = op.get_bind()
for id, authority_id in conn.execute(text('select id, authority_id from roles where authority_id is not null')):
stmt = text('insert into roles_authoritties (role_id, authority_id) values (:role_id, :authority_id)')
for id, authority_id in conn.execute(
text("select id, authority_id from roles where authority_id is not null")
):
stmt = text(
"insert into roles_authoritties (role_id, authority_id) values (:role_id, :authority_id)"
)
stmt = stmt.bindparams(role_id=id, authority_id=authority_id)
op.execute(stmt)
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_index('roles_users_ix', table_name='roles_users')
op.drop_index('certificate_source_associations_ix', table_name='certificate_source_associations')
op.drop_index('certificate_replacement_associations_ix', table_name='certificate_replacement_associations')
op.drop_index('certificate_notification_associations_ix', table_name='certificate_notification_associations')
op.drop_index('certificate_destination_associations_ix', table_name='certificate_destination_associations')
op.drop_index('certificate_associations_ix', table_name='certificate_associations')
op.drop_index('roles_certificates_ix', table_name='roles_certificates')
op.drop_table('roles_certificates')
op.drop_index('roles_authorities_ix', table_name='roles_authorities')
op.drop_table('roles_authorities')
op.drop_index("roles_users_ix", table_name="roles_users")
op.drop_index(
"certificate_source_associations_ix",
table_name="certificate_source_associations",
)
op.drop_index(
"certificate_replacement_associations_ix",
table_name="certificate_replacement_associations",
)
op.drop_index(
"certificate_notification_associations_ix",
table_name="certificate_notification_associations",
)
op.drop_index(
"certificate_destination_associations_ix",
table_name="certificate_destination_associations",
)
op.drop_index("certificate_associations_ix", table_name="certificate_associations")
op.drop_index("roles_certificates_ix", table_name="roles_certificates")
op.drop_table("roles_certificates")
op.drop_index("roles_authorities_ix", table_name="roles_authorities")
op.drop_table("roles_authorities")
### end Alembic commands ###

14
lemur/migrations/versions/449c3d5c7299_.py
View File

@ -7,8 +7,8 @@ Create Date: 2018-02-24 22:51:35.369229
"""
# revision identifiers, used by Alembic.
revision = '449c3d5c7299'
down_revision = '5770674184de'
revision = "449c3d5c7299"
down_revision = "5770674184de"
from alembic import op
from flask_sqlalchemy import SQLAlchemy
@ -21,6 +21,16 @@ COLUMNS = ["notification_id", "certificate_id"]
def upgrade():
connection = op.get_bind()
# Delete duplicate entries
connection.execute(
"""\
DELETE FROM certificate_notification_associations WHERE ctid NOT IN (
-- Select the first tuple ID for each (notification_id, certificate_id) combination and keep that
SELECT min(ctid) FROM certificate_notification_associations GROUP BY notification_id, certificate_id
)
"""
)
op.create_unique_constraint(CONSTRAINT_NAME, TABLE, COLUMNS)

9
lemur/migrations/versions/4c50b903d1ae_.py
View File

@ -7,20 +7,21 @@ Create Date: 2015-12-30 10:19:30.057791
"""
# revision identifiers, used by Alembic.
revision = '4c50b903d1ae'
down_revision = '33de094da890'
revision = "4c50b903d1ae"
down_revision = "33de094da890"
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('domains', sa.Column('sensitive', sa.Boolean(), nullable=True))
op.add_column("domains", sa.Column("sensitive", sa.Boolean(), nullable=True))
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_column('domains', 'sensitive')
op.drop_column("domains", "sensitive")
### end Alembic commands ###

204
lemur/migrations/versions/556ceb3e3c3e_.py
View File

@ -7,8 +7,8 @@ Create Date: 2018-01-05 01:18:45.571595
"""
# revision identifiers, used by Alembic.
revision = '556ceb3e3c3e'
down_revision = '449c3d5c7299'
revision = "556ceb3e3c3e"
down_revision = "449c3d5c7299"
from alembic import op
import sqlalchemy as sa
@ -16,84 +16,150 @@ from lemur.utils import Vault
from sqlalchemy.dialects import postgresql
from sqlalchemy_utils import ArrowType
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('pending_certs',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('external_id', sa.String(length=128), nullable=True),
sa.Column('owner', sa.String(length=128), nullable=False),
sa.Column('name', sa.String(length=256), nullable=True),
sa.Column('description', sa.String(length=1024), nullable=True),
sa.Column('notify', sa.Boolean(), nullable=True),
sa.Column('number_attempts', sa.Integer(), nullable=True),
sa.Column('rename', sa.Boolean(), nullable=True),
sa.Column('cn', sa.String(length=128), nullable=True),
sa.Column('csr', sa.Text(), nullable=False),
sa.Column('chain', sa.Text(), nullable=True),
sa.Column('private_key', Vault(), nullable=True),
sa.Column('date_created', ArrowType(), server_default=sa.text('now()'), nullable=False),
sa.Column('status', sa.String(length=128), nullable=True),
sa.Column('rotation', sa.Boolean(), nullable=True),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('authority_id', sa.Integer(), nullable=True),
sa.Column('root_authority_id', sa.Integer(), nullable=True),
sa.Column('rotation_policy_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['authority_id'], ['authorities.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['root_authority_id'], ['authorities.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['rotation_policy_id'], ['rotation_policies.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
op.create_table(
"pending_certs",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("external_id", sa.String(length=128), nullable=True),
sa.Column("owner", sa.String(length=128), nullable=False),
sa.Column("name", sa.String(length=256), nullable=True),
sa.Column("description", sa.String(length=1024), nullable=True),
sa.Column("notify", sa.Boolean(), nullable=True),
sa.Column("number_attempts", sa.Integer(), nullable=True),
sa.Column("rename", sa.Boolean(), nullable=True),
sa.Column("cn", sa.String(length=128), nullable=True),
sa.Column("csr", sa.Text(), nullable=False),
sa.Column("chain", sa.Text(), nullable=True),
sa.Column("private_key", Vault(), nullable=True),
sa.Column(
"date_created", ArrowType(), server_default=sa.text("now()"), nullable=False
),
sa.Column("status", sa.String(length=128), nullable=True),
sa.Column("rotation", sa.Boolean(), nullable=True),
sa.Column("user_id", sa.Integer(), nullable=True),
sa.Column("authority_id", sa.Integer(), nullable=True),
sa.Column("root_authority_id", sa.Integer(), nullable=True),
sa.Column("rotation_policy_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["authority_id"], ["authorities.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["root_authority_id"], ["authorities.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(["rotation_policy_id"], ["rotation_policies.id"]),
sa.ForeignKeyConstraint(["user_id"], ["users.id"]),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("name"),
)
op.create_table('pending_cert_destination_associations',
sa.Column('destination_id', sa.Integer(), nullable=True),
sa.Column('pending_cert_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['destination_id'], ['destinations.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade')
op.create_table(
"pending_cert_destination_associations",
sa.Column("destination_id", sa.Integer(), nullable=True),
sa.Column("pending_cert_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["destination_id"], ["destinations.id"], ondelete="cascade"
),
sa.ForeignKeyConstraint(
["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
),
)
op.create_index('pending_cert_destination_associations_ix', 'pending_cert_destination_associations', ['destination_id', 'pending_cert_id'], unique=False)
op.create_table('pending_cert_notification_associations',
sa.Column('notification_id', sa.Integer(), nullable=True),
sa.Column('pending_cert_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade')
op.create_index(
"pending_cert_destination_associations_ix",
"pending_cert_destination_associations",
["destination_id", "pending_cert_id"],
unique=False,
)
op.create_index('pending_cert_notification_associations_ix', 'pending_cert_notification_associations', ['notification_id', 'pending_cert_id'], unique=False)
op.create_table('pending_cert_replacement_associations',
sa.Column('replaced_certificate_id', sa.Integer(), nullable=True),
sa.Column('pending_cert_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['replaced_certificate_id'], ['certificates.id'], ondelete='cascade')
op.create_table(
"pending_cert_notification_associations",
sa.Column("notification_id", sa.Integer(), nullable=True),
sa.Column("pending_cert_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["notification_id"], ["notifications.id"], ondelete="cascade"
),
sa.ForeignKeyConstraint(
["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
),
)
op.create_index('pending_cert_replacement_associations_ix', 'pending_cert_replacement_associations', ['replaced_certificate_id', 'pending_cert_id'], unique=False)
op.create_table('pending_cert_role_associations',
sa.Column('pending_cert_id', sa.Integer(), nullable=True),
sa.Column('role_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
op.create_index(
"pending_cert_notification_associations_ix",
"pending_cert_notification_associations",
["notification_id", "pending_cert_id"],
unique=False,
)
op.create_index('pending_cert_role_associations_ix', 'pending_cert_role_associations', ['pending_cert_id', 'role_id'], unique=False)
op.create_table('pending_cert_source_associations',
sa.Column('source_id', sa.Integer(), nullable=True),
sa.Column('pending_cert_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='cascade')
op.create_table(
"pending_cert_replacement_associations",
sa.Column("replaced_certificate_id", sa.Integer(), nullable=True),
sa.Column("pending_cert_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
),
sa.ForeignKeyConstraint(
["replaced_certificate_id"], ["certificates.id"], ondelete="cascade"
),
)
op.create_index(
"pending_cert_replacement_associations_ix",
"pending_cert_replacement_associations",
["replaced_certificate_id", "pending_cert_id"],
unique=False,
)
op.create_table(
"pending_cert_role_associations",
sa.Column("pending_cert_id", sa.Integer(), nullable=True),
sa.Column("role_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(["pending_cert_id"], ["pending_certs.id"]),
sa.ForeignKeyConstraint(["role_id"], ["roles.id"]),
)
op.create_index(
"pending_cert_role_associations_ix",
"pending_cert_role_associations",
["pending_cert_id", "role_id"],
unique=False,
)
op.create_table(
"pending_cert_source_associations",
sa.Column("source_id", sa.Integer(), nullable=True),
sa.Column("pending_cert_id", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
),
sa.ForeignKeyConstraint(["source_id"], ["sources.id"], ondelete="cascade"),
)
op.create_index(
"pending_cert_source_associations_ix",
"pending_cert_source_associations",
["source_id", "pending_cert_id"],
unique=False,
)
op.create_index('pending_cert_source_associations_ix', 'pending_cert_source_associations', ['source_id', 'pending_cert_id'], unique=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index('pending_cert_source_associations_ix', table_name='pending_cert_source_associations')
op.drop_table('pending_cert_source_associations')
op.drop_index('pending_cert_role_associations_ix', table_name='pending_cert_role_associations')
op.drop_table('pending_cert_role_associations')
op.drop_index('pending_cert_replacement_associations_ix', table_name='pending_cert_replacement_associations')
op.drop_table('pending_cert_replacement_associations')
op.drop_index('pending_cert_notification_associations_ix', table_name='pending_cert_notification_associations')
op.drop_table('pending_cert_notification_associations')
op.drop_index('pending_cert_destination_associations_ix', table_name='pending_cert_destination_associations')
op.drop_table('pending_cert_destination_associations')
op.drop_table('pending_certs')
op.drop_index(
"pending_cert_source_associations_ix",
table_name="pending_cert_source_associations",
)
op.drop_table("pending_cert_source_associations")
op.drop_index(
"pending_cert_role_associations_ix", table_name="pending_cert_role_associations"
)
op.drop_table("pending_cert_role_associations")
op.drop_index(
"pending_cert_replacement_associations_ix",
table_name="pending_cert_replacement_associations",
)
op.drop_table("pending_cert_replacement_associations")
op.drop_index(
"pending_cert_notification_associations_ix",
table_name="pending_cert_notification_associations",
)
op.drop_table("pending_cert_notification_associations")
op.drop_index(
"pending_cert_destination_associations_ix",
table_name="pending_cert_destination_associations",
)
op.drop_table("pending_cert_destination_associations")
op.drop_table("pending_certs")
# ### end Alembic commands ###

8
lemur/migrations/versions/5770674184de_.py
View File

@ -7,8 +7,8 @@ Create Date: 2018-02-23 15:27:30.335435
"""
# revision identifiers, used by Alembic.
revision = '5770674184de'
down_revision = 'ce547319f7be'
revision = "5770674184de"
down_revision = "ce547319f7be"
from flask_sqlalchemy import SQLAlchemy
from lemur.models import certificate_notification_associations
@ -32,7 +32,9 @@ def upgrade():
# If we've seen a pair already, delete the duplicates
if seen.get("{}-{}".format(x.certificate_id, x.notification_id)):
print("Deleting duplicate: {}".format(x))
d = session.query(certificate_notification_associations).filter(certificate_notification_associations.c.id==x.id)
d = session.query(certificate_notification_associations).filter(
certificate_notification_associations.c.id == x.id
)
d.delete(synchronize_session=False)
seen["{}-{}".format(x.certificate_id, x.notification_id)] = True
db.session.commit()

15
lemur/migrations/versions/5ae0ecefb01f_.py
View File

@ -7,8 +7,8 @@ Create Date: 2018-08-14 08:16:43.329316
"""
# revision identifiers, used by Alembic.
revision = '5ae0ecefb01f'
down_revision = '1db4f82bc780'
revision = "5ae0ecefb01f"
down_revision = "1db4f82bc780"
from alembic import op
import sqlalchemy as sa
@ -16,17 +16,14 @@ import sqlalchemy as sa
def upgrade():
op.alter_column(
table_name='pending_certs',
column_name='status',
nullable=True,
type_=sa.TEXT()
table_name="pending_certs", column_name="status", nullable=True, type_=sa.TEXT()
)
def downgrade():
op.alter_column(
table_name='pending_certs',
column_name='status',
table_name="pending_certs",
column_name="status",
nullable=True,
type_=sa.VARCHAR(128)
type_=sa.VARCHAR(128),
)

10
lemur/migrations/versions/5bc47fa7cac4_.py
View File

@ -7,16 +7,18 @@ Create Date: 2017-12-08 14:19:11.903864
"""
# revision identifiers, used by Alembic.
revision = '5bc47fa7cac4'
down_revision = 'c05a8998b371'
revision = "5bc47fa7cac4"
down_revision = "c05a8998b371"
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('roles', sa.Column('third_party', sa.Boolean(), nullable=True, default=False))
op.add_column(
"roles", sa.Column("third_party", sa.Boolean(), nullable=True, default=False)
)
def downgrade():
op.drop_column('roles', 'third_party')
op.drop_column("roles", "third_party")

16
lemur/migrations/versions/5e680529b666_.py
View File

@ -7,20 +7,20 @@ Create Date: 2017-01-26 05:05:25.168125
"""
# revision identifiers, used by Alembic.
revision = '5e680529b666'
down_revision = '131ec6accff5'
revision = "5e680529b666"
down_revision = "131ec6accff5"
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('endpoints', sa.Column('sensitive', sa.Boolean(), nullable=True))
op.add_column('endpoints', sa.Column('source_id', sa.Integer(), nullable=True))
op.create_foreign_key(None, 'endpoints', 'sources', ['source_id'], ['id'])
op.add_column("endpoints", sa.Column("sensitive", sa.Boolean(), nullable=True))
op.add_column("endpoints", sa.Column("source_id", sa.Integer(), nullable=True))
op.create_foreign_key(None, "endpoints", "sources", ["source_id"], ["id"])
def downgrade():
op.drop_constraint(None, 'endpoints', type_='foreignkey')
op.drop_column('endpoints', 'source_id')
op.drop_column('endpoints', 'sensitive')
op.drop_constraint(None, "endpoints", type_="foreignkey")
op.drop_column("endpoints", "source_id")
op.drop_column("endpoints", "sensitive")

Some files were not shown because too many files have changed in this diff Show More