Initial LetsEncrypt / Celery docs

This commit is contained in:
Curtis Castrapel 2019-07-09 11:13:11 -07:00
parent c7398d9e2f
commit 8eb639e366
10 changed files with 229 additions and 99 deletions

View File

@ -36,7 +36,7 @@ endif
@echo "" @echo ""
dev-docs: dev-docs:
pip install -r docs/requirements.txt pip install -r requirements-docs.txt
reset-db: reset-db:
@echo "--> Dropping existing 'lemur' database" @echo "--> Dropping existing 'lemur' database"

View File

@ -1090,7 +1090,9 @@ Verisign/Symantec
----------------- -----------------
:Authors: :Authors:
Kevin Glisson <kglisson@netflix.com> Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type: :Type:
Issuer Issuer
:Description: :Description:
@ -1116,6 +1118,8 @@ Acme
:Authors: :Authors:
Kevin Glisson <kglisson@netflix.com>, Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>,
Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com> Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
:Type: :Type:
Issuer Issuer
@ -1127,7 +1131,9 @@ Atlas
----- -----
:Authors: :Authors:
Kevin Glisson <kglisson@netflix.com> Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type: :Type:
Metric Metric
:Description: :Description:
@ -1138,7 +1144,9 @@ Email
----- -----
:Authors: :Authors:
Kevin Glisson <kglisson@netflix.com> Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type: :Type:
Notification Notification
:Description: :Description:
@ -1160,7 +1168,9 @@ AWS
---- ----
:Authors: :Authors:
Kevin Glisson <kglisson@netflix.com> Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type: :Type:
Source Source
:Description: :Description:
@ -1171,7 +1181,9 @@ AWS
---- ----
:Authors: :Authors:
Kevin Glisson <kglisson@netflix.com> Kevin Glisson <kglisson@netflix.com>,
Curtis Castrapel <ccastrapel@netflix.com>,
Hossein Shafagh <hshafagh@netflix.com>
:Type: :Type:
Destination Destination
:Description: :Description:

View File

@ -18,17 +18,18 @@ from unittest.mock import MagicMock
# If extensions (or modules to document with autodoc) are in another directory, # If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the # add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here. # documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('..')) sys.path.insert(0, os.path.abspath(".."))
# Mock packages that cannot be installed on rtd # Mock packages that cannot be installed on rtd
on_rtd = os.environ.get('READTHEDOCS') == 'True' on_rtd = os.environ.get("READTHEDOCS") == "True"
if on_rtd: if on_rtd:
class Mock(MagicMock): class Mock(MagicMock):
@classmethod @classmethod
def __getattr__(cls, name): def __getattr__(cls, name):
return MagicMock() return MagicMock()
MOCK_MODULES = ['ldap'] MOCK_MODULES = ["ldap"]
sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
# -- General configuration ------------------------------------------------ # -- General configuration ------------------------------------------------
@ -39,27 +40,23 @@ if on_rtd:
# Add any Sphinx extension module names here, as strings. They can be # Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones. # ones.
extensions = [ extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask", "sphinx.ext.todo"]
'sphinx.ext.autodoc',
'sphinxcontrib.autohttp.flask',
'sphinx.ext.todo',
]
# Add any paths that contain templates here, relative to this directory. # Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates'] templates_path = ["_templates"]
# The suffix of source filenames. # The suffix of source filenames.
source_suffix = '.rst' source_suffix = ".rst"
# The encoding of source files. # The encoding of source files.
# source_encoding = 'utf-8-sig' # source_encoding = 'utf-8-sig'
# The master toctree document. # The master toctree document.
master_doc = 'index' master_doc = "index"
# General information about the project. # General information about the project.
project = u'lemur' project = u"lemur"
copyright = u'2018, Netflix Inc.' copyright = u"2018, Netflix Inc."
# The version info for the project you're documenting, acts as replacement for # The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the # |version| and |release|, also used in various other places throughout the
@ -84,7 +81,7 @@ version = release = about["__version__"]
# List of patterns, relative to source directory, that match files and # List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files. # directories to ignore when looking for source files.
exclude_patterns = ['_build'] exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all # The reST default role (used for this markup: `text`) to use for all
# documents. # documents.
@ -102,7 +99,7 @@ exclude_patterns = ['_build']
# show_authors = False # show_authors = False
# The name of the Pygments (syntax highlighting) style to use. # The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx' pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting. # A list of ignored prefixes for module index sorting.
# modindex_common_prefix = [] # modindex_common_prefix = []
@ -114,11 +111,12 @@ pygments_style = 'sphinx'
# -- Options for HTML output ---------------------------------------------- # -- Options for HTML output ----------------------------------------------
# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True' on_rtd = os.environ.get("READTHEDOCS", None) == "True"
if not on_rtd: # only import and set the theme if we're building docs locally if not on_rtd: # only import and set the theme if we're building docs locally
import sphinx_rtd_theme import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
html_theme = "sphinx_rtd_theme"
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Theme options are theme-specific and customize the look and feel of a theme # Theme options are theme-specific and customize the look and feel of a theme
@ -148,7 +146,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally
# Add any paths that contain custom static files (such as style sheets) here, # Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files, # relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css". # so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static'] html_static_path = ["_static"]
# Add any extra paths that contain custom files (such as robots.txt or # Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied # .htaccess) here, relative to this directory. These files are copied
@ -197,7 +195,7 @@ html_static_path = ['_static']
# html_file_suffix = None # html_file_suffix = None
# Output file base name for HTML help builder. # Output file base name for HTML help builder.
htmlhelp_basename = 'lemurdoc' htmlhelp_basename = "lemurdoc"
# -- Options for LaTeX output --------------------------------------------- # -- Options for LaTeX output ---------------------------------------------
@ -205,10 +203,8 @@ htmlhelp_basename = 'lemurdoc'
latex_elements = { latex_elements = {
# The paper size ('letterpaper' or 'a4paper'). # The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper', #'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt'). # The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt', #'pointsize': '10pt',
# Additional stuff for the LaTeX preamble. # Additional stuff for the LaTeX preamble.
#'preamble': '', #'preamble': '',
} }
@ -217,8 +213,7 @@ latex_elements = {
# (source start file, target name, title, # (source start file, target name, title,
# author, documentclass [howto, manual, or own class]). # author, documentclass [howto, manual, or own class]).
latex_documents = [ latex_documents = [
('index', 'lemur.tex', u'Lemur Documentation', ("index", "lemur.tex", u"Lemur Documentation", u"Netflix Security", "manual")
u'Kevin Glisson', 'manual'),
] ]
# The name of an image file (relative to this directory) to place at the top of # The name of an image file (relative to this directory) to place at the top of
@ -246,10 +241,7 @@ latex_documents = [
# One entry per manual page. List of tuples # One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section). # (source start file, name, description, authors, manual section).
man_pages = [ man_pages = [("index", "Lemur", u"Lemur Documentation", [u"Netflix Security"], 1)]
('index', 'Lemur', u'Lemur Documentation',
[u'Kevin Glisson'], 1)
]
# If true, show URL addresses after external links. # If true, show URL addresses after external links.
# man_show_urls = False # man_show_urls = False
@ -261,9 +253,15 @@ man_pages = [
# (source start file, target name, title, author, # (source start file, target name, title, author,
# dir menu entry, description, category) # dir menu entry, description, category)
texinfo_documents = [ texinfo_documents = [
('index', 'Lemur', u'Lemur Documentation', (
u'Kevin Glisson', 'Lemur', 'SSL Certificate Management', "index",
'Miscellaneous'), "Lemur",
u"Lemur Documentation",
u"Netflix Security",
"Lemur",
"SSL Certificate Management",
"Miscellaneous",
)
] ]
# Documents to append as an appendix to all manuals. # Documents to append as an appendix to all manuals.

View File

@ -22,12 +22,18 @@ Once you've got all that, the rest is simple:
# If you have a fork, you'll want to clone it instead # If you have a fork, you'll want to clone it instead
git clone git://github.com/netflix/lemur.git git clone git://github.com/netflix/lemur.git
# Create a python virtualenv # Create and activate python virtualenv from within the lemur repo
mkvirtualenv lemur python3 -m venv env
. env/bin/activate
# Install doc requirements
# Make the magic happen
make dev-docs make dev-docs
# Make the docs
cd docs
make html
Running ``make dev-docs`` will install the basic requirements to get Sphinx running. Running ``make dev-docs`` will install the basic requirements to get Sphinx running.
@ -58,7 +64,7 @@ Once you've got all that, the rest is simple:
git clone git://github.com/lemur/lemur.git git clone git://github.com/lemur/lemur.git
# Create a python virtualenv # Create a python virtualenv
mkvirtualenv lemur python3 -m venv env
# Make the magic happen # Make the magic happen
make make
@ -135,7 +141,7 @@ The test suite consists of multiple parts, testing both the Python and JavaScrip
make test make test
If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``test-js`` for the JavaScript tests. If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``make test-js`` for the JavaScript tests.
You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests. You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests.

Binary file not shown.

After

Width:  |  Height:  |  Size: 86 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 132 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 218 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 89 KiB

View File

@ -318,7 +318,7 @@ Periodic Tasks
============== ==============
Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
a cron job that runs the commands. celery tasks or cron jobs that run these commands.
There are currently three commands that could/should be run on a periodic basis: There are currently three commands that could/should be run on a periodic basis:
@ -326,11 +326,124 @@ There are currently three commands that could/should be run on a periodic basis:
- `check_revoked` - `check_revoked`
- `sync` - `sync`
If you are using LetsEncrypt, you must also run the following:
- `fetch_all_pending_acme_certs`
- `remove_old_acme_certs`
How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day. How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
`sync` is typically run every 15 minutes. `sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
`remove_old_acme_certs` can be ran more rarely, such as once every week.
Example cron entries:: Example cron entries::
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
*/15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
Example Celery configuration (To be placed in your configuration file)::
CELERYBEAT_SCHEDULE = {
'fetch_all_pending_acme_certs': {
'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
'options': {
'expires': 180
},
'schedule': crontab(minute="*"),
},
'remove_old_acme_certs': {
'task': 'lemur.common.celery.remove_old_acme_certs',
'options': {
'expires': 180
},
'schedule': crontab(hour=7, minute=30, day_of_week=1),
},
'clean_all_sources': {
'task': 'lemur.common.celery.clean_all_sources',
'options': {
'expires': 180
},
'schedule': crontab(hour=1, minute=0, day_of_week=1),
},
'sync_all_sources': {
'task': 'lemur.common.celery.sync_all_sources',
'options': {
'expires': 180
},
'schedule': crontab(hour="*/3", minute=5),
},
'sync_source_destination': {
'task': 'lemur.common.celery.sync_source_destination',
'options': {
'expires': 180
},
'schedule': crontab(hour="*"),
}
}
To enable celery support, you must also have configuration values that tell Celery which broker and backend to use.
Here are the Celery configuration variables that should be set::
CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379'
CELERY_BROKER_URL = 'redis://your_redis_url:6379'
CELERY_IMPORTS = ('lemur.common.celery')
CELERY_TIMEZONE = 'UTC'
You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
The scheduler can be started with::
LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery beat
And the worker can be started with desired options such as the following::
LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery worker --concurrency 10 -E -n lemurworker1@%%h
supervisor or systemd configurations should be created for these in production environments as appropriate.
Add support for LetsEncrypt
===========================
LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
LetsEncrypt requires that we prove ownership of a domain before we're able to issue a certificate for that domain, each
time we want a certificate.
The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
through the creation of DNS TXT records.
In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
prove ownership of all of these domains through this method. The token is typically written to a TXT record at
-acme_challenge.domain.com. Once we create the appropriate TXT record(s), Lemur will try to validate propagation
before requesting that LetsEncrypt finalize the certificate request and send us the certificate.
.. figure:: letsencrypt_flow.png
To start issuing certificates through LetsEncrypt, you must enable Celery support within Lemur first. After doing so,
you need to create a LetsEncrypt authority. To do this, visit
Authorities -> Create. Set the applicable attributes and click "More Options".
.. figure:: letsencrypt_authority_1.png
You will need to set "Certificate" to LetsEncrypt's active chain of trust for the authority you want to use. To find
the active chain of trust at the time of writing, please visit `LetsEncrypt
<https://letsencrypt.org/certificates/>`_.
Under Acme_url, enter in the appropriate endpoint URL. Lemur supports LetsEncrypt's V2 API, and we recommend you to use
this. At the time of writing, the staging and production URLs for LetsEncrypt V2 are
https://acme-staging-v02.api.letsencrypt.org/directory and https://acme-v02.api.letsencrypt.org/directory.
.. figure:: letsencrypt_authority_2.png
After creating the authorities, we will need to create a DNS provider. Visit `Admin` -> `DNS Providers` and click
`Create`. Lemur comes with a few provider plugins built in, with different options. Create a DNS provider with the
appropriate choices.
.. figure:: create_dns_provider.png
By default, users will need to select the DNS provider that is authoritative over their domain in order for the
LetsEncrypt flow to function. However, Lemur will attempt to automatically determine the appropriate provider if
possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.

View File

@ -5,7 +5,8 @@
:license: Apache, see LICENSE for more details. :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com> .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
.. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
""" """
import time import time