Merge pull request #2822 from castrapel/master

Initial LetsEncrypt / Celery docs

Makefile

@@ -36,7 +36,7 @@ endif
 	@echo ""
 
 dev-docs:
-	pip install -r docs/requirements.txt
+	pip install -r requirements-docs.txt
 
 reset-db:
 	@echo "--> Dropping existing 'lemur' database"

docs/administration.rst

@@ -320,7 +320,7 @@ LDAP support requires the pyldap python library, which also depends on the follo
 To configure the use of an LDAP server, a number of settings need to be configured in `lemur.conf.py`.
 
 Here is an example LDAP configuration stanza you can add to your config. Adjust to suit your environment of course.
- 
+
 .. code-block:: python
 
         LDAP_AUTH = True
@@ -718,7 +718,7 @@ The following configuration properties are required to use the CFSSL issuer plug
 Hashicorp Vault Source/Destination Plugin
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination. 
+Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination.
 
 .. note:: This plugin does not supersede or overlap the 3rd party Vault Issuer plugin.
 
@@ -1090,7 +1090,9 @@ Verisign/Symantec
 -----------------
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Issuer
 :Description:
@@ -1116,6 +1118,8 @@ Acme
 
 :Authors:
     Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>,
     Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 :Type:
     Issuer
@@ -1127,7 +1131,9 @@ Atlas
 -----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Metric
 :Description:
@@ -1138,7 +1144,9 @@ Email
 -----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Notification
 :Description:
@@ -1160,7 +1168,9 @@ AWS
 ----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Source
 :Description:
@@ -1171,7 +1181,9 @@ AWS
 ----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Destination
 :Description:

docs/conf.py

@@ -18,48 +18,45 @@ from unittest.mock import MagicMock
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
 
 # Mock packages that cannot be installed on rtd
-on_rtd = os.environ.get('READTHEDOCS') == 'True'
+on_rtd = os.environ.get("READTHEDOCS") == "True"
 if on_rtd:
+
     class Mock(MagicMock):
         @classmethod
         def __getattr__(cls, name):
             return MagicMock()
 
-    MOCK_MODULES = ['ldap']
+    MOCK_MODULES = ["ldap"]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinxcontrib.autohttp.flask',
-    'sphinx.ext.todo',
-]
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask", "sphinx.ext.todo"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'lemur'
-copyright = u'2018, Netflix Inc.'
+project = u"lemur"
+copyright = u"2018, Netflix Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -74,185 +71,180 @@ version = release = about["__version__"]
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 
 # -- Options for HTML output ----------------------------------------------
 
 # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 
 if not on_rtd:  # only import and set the theme if we're building docs locally
     import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
+
+    html_theme = "sphinx_rtd_theme"
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'lemurdoc'
+htmlhelp_basename = "lemurdoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'lemur.tex', u'Lemur Documentation',
-   u'Kevin Glisson', 'manual'),
+    ("index", "lemur.tex", u"Lemur Documentation", u"Netflix Security", "manual")
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'Lemur', u'Lemur Documentation',
-     [u'Kevin Glisson'], 1)
-]
+man_pages = [("index", "Lemur", u"Lemur Documentation", [u"Netflix Security"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -261,19 +253,25 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Lemur', u'Lemur Documentation',
-   u'Kevin Glisson', 'Lemur', 'SSL Certificate Management',
-   'Miscellaneous'),
+    (
+        "index",
+        "Lemur",
+        u"Lemur Documentation",
+        u"Netflix Security",
+        "Lemur",
+        "SSL Certificate Management",
+        "Miscellaneous",
+    )
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False

docs/developer/index.rst

@@ -22,12 +22,18 @@ Once you've got all that, the rest is simple:
     # If you have a fork, you'll want to clone it instead
     git clone git://github.com/netflix/lemur.git
 
-    # Create a python virtualenv
-    mkvirtualenv lemur
+    # Create and activate python virtualenv from within the lemur repo
+    python3 -m venv env
+    . env/bin/activate
+
+    # Install doc requirements
 
-    # Make the magic happen
     make dev-docs
 
+    # Make the docs
+    cd docs
+    make html
+
 Running ``make dev-docs`` will install the basic requirements to get Sphinx running.
 
 
@@ -58,7 +64,7 @@ Once you've got all that, the rest is simple:
     git clone git://github.com/lemur/lemur.git
 
     # Create a python virtualenv
-    mkvirtualenv lemur
+    python3 -m venv env
 
     # Make the magic happen
     make
@@ -135,7 +141,7 @@ The test suite consists of multiple parts, testing both the Python and JavaScrip
 
     make test
 
-If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``test-js`` for the JavaScript tests.
+If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``make test-js`` for the JavaScript tests.
 
 
 You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests.

docs/production/index.rst

@@ -217,23 +217,23 @@ An example apache config::
         # HSTS (mod_headers is required) (15768000 seconds = 6 months)
         Header always set Strict-Transport-Security "max-age=15768000"
         ...
-        
+
      # Set the lemur DocumentRoot to static/dist
      DocumentRoot /www/lemur/lemur/static/dist
-     
+
      # Uncomment to force http 1.0 connections to proxy
      # SetEnv force-proxy-request-1.0 1
-     
+
      #Don't keep proxy connections alive
      SetEnv proxy-nokeepalive 1
-     
+
      # Only need to do reverse proxy
      ProxyRequests Off
-     
+
      # Proxy requests to the api to the lemur service (and sanitize redirects from it)
      ProxyPass "/api" "http://127.0.0.1:8000/api"
      ProxyPassReverse "/api" "http://127.0.0.1:8000/api"
-     
+
     </VirtualHost>
 
 Also included in the configurations above are several best practices when it comes to deploying TLS. Things like enabling
@@ -318,7 +318,7 @@ Periodic Tasks
 ==============
 
 Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
-a cron job that runs the commands.
+celery tasks or cron jobs that run these commands.
 
 There are currently three commands that could/should be run on a periodic basis:
 
@@ -326,11 +326,124 @@ There are currently three commands that could/should be run on a periodic basis:
 - `check_revoked`
 - `sync`
 
+If you are using LetsEncrypt, you must also run the following:
+
+- `fetch_all_pending_acme_certs`
+- `remove_old_acme_certs`
+
 How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
-`sync` is typically run every 15 minutes.
+`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
+`remove_old_acme_certs` can be ran more rarely, such as once every week.
 
 Example cron entries::
 
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
     */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
+
+
+Example Celery configuration (To be placed in your configuration file)::
+
+    CELERYBEAT_SCHEDULE = {
+        'fetch_all_pending_acme_certs': {
+            'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(minute="*"),
+        },
+        'remove_old_acme_certs': {
+            'task': 'lemur.common.celery.remove_old_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=7, minute=30, day_of_week=1),
+        },
+        'clean_all_sources': {
+            'task': 'lemur.common.celery.clean_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=1, minute=0, day_of_week=1),
+        },
+        'sync_all_sources': {
+            'task': 'lemur.common.celery.sync_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*/3", minute=5),
+        },
+        'sync_source_destination': {
+            'task': 'lemur.common.celery.sync_source_destination',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*"),
+        }
+    }
+
+To enable celery support, you must also have configuration values that tell Celery which broker and backend to use.
+Here are the Celery configuration variables that should be set::
+
+    CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379'
+    CELERY_BROKER_URL = 'redis://your_redis_url:6379'
+    CELERY_IMPORTS = ('lemur.common.celery')
+    CELERY_TIMEZONE = 'UTC'
+
+You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
+The scheduler can be started with::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery beat
+
+And the worker can be started with desired options such as the following::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery worker --concurrency 10 -E -n lemurworker1@%%h
+
+supervisor or systemd configurations should be created for these in production environments as appropriate.
+
+Add support for LetsEncrypt
+===========================
+
+LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
+for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
+LetsEncrypt requires that we prove ownership of a domain before we're able to issue a certificate for that domain, each
+time we want a certificate.
+
+The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
+through the creation of DNS TXT records.
+
+In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
+token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
+prove ownership of all of these domains through this method. The token is typically written to a TXT record at
+-acme_challenge.domain.com. Once we create the appropriate TXT record(s), Lemur will try to validate propagation
+before requesting that LetsEncrypt finalize the certificate request and send us the certificate.
+
+.. figure:: letsencrypt_flow.png
+
+To start issuing certificates through LetsEncrypt, you must enable Celery support within Lemur first. After doing so,
+you need to create a LetsEncrypt authority. To do this, visit
+Authorities -> Create. Set the applicable attributes and click "More Options".
+
+.. figure:: letsencrypt_authority_1.png
+
+You will need to set "Certificate" to LetsEncrypt's active chain of trust for the authority you want to use. To find
+the active chain of trust at the time of writing, please visit `LetsEncrypt
+<https://letsencrypt.org/certificates/>`_.
+
+Under Acme_url, enter in the appropriate endpoint URL. Lemur supports LetsEncrypt's V2 API, and we recommend you to use
+this. At the time of writing, the staging and production URLs for LetsEncrypt V2 are
+https://acme-staging-v02.api.letsencrypt.org/directory and https://acme-v02.api.letsencrypt.org/directory.
+
+.. figure:: letsencrypt_authority_2.png
+
+After creating the authorities, we will need to create a DNS provider. Visit `Admin` -> `DNS Providers` and click
+`Create`. Lemur comes with a few provider plugins built in, with different options. Create a DNS provider with the
+appropriate choices.
+
+.. figure:: create_dns_provider.png
+
+By default, users will need to select the DNS provider that is authoritative over their domain in order for the
+LetsEncrypt flow to function. However, Lemur will attempt to automatically determine the appropriate provider if
+possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
+This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
+Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.

lemur/__init__.py

@@ -5,7 +5,8 @@
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
+.. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
 
 """
 import time