Merge branch 'master' into acme-fixed-attempts

docs/quickstart/index.rst

@@ -1,7 +1,8 @@
 Quickstart
 **********
 
-This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.  This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used.
+This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.
+This guide assumes a clean Ubuntu 18.04/20.04 instance, commands may differ based on the OS and configuration being used.
 
 For a quicker alternative, see the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
 
@@ -11,11 +12,13 @@ Dependencies
 
 Some basic prerequisites which you'll need in order to run Lemur:
 
-* A UNIX-based operating system (we test on Ubuntu, develop on OS X)
+* A UNIX-based operating system (we test on Ubuntu, develop on macOS)
 * Python 3.7 or greater
 * PostgreSQL 9.4 or greater
 * Nginx
+* Node v10.x (LTS)
 
+.. note:: Ubuntu 18.04 supports by default Python 3.6.x and Node v8.x
 .. note:: Lemur was built with AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
 
 
@@ -27,7 +30,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack
 .. code-block:: bash
 
     sudo apt-get update
-    sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql
+    sudo apt-get install nodejs npm python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor postgresql
 
 .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
 

lemur/api_keys/service.py

@@ -7,6 +7,7 @@
 """
 from lemur import database
 from lemur.api_keys.models import ApiKey
+from lemur.logs import service as log_service
 
 
 def get(aid):
@@ -24,6 +25,7 @@ def delete(access_key):
     :param access_key:
     :return:
     """
+    log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key")
     database.delete(access_key)
 
 
@@ -34,8 +36,9 @@ def revoke(aid):
     :return:
     """
     api_key = get(aid)
-    setattr(api_key, "revoked", False)
+    setattr(api_key, "revoked", True)
 
+    log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key")
     return database.update(api_key)
 
 
@@ -55,6 +58,9 @@ def create(**kwargs):
     :return:
     """
     api_key = ApiKey(**kwargs)
+    # this logs only metadata about the api key
+    log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}")
+
     database.create(api_key)
     return api_key
 
@@ -69,6 +75,7 @@ def update(api_key, **kwargs):
     for key, value in kwargs.items():
         setattr(api_key, key, value)
 
+    log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}")
     return database.update(api_key)
 
 

lemur/auth/views.py

@@ -20,6 +20,7 @@ from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
+from lemur.logs import service as log_service
 from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
 from lemur.auth import ldap
 
@@ -198,7 +199,6 @@ def update_user(user, profile, roles):
     :param profile:
     :param roles:
     """
-
     # if we get an sso user create them an account
     if not user:
         user = user_service.create(
@@ -212,10 +212,16 @@ def update_user(user, profile, roles):
 
     else:
         # we add 'lemur' specific roles, so they do not get marked as removed
+        removed_roles = []
         for ur in user.roles:
             if not ur.third_party:
                 roles.append(ur)
+            elif ur not in roles:
+                # This is a role assigned in lemur, but not returned by sso during current login
+                removed_roles.append(ur.name)
 
+        if removed_roles:
+            log_service.audit_log("unassign_role", user.name, f"Un-assigning roles {removed_roles}")
         # update any changes to the user
         user_service.update(
             user.id,

lemur/certificates/cli.py

@@ -119,13 +119,20 @@ def request_rotation(endpoint, certificate, message, commit):
             status = SUCCESS_METRIC_STATUS
 
         except Exception as e:
+            sentry.captureException(extra={"certificate_name": str(certificate.name),
+                                           "endpoint": str(endpoint.dnsname)})
+            current_app.logger.exception(
+                f"Error rotating certificate: {certificate.name}", exc_info=True
+            )
             print(
                 "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
                     endpoint.name, certificate.name, e
                 )
             )
 
-    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status})
+    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status,
+                                                                 "certificate_name": str(certificate.name),
+                                                                 "endpoint": str(endpoint.dnsname)})
 
 
 def request_reissue(certificate, commit):
@@ -224,7 +231,7 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
             print(
                 f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
             )
-            log_data["message"] = "Rotating endpoint"
+            log_data["message"] = "Rotating one endpoint"
             log_data["endpoint"] = endpoint.dnsname
             log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
@@ -232,8 +239,6 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
         elif old_cert and new_cert:
             print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
-
-            log_data["message"] = "Rotating all endpoints"
             log_data["certificate"] = new_cert.name
             log_data["certificate_old"] = old_cert.name
             log_data["message"] = "Rotating endpoint from old to new cert"
@@ -244,41 +249,23 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
                 current_app.logger.info(log_data)
 
         else:
+            # No certificate name or endpoint is provided. We will now fetch all endpoints,
+            # which are associated with a certificate that has been replaced
             print("[+] Rotating all endpoints that have new certificates available")
-            log_data["message"] = "Rotating all endpoints that have new certificates available"
             for endpoint in endpoint_service.get_all_pending_rotation():
-                log_data["endpoint"] = endpoint.dnsname
-                if len(endpoint.certificate.replaced) == 1:
-                    print(
-                        f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
-                    )
-                    log_data["certificate"] = endpoint.certificate.replaced[0].name
-                    request_rotation(
-                        endpoint, endpoint.certificate.replaced[0], message, commit
-                    )
-                    current_app.logger.info(log_data)
 
-                else:
+                log_data["message"] = "Rotating endpoint from old to new cert"
-                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
+                if len(endpoint.certificate.replaced) > 1:
-                    print(log_data)
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
-                    metrics.send(
+                                          f"{len(endpoint.certificate.replaced)}"
-                        "endpoint_rotation",
+
-                        "counter",
+                log_data["endpoint"] = endpoint.dnsname
-                        1,
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
-                        metric_tags={
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                            "status": FAILURE_METRIC_STATUS,
+                print(
-                            "old_certificate_name": str(old_cert),
+                    f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
-                            "new_certificate_name": str(
+                )
-                                endpoint.certificate.replaced[0].name
+                current_app.logger.info(log_data)
-                            ),
-                            "endpoint_name": str(endpoint.name),
-                            "message": str(message),
-                        },
-                    )
-                    print(
-                        f"[!] Failed to rotate endpoint {endpoint.name} reason: "
-                        "Multiple replacement certificates found."
-                    )
 
         status = SUCCESS_METRIC_STATUS
         print("[+] Done!")
@@ -369,6 +356,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
     :param message: Send a rotation notification to the certificates owner.
     :param commit: Persist changes.
     :param region: Region in which to rotate the endpoint.
+    #todo: merge this method with rotate()
     """
     if commit:
         print("[!] Running in COMMIT mode.")
@@ -418,24 +406,20 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                         1,
                         metric_tags={
                             "region": region,
-                            "old_certificate_name": str(old_cert),
                             "new_certificate_name": str(endpoint.certificate.replaced[0].name),
                             "endpoint_name": str(endpoint.dnsname),
                         },
                     )
+                    continue
 
-                if len(endpoint.certificate.replaced) == 1:
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
-                    log_data["certificate"] = endpoint.certificate.replaced[0].name
+                log_data["message"] = "Rotating all endpoints in region"
-                    log_data["message"] = "Rotating all endpoints in region"
+                if len(endpoint.certificate.replaced) > 1:
-                    print(log_data)
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
-                    current_app.logger.info(log_data)
+                                          f"{len(endpoint.certificate.replaced)}"
-                    request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
+
-                    status = SUCCESS_METRIC_STATUS
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                else:
+                current_app.logger.info(log_data)
-                    status = FAILURE_METRIC_STATUS
-                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
-                    print(log_data)
-                    current_app.logger.info(log_data)
 
                 metrics.send(
                     "endpoint_rotation_region",
@@ -443,7 +427,6 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                     1,
                     metric_tags={
                         "status": FAILURE_METRIC_STATUS,
-                        "old_certificate_name": str(old_cert),
                         "new_certificate_name": str(endpoint.certificate.replaced[0].name),
                         "endpoint_name": str(endpoint.dnsname),
                         "message": str(message),

lemur/logs/service.py

@@ -7,7 +7,7 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from flask import current_app
+from flask import current_app, g
 
 from lemur import database
 from lemur.logs.models import Log
@@ -34,6 +34,20 @@ def create(user, type, certificate=None):
     database.commit()
 
 
+def audit_log(action, entity, message):
+    """
+    Logs given action
+    :param action: The action being logged e.g. assign_role, create_role etc
+    :param entity: The entity undergoing the action e.g. name of the role
+    :param message: Additional info e.g. Role being assigned to user X
+    :return:
+    """
+    user = g.current_user.email if hasattr(g, 'current_user') else "LEMUR"
+    current_app.logger.info(
+        f"[lemur-audit] action: {action}, user: {user}, entity: {entity}, details: {message}"
+    )
+
+
 def get_all():
     """
     Retrieve all logs from the database.

lemur/roles/service.py

@@ -12,6 +12,7 @@
 from lemur import database
 from lemur.roles.models import Role
 from lemur.users.models import User
+from lemur.logs import service as log_service
 
 
 def update(role_id, name, description, users):
@@ -29,6 +30,8 @@ def update(role_id, name, description, users):
     role.description = description
     role.users = users
     database.update(role)
+
+    log_service.audit_log("update_role", name, f"Role with id {role_id} updated")
     return role
 
 
@@ -44,6 +47,8 @@ def set_third_party(role_id, third_party_status=False):
     role = get(role_id)
     role.third_party = third_party_status
     database.update(role)
+
+    log_service.audit_log("update_role", role.name, f"Updated third_party_status={third_party_status}")
     return role
 
 
@@ -71,6 +76,7 @@ def create(
     if users:
         role.users = users
 
+    log_service.audit_log("create_role", name, "Creating new role")
     return database.create(role)
 
 
@@ -101,7 +107,10 @@ def delete(role_id):
     :param role_id:
     :return:
     """
-    return database.delete(get(role_id))
+
+    role = get(role_id)
+    log_service.audit_log("delete_role", role.name, "Deleting role")
+    return database.delete(role)
 
 
 def render(args):

lemur/users/service.py

@@ -8,6 +8,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from lemur import database
+from lemur.logs import service as log_service
 from lemur.users.models import User
 
 
@@ -31,6 +32,7 @@ def create(username, password, email, active, profile_picture, roles):
         profile_picture=profile_picture,
     )
     user.roles = roles
+    log_service.audit_log("create_user", username, "Creating new user")
     return database.create(user)
 
 
@@ -52,6 +54,8 @@ def update(user_id, username, email, active, profile_picture, roles):
     user.active = active
     user.profile_picture = profile_picture
     update_roles(user, roles)
+
+    log_service.audit_log("update_user", username, f"Updating user with id {user_id}")
     return database.update(user)
 
 
@@ -64,19 +68,29 @@ def update_roles(user, roles):
     :param user:
     :param roles:
     """
+    removed_roles = []
     for ur in user.roles:
         for r in roles:
             if r.id == ur.id:
                 break
         else:
             user.roles.remove(ur)
+            removed_roles.append(ur.name)
 
+    if removed_roles:
+        log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
+
+    added_roles = []
     for r in roles:
         for ur in user.roles:
             if r.id == ur.id:
                 break
         else:
             user.roles.append(r)
+            added_roles.append(r.name)
+
+    if added_roles:
+        log_service.audit_log("assign_role", user.username, f"Assigning roles {added_roles}")
 
 
 def get(user_id):