Merge branch 'master' into feature/travis-python38

lemur/api_keys/service.py

@@ -7,6 +7,7 @@
 """
 from lemur import database
 from lemur.api_keys.models import ApiKey
+from lemur.logs import service as log_service
 
 
 def get(aid):
@@ -24,6 +25,7 @@ def delete(access_key):
     :param access_key:
     :return:
     """
+    log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key")
     database.delete(access_key)
 
 
@@ -34,8 +36,9 @@ def revoke(aid):
     :return:
     """
     api_key = get(aid)
-    setattr(api_key, "revoked", False)
+    setattr(api_key, "revoked", True)
 
+    log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key")
     return database.update(api_key)
 
 
@@ -55,6 +58,9 @@ def create(**kwargs):
     :return:
     """
     api_key = ApiKey(**kwargs)
+    # this logs only metadata about the api key
+    log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}")
+
     database.create(api_key)
     return api_key
 
@@ -69,6 +75,7 @@ def update(api_key, **kwargs):
     for key, value in kwargs.items():
         setattr(api_key, key, value)
 
+    log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}")
     return database.update(api_key)
 
 

lemur/auth/views.py

@@ -20,6 +20,7 @@ from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
+from lemur.logs import service as log_service
 from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
 from lemur.auth import ldap
 
@@ -198,7 +199,6 @@ def update_user(user, profile, roles):
     :param profile:
     :param roles:
     """
-
     # if we get an sso user create them an account
     if not user:
         user = user_service.create(
@@ -212,10 +212,16 @@ def update_user(user, profile, roles):
 
     else:
         # we add 'lemur' specific roles, so they do not get marked as removed
+        removed_roles = []
         for ur in user.roles:
             if not ur.third_party:
                 roles.append(ur)
+            elif ur not in roles:
+                # This is a role assigned in lemur, but not returned by sso during current login
+                removed_roles.append(ur.name)
 
+        if removed_roles:
+            log_service.audit_log("unassign_role", user.name, f"Un-assigning roles {removed_roles}")
         # update any changes to the user
         user_service.update(
             user.id,

lemur/certificates/cli.py

@@ -119,13 +119,20 @@ def request_rotation(endpoint, certificate, message, commit):
             status = SUCCESS_METRIC_STATUS
 
         except Exception as e:
+            sentry.captureException(extra={"certificate_name": str(certificate.name),
+                                           "endpoint": str(endpoint.dnsname)})
+            current_app.logger.exception(
+                f"Error rotating certificate: {certificate.name}", exc_info=True
+            )
             print(
                 "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
                     endpoint.name, certificate.name, e
                 )
             )
 
-    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status})
+    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status,
+                                                                 "certificate_name": str(certificate.name),
+                                                                 "endpoint": str(endpoint.dnsname)})
 
 
 def request_reissue(certificate, commit):
@@ -224,7 +231,7 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
             print(
                 f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
             )
-            log_data["message"] = "Rotating endpoint"
+            log_data["message"] = "Rotating one endpoint"
             log_data["endpoint"] = endpoint.dnsname
             log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
@@ -232,8 +239,6 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
         elif old_cert and new_cert:
             print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
-
-            log_data["message"] = "Rotating all endpoints"
             log_data["certificate"] = new_cert.name
             log_data["certificate_old"] = old_cert.name
             log_data["message"] = "Rotating endpoint from old to new cert"
@@ -244,41 +249,23 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
                 current_app.logger.info(log_data)
 
         else:
+            # No certificate name or endpoint is provided. We will now fetch all endpoints,
+            # which are associated with a certificate that has been replaced
             print("[+] Rotating all endpoints that have new certificates available")
-            log_data["message"] = "Rotating all endpoints that have new certificates available"
             for endpoint in endpoint_service.get_all_pending_rotation():
-                log_data["endpoint"] = endpoint.dnsname
-                if len(endpoint.certificate.replaced) == 1:
-                    print(
-                        f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
-                    )
-                    log_data["certificate"] = endpoint.certificate.replaced[0].name
-                    request_rotation(
-                        endpoint, endpoint.certificate.replaced[0], message, commit
-                    )
-                    current_app.logger.info(log_data)
 
-                else:
+                log_data["message"] = "Rotating endpoint from old to new cert"
-                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
+                if len(endpoint.certificate.replaced) > 1:
-                    print(log_data)
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
-                    metrics.send(
+                                          f"{len(endpoint.certificate.replaced)}"
-                        "endpoint_rotation",
+
-                        "counter",
+                log_data["endpoint"] = endpoint.dnsname
-                        1,
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
-                        metric_tags={
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                            "status": FAILURE_METRIC_STATUS,
+                print(
-                            "old_certificate_name": str(old_cert),
+                    f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
-                            "new_certificate_name": str(
+                )
-                                endpoint.certificate.replaced[0].name
+                current_app.logger.info(log_data)
-                            ),
-                            "endpoint_name": str(endpoint.name),
-                            "message": str(message),
-                        },
-                    )
-                    print(
-                        f"[!] Failed to rotate endpoint {endpoint.name} reason: "
-                        "Multiple replacement certificates found."
-                    )
 
         status = SUCCESS_METRIC_STATUS
         print("[+] Done!")
@@ -369,6 +356,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
     :param message: Send a rotation notification to the certificates owner.
     :param commit: Persist changes.
     :param region: Region in which to rotate the endpoint.
+    #todo: merge this method with rotate()
     """
     if commit:
         print("[!] Running in COMMIT mode.")
@@ -418,24 +406,20 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                         1,
                         metric_tags={
                             "region": region,
-                            "old_certificate_name": str(old_cert),
                             "new_certificate_name": str(endpoint.certificate.replaced[0].name),
                             "endpoint_name": str(endpoint.dnsname),
                         },
                     )
+                    continue
 
-                if len(endpoint.certificate.replaced) == 1:
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
-                    log_data["certificate"] = endpoint.certificate.replaced[0].name
+                log_data["message"] = "Rotating all endpoints in region"
-                    log_data["message"] = "Rotating all endpoints in region"
+                if len(endpoint.certificate.replaced) > 1:
-                    print(log_data)
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
-                    current_app.logger.info(log_data)
+                                          f"{len(endpoint.certificate.replaced)}"
-                    request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
+
-                    status = SUCCESS_METRIC_STATUS
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                else:
+                current_app.logger.info(log_data)
-                    status = FAILURE_METRIC_STATUS
-                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
-                    print(log_data)
-                    current_app.logger.info(log_data)
 
                 metrics.send(
                     "endpoint_rotation_region",
@@ -443,7 +427,6 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                     1,
                     metric_tags={
                         "status": FAILURE_METRIC_STATUS,
-                        "old_certificate_name": str(old_cert),
                         "new_certificate_name": str(endpoint.certificate.replaced[0].name),
                         "endpoint_name": str(endpoint.dnsname),
                         "message": str(message),

lemur/logs/service.py

@@ -7,7 +7,7 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from flask import current_app
+from flask import current_app, g
 
 from lemur import database
 from lemur.logs.models import Log
@@ -34,6 +34,20 @@ def create(user, type, certificate=None):
     database.commit()
 
 
+def audit_log(action, entity, message):
+    """
+    Logs given action
+    :param action: The action being logged e.g. assign_role, create_role etc
+    :param entity: The entity undergoing the action e.g. name of the role
+    :param message: Additional info e.g. Role being assigned to user X
+    :return:
+    """
+    user = g.current_user.email if hasattr(g, 'current_user') else "LEMUR"
+    current_app.logger.info(
+        f"[lemur-audit] action: {action}, user: {user}, entity: {entity}, details: {message}"
+    )
+
+
 def get_all():
     """
     Retrieve all logs from the database.

lemur/roles/service.py

@@ -12,6 +12,7 @@
 from lemur import database
 from lemur.roles.models import Role
 from lemur.users.models import User
+from lemur.logs import service as log_service
 
 
 def update(role_id, name, description, users):
@@ -29,6 +30,8 @@ def update(role_id, name, description, users):
     role.description = description
     role.users = users
     database.update(role)
+
+    log_service.audit_log("update_role", name, f"Role with id {role_id} updated")
     return role
 
 
@@ -44,6 +47,8 @@ def set_third_party(role_id, third_party_status=False):
     role = get(role_id)
     role.third_party = third_party_status
     database.update(role)
+
+    log_service.audit_log("update_role", role.name, f"Updated third_party_status={third_party_status}")
     return role
 
 
@@ -71,6 +76,7 @@ def create(
     if users:
         role.users = users
 
+    log_service.audit_log("create_role", name, "Creating new role")
     return database.create(role)
 
 
@@ -101,7 +107,10 @@ def delete(role_id):
     :param role_id:
     :return:
     """
-    return database.delete(get(role_id))
+
+    role = get(role_id)
+    log_service.audit_log("delete_role", role.name, "Deleting role")
+    return database.delete(role)
 
 
 def render(args):

lemur/users/service.py

@@ -8,6 +8,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from lemur import database
+from lemur.logs import service as log_service
 from lemur.users.models import User
 
 
@@ -31,6 +32,7 @@ def create(username, password, email, active, profile_picture, roles):
         profile_picture=profile_picture,
     )
     user.roles = roles
+    log_service.audit_log("create_user", username, "Creating new user")
     return database.create(user)
 
 
@@ -52,6 +54,8 @@ def update(user_id, username, email, active, profile_picture, roles):
     user.active = active
     user.profile_picture = profile_picture
     update_roles(user, roles)
+
+    log_service.audit_log("update_user", username, f"Updating user with id {user_id}")
     return database.update(user)
 
 
@@ -64,19 +68,29 @@ def update_roles(user, roles):
     :param user:
     :param roles:
     """
+    removed_roles = []
     for ur in user.roles:
         for r in roles:
             if r.id == ur.id:
                 break
         else:
             user.roles.remove(ur)
+            removed_roles.append(ur.name)
 
+    if removed_roles:
+        log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
+
+    added_roles = []
     for r in roles:
         for ur in user.roles:
             if r.id == ur.id:
                 break
         else:
             user.roles.append(r)
+            added_roles.append(r.name)
+
+    if added_roles:
+        log_service.audit_log("assign_role", user.username, f"Assigning roles {added_roles}")
 
 
 def get(user_id):