diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 0fa55a09..3056029d 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -1,7 +1,8 @@ Quickstart ********** -This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service. This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used. +This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service. +This guide assumes a clean Ubuntu 18.04/20.04 instance, commands may differ based on the OS and configuration being used. For a quicker alternative, see the Lemur docker file on `Github `_. @@ -11,11 +12,13 @@ Dependencies Some basic prerequisites which you'll need in order to run Lemur: -* A UNIX-based operating system (we test on Ubuntu, develop on OS X) +* A UNIX-based operating system (we test on Ubuntu, develop on macOS) * Python 3.7 or greater * PostgreSQL 9.4 or greater * Nginx +* Node v10.x (LTS) +.. note:: Ubuntu 18.04 supports by default Python 3.6.x and Node v8.x .. note:: Lemur was built with AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us. Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment. @@ -27,7 +30,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack .. code-block:: bash sudo apt-get update - sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql + sudo apt-get install nodejs npm python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor postgresql .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git). diff --git a/lemur/api_keys/service.py b/lemur/api_keys/service.py index ea681a62..3cb896aa 100644 --- a/lemur/api_keys/service.py +++ b/lemur/api_keys/service.py @@ -7,6 +7,7 @@ """ from lemur import database from lemur.api_keys.models import ApiKey +from lemur.logs import service as log_service def get(aid): @@ -24,6 +25,7 @@ def delete(access_key): :param access_key: :return: """ + log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key") database.delete(access_key) @@ -34,8 +36,9 @@ def revoke(aid): :return: """ api_key = get(aid) - setattr(api_key, "revoked", False) + setattr(api_key, "revoked", True) + log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key") return database.update(api_key) @@ -55,6 +58,9 @@ def create(**kwargs): :return: """ api_key = ApiKey(**kwargs) + # this logs only metadata about the api key + log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}") + database.create(api_key) return api_key @@ -69,6 +75,7 @@ def update(api_key, **kwargs): for key, value in kwargs.items(): setattr(api_key, key, value) + log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}") return database.update(api_key) diff --git a/lemur/auth/views.py b/lemur/auth/views.py index eaed419d..85a8f636 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -20,6 +20,7 @@ from lemur.common.utils import get_psuedo_random_string from lemur.users import service as user_service from lemur.roles import service as role_service +from lemur.logs import service as log_service from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key from lemur.auth import ldap @@ -198,7 +199,6 @@ def update_user(user, profile, roles): :param profile: :param roles: """ - # if we get an sso user create them an account if not user: user = user_service.create( @@ -212,10 +212,16 @@ def update_user(user, profile, roles): else: # we add 'lemur' specific roles, so they do not get marked as removed + removed_roles = [] for ur in user.roles: if not ur.third_party: roles.append(ur) + elif ur not in roles: + # This is a role assigned in lemur, but not returned by sso during current login + removed_roles.append(ur.name) + if removed_roles: + log_service.audit_log("unassign_role", user.name, f"Un-assigning roles {removed_roles}") # update any changes to the user user_service.update( user.id, diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 4b4b22e9..e00f6ea7 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -119,13 +119,20 @@ def request_rotation(endpoint, certificate, message, commit): status = SUCCESS_METRIC_STATUS except Exception as e: + sentry.captureException(extra={"certificate_name": str(certificate.name), + "endpoint": str(endpoint.dnsname)}) + current_app.logger.exception( + f"Error rotating certificate: {certificate.name}", exc_info=True + ) print( "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format( endpoint.name, certificate.name, e ) ) - metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status}) + metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status, + "certificate_name": str(certificate.name), + "endpoint": str(endpoint.dnsname)}) def request_reissue(certificate, commit): @@ -224,7 +231,7 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c print( f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}" ) - log_data["message"] = "Rotating endpoint" + log_data["message"] = "Rotating one endpoint" log_data["endpoint"] = endpoint.dnsname log_data["certificate"] = new_cert.name request_rotation(endpoint, new_cert, message, commit) @@ -232,8 +239,6 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c elif old_cert and new_cert: print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}") - - log_data["message"] = "Rotating all endpoints" log_data["certificate"] = new_cert.name log_data["certificate_old"] = old_cert.name log_data["message"] = "Rotating endpoint from old to new cert" @@ -244,41 +249,23 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c current_app.logger.info(log_data) else: + # No certificate name or endpoint is provided. We will now fetch all endpoints, + # which are associated with a certificate that has been replaced print("[+] Rotating all endpoints that have new certificates available") - log_data["message"] = "Rotating all endpoints that have new certificates available" for endpoint in endpoint_service.get_all_pending_rotation(): - log_data["endpoint"] = endpoint.dnsname - if len(endpoint.certificate.replaced) == 1: - print( - f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}" - ) - log_data["certificate"] = endpoint.certificate.replaced[0].name - request_rotation( - endpoint, endpoint.certificate.replaced[0], message, commit - ) - current_app.logger.info(log_data) - else: - log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found" - print(log_data) - metrics.send( - "endpoint_rotation", - "counter", - 1, - metric_tags={ - "status": FAILURE_METRIC_STATUS, - "old_certificate_name": str(old_cert), - "new_certificate_name": str( - endpoint.certificate.replaced[0].name - ), - "endpoint_name": str(endpoint.name), - "message": str(message), - }, - ) - print( - f"[!] Failed to rotate endpoint {endpoint.name} reason: " - "Multiple replacement certificates found." - ) + log_data["message"] = "Rotating endpoint from old to new cert" + if len(endpoint.certificate.replaced) > 1: + log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \ + f"{len(endpoint.certificate.replaced)}" + + log_data["endpoint"] = endpoint.dnsname + log_data["certificate"] = endpoint.certificate.replaced[0].name + request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + print( + f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}" + ) + current_app.logger.info(log_data) status = SUCCESS_METRIC_STATUS print("[+] Done!") @@ -369,6 +356,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes :param message: Send a rotation notification to the certificates owner. :param commit: Persist changes. :param region: Region in which to rotate the endpoint. + #todo: merge this method with rotate() """ if commit: print("[!] Running in COMMIT mode.") @@ -418,24 +406,20 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes 1, metric_tags={ "region": region, - "old_certificate_name": str(old_cert), "new_certificate_name": str(endpoint.certificate.replaced[0].name), "endpoint_name": str(endpoint.dnsname), }, ) + continue - if len(endpoint.certificate.replaced) == 1: - log_data["certificate"] = endpoint.certificate.replaced[0].name - log_data["message"] = "Rotating all endpoints in region" - print(log_data) - current_app.logger.info(log_data) - request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) - status = SUCCESS_METRIC_STATUS - else: - status = FAILURE_METRIC_STATUS - log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found" - print(log_data) - current_app.logger.info(log_data) + log_data["certificate"] = endpoint.certificate.replaced[0].name + log_data["message"] = "Rotating all endpoints in region" + if len(endpoint.certificate.replaced) > 1: + log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \ + f"{len(endpoint.certificate.replaced)}" + + request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + current_app.logger.info(log_data) metrics.send( "endpoint_rotation_region", @@ -443,7 +427,6 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes 1, metric_tags={ "status": FAILURE_METRIC_STATUS, - "old_certificate_name": str(old_cert), "new_certificate_name": str(endpoint.certificate.replaced[0].name), "endpoint_name": str(endpoint.dnsname), "message": str(message), diff --git a/lemur/logs/service.py b/lemur/logs/service.py index f4949911..5e5f30db 100644 --- a/lemur/logs/service.py +++ b/lemur/logs/service.py @@ -7,7 +7,7 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from flask import current_app +from flask import current_app, g from lemur import database from lemur.logs.models import Log @@ -34,6 +34,20 @@ def create(user, type, certificate=None): database.commit() +def audit_log(action, entity, message): + """ + Logs given action + :param action: The action being logged e.g. assign_role, create_role etc + :param entity: The entity undergoing the action e.g. name of the role + :param message: Additional info e.g. Role being assigned to user X + :return: + """ + user = g.current_user.email if hasattr(g, 'current_user') else "LEMUR" + current_app.logger.info( + f"[lemur-audit] action: {action}, user: {user}, entity: {entity}, details: {message}" + ) + + def get_all(): """ Retrieve all logs from the database. diff --git a/lemur/roles/service.py b/lemur/roles/service.py index fa4c9c97..cb733d40 100644 --- a/lemur/roles/service.py +++ b/lemur/roles/service.py @@ -12,6 +12,7 @@ from lemur import database from lemur.roles.models import Role from lemur.users.models import User +from lemur.logs import service as log_service def update(role_id, name, description, users): @@ -29,6 +30,8 @@ def update(role_id, name, description, users): role.description = description role.users = users database.update(role) + + log_service.audit_log("update_role", name, f"Role with id {role_id} updated") return role @@ -44,6 +47,8 @@ def set_third_party(role_id, third_party_status=False): role = get(role_id) role.third_party = third_party_status database.update(role) + + log_service.audit_log("update_role", role.name, f"Updated third_party_status={third_party_status}") return role @@ -71,6 +76,7 @@ def create( if users: role.users = users + log_service.audit_log("create_role", name, "Creating new role") return database.create(role) @@ -101,7 +107,10 @@ def delete(role_id): :param role_id: :return: """ - return database.delete(get(role_id)) + + role = get(role_id) + log_service.audit_log("delete_role", role.name, "Deleting role") + return database.delete(role) def render(args): diff --git a/lemur/users/service.py b/lemur/users/service.py index 8fb91aa3..d708d295 100644 --- a/lemur/users/service.py +++ b/lemur/users/service.py @@ -8,6 +8,7 @@ .. moduleauthor:: Kevin Glisson """ from lemur import database +from lemur.logs import service as log_service from lemur.users.models import User @@ -31,6 +32,7 @@ def create(username, password, email, active, profile_picture, roles): profile_picture=profile_picture, ) user.roles = roles + log_service.audit_log("create_user", username, "Creating new user") return database.create(user) @@ -52,6 +54,8 @@ def update(user_id, username, email, active, profile_picture, roles): user.active = active user.profile_picture = profile_picture update_roles(user, roles) + + log_service.audit_log("update_user", username, f"Updating user with id {user_id}") return database.update(user) @@ -64,19 +68,29 @@ def update_roles(user, roles): :param user: :param roles: """ + removed_roles = [] for ur in user.roles: for r in roles: if r.id == ur.id: break else: user.roles.remove(ur) + removed_roles.append(ur.name) + if removed_roles: + log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}") + + added_roles = [] for r in roles: for ur in user.roles: if r.id == ur.id: break else: user.roles.append(r) + added_roles.append(r.name) + + if added_roles: + log_service.audit_log("assign_role", user.username, f"Assigning roles {added_roles}") def get(user_id):