diff --git a/lemur/auth/views.py b/lemur/auth/views.py
index f9eb11d0..85a8f636 100644
--- a/lemur/auth/views.py
+++ b/lemur/auth/views.py
@@ -212,12 +212,16 @@ def update_user(user, profile, roles):
 
     else:
         # we add 'lemur' specific roles, so they do not get marked as removed
+        removed_roles = []
         for ur in user.roles:
             if not ur.third_party:
                 roles.append(ur)
-            else:
-                log_service.audit_log("unassign_role", ur.name, f"Un-assigning the role for {user.name}")
+            elif ur not in roles:
+                # This is a role assigned in lemur, but not returned by sso during current login
+                removed_roles.append(ur.name)
 
+        if removed_roles:
+            log_service.audit_log("unassign_role", user.name, f"Un-assigning roles {removed_roles}")
         # update any changes to the user
         user_service.update(
             user.id,
diff --git a/lemur/users/service.py b/lemur/users/service.py
index ffc81f5c..d708d295 100644
--- a/lemur/users/service.py
+++ b/lemur/users/service.py
@@ -68,21 +68,29 @@ def update_roles(user, roles):
     :param user:
     :param roles:
     """
+    removed_roles = []
     for ur in user.roles:
         for r in roles:
             if r.id == ur.id:
                 break
         else:
             user.roles.remove(ur)
-            log_service.audit_log("unassign_role", ur.name, f"Un-assigning the role for user {user.username}")
+            removed_roles.append(ur.name)
 
+    if removed_roles:
+        log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
+
+    added_roles = []
     for r in roles:
         for ur in user.roles:
             if r.id == ur.id:
                 break
         else:
             user.roles.append(r)
-            log_service.audit_log("assign_role", r.name, f"Assigning the role to user {user.username}")
+            added_roles.append(r.name)
+
+    if added_roles:
+        log_service.audit_log("assign_role", user.username, f"Assigning roles {added_roles}")
 
 
 def get(user_id):