From e25f97fce7da04b316ba2f6fc7359630f236f99f Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:50:24 -0700 Subject: [PATCH 1/2] Bump time limit for clean_source Celery job For larger accounts, I've hit SoftTimeLimit exceptions before completion of this celery job. Bumping up the time limit on this job. --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 4af33d86..ebf85ed7 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -332,7 +332,7 @@ def clean_all_sources(): metrics.send(f"{function}.success", 'counter', 1) -@celery.task(soft_time_limit=600) +@celery.task(soft_time_limit=3600) def clean_source(source): """ This celery task will clean the specified source. This is a destructive operation that will delete unused From d825616ea6cef6fd02ee80eedd7a75f5fc4cea61 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:53:17 -0700 Subject: [PATCH 2/2] No need to retry 25 times on DeleteConflict errors --- lemur/plugins/lemur_aws/iam.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 13590ddd..8d80e020 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -24,6 +24,12 @@ def retry_throttled(exception): if exception.response["Error"]["Code"] == "NoSuchEntity": return False + # No need to retry deletion requests if there is a DeleteConflict error. + # This error indicates that the certificate is still attached to an entity + # and cannot be deleted. + if exception.response["Error"]["Code"] == "DeleteConflict": + return False + metrics.send("iam_retry", "counter", 1, metric_tags={"exception": str(exception)}) return True