From 1b6907a40472370233febcef68cd25bf68cd39b9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 May 2020 16:28:01 -0700 Subject: [PATCH 1/5] Certificate rotation region by region example scheudule: CELERYBEAT_SCHEDULE = { 'certificate_rotate': { 'task': 'lemur.common.celery.certificate_rotate', 'options': { 'expires': 180 }, 'schedule': crontab(minute="*"), 'kwargs': {'region': 'us-east-1'} } } --- lemur/certificates/cli.py | 165 ++++++++++++++++++++++++++++++++++++++ lemur/common/celery.py | 10 ++- 2 files changed, 173 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index ca6b0248..15616064 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -285,6 +285,171 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c ) +@manager.option( + "-e", + "--endpoint", + dest="endpoint_name", + help="Name of the endpoint you wish to rotate.", +) +@manager.option( + "-n", + "--new-certificate", + dest="new_certificate_name", + help="Name of the certificate you wish to rotate to.", +) +@manager.option( + "-o", + "--old-certificate", + dest="old_certificate_name", + help="Name of the certificate you wish to rotate.", +) +@manager.option( + "-a", + "--notify", + dest="message", + action="store_true", + help="Send a rotation notification to the certificates owner.", +) +@manager.option( + "-c", + "--commit", + dest="commit", + action="store_true", + default=False, + help="Persist changes.", +) +@manager.option( + "-r", + "--region", + dest="region", + action="store_true", + required=True, + help="Region in which to rotate the endpoint.", +) +def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, message, commit, region): + """ + Rotates an endpoint in a defined region it if it has not already been replaced. If it has + been replaced, will use the replacement certificate for the rotation. + :param old_certificate_name: Name of the certificate you wish to rotate. + :param new_certificate_name: Name of the certificate you wish to rotate to. + :param endpoint_name: Name of the endpoint you wish to rotate. + :param message: Send a rotation notification to the certificates owner. + :param commit: Persist changes. + :param region: Region in which to rotate the endpoint. + """ + if commit: + print("[!] Running in COMMIT mode.") + + print("[+] Starting endpoint rotation.") + status = FAILURE_METRIC_STATUS + + try: + old_cert = validate_certificate(old_certificate_name) + new_cert = validate_certificate(new_certificate_name) + endpoint = validate_endpoint(endpoint_name) + + if endpoint and new_cert: + if region in endpoint.dnsname: + log_data = f"[+] Rotating endpoint in region: {endpoint.dnsname} to certificate {new_cert.name}" + print(log_data) + current_app.logger.info(log_data) + #request_rotation(endpoint, new_cert, message, commit) + else: + log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" + print(log_data) + current_app.logger.info(log_data) + + elif old_cert and new_cert: + log_data = f"[+] Rotating all endpoints in {region} from {old_cert.name} to {new_cert.name}" + print(log_data) + current_app.logger.info(log_data) + for endpoint in old_cert.endpoints: + if region in endpoint.dnsname: + log_data = f"[+] Rotating {endpoint.dnsname} in {region}" + print(log_data) + current_app.logger.info(log_data) + #request_rotation(endpoint, new_cert, message, commit) + else: + log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" + print(log_data) + current_app.logger.info(log_data) + else: + log_data = f"[+] Rotating all endpoints that have new certificates available in {region}" + print(log_data) + current_app.logger.info(log_data) + for endpoint in endpoint_service.get_all_pending_rotation(): + if region not in endpoint.dnsname: + log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" + print(log_data) + current_app.logger.info(log_data) + metrics.send( + "endpoint_rotation_region_skipped", + "counter", + 1, + metric_tags={ + "region": region, + "old_certificate_name": str(old_cert), + "new_certificate_name": str(endpoint.certificate.replaced[0].name), + "endpoint_name": str(endpoint.name), + }, + ) + + if len(endpoint.certificate.replaced) == 1: + log_data = f"[+] Rotating {endpoint.dnsname} in {region} to {endpoint.certificate.replaced[0].name}" + print(log_data) + current_app.logger.info(log_data) + #request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + status = SUCCESS_METRIC_STATUS + else: + status = FAILURE_METRIC_STATUS + log_data = f"[!] Failed to rotate endpoint {endpoint.dnsname} reason: " \ + f"Multiple replacement certificates found." + print(log_data) + current_app.logger.info(log_data) + + metrics.send( + "endpoint_rotation_region", + "counter", + 1, + metric_tags={ + "status": FAILURE_METRIC_STATUS, + "old_certificate_name": str(old_cert), + "new_certificate_name": str(endpoint.certificate.replaced[0].name), + "endpoint_name": str(endpoint.dnsname), + "message": str(message), + "region": str(region), + }, + ) + status = SUCCESS_METRIC_STATUS + print("[+] Done!") + + except Exception as e: + sentry.captureException( + extra={ + "old_certificate_name": str(old_certificate_name), + "new_certificate_name": str(new_certificate_name), + "endpoint": str(endpoint_name), + "message": str(message), + "region": str(region), + } + ) + + metrics.send( + "endpoint_rotation_region_job", + "counter", + 1, + metric_tags={ + "status": status, + "old_certificate_name": str(old_certificate_name), + "new_certificate_name": str(new_certificate_name), + "endpoint_name": str(endpoint_name), + "message": str(message), + "endpoint": str(globals().get("endpoint")), + "region": str(region), + }, + ) + + @manager.option( "-o", "--old-certificate", diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 5df470ab..a490b13b 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -631,7 +631,8 @@ def certificate_reissue(): @celery.task(soft_time_limit=3600) -def certificate_rotate(): +def certificate_rotate(**kwargs): + """ This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert :return: @@ -641,6 +642,7 @@ def certificate_rotate(): if celery.current_task: task_id = celery.current_task.request.id + region = kwargs.get("region") log_data = { "function": function, "message": "rotating certificates", @@ -654,7 +656,11 @@ def certificate_rotate(): current_app.logger.debug(log_data) try: - cli_certificate.rotate(None, None, None, None, True) + if region: + log_data["region"] = region + cli_certificate.rotate_region(None, None, None, None, True, region) + else: + cli_certificate.rotate(None, None, None, None, True) except SoftTimeLimitExceeded: log_data["message"] = "Certificate rotate: Time limit exceeded." current_app.logger.error(log_data) From 843ffad60e404ab03af4a49b2ffa41eb3650b9fb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 May 2020 17:10:50 -0700 Subject: [PATCH 2/5] removing testing comments --- lemur/certificates/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 15616064..c55fa8a2 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -353,7 +353,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes log_data = f"[+] Rotating endpoint in region: {endpoint.dnsname} to certificate {new_cert.name}" print(log_data) current_app.logger.info(log_data) - #request_rotation(endpoint, new_cert, message, commit) + request_rotation(endpoint, new_cert, message, commit) else: log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" print(log_data) @@ -368,7 +368,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes log_data = f"[+] Rotating {endpoint.dnsname} in {region}" print(log_data) current_app.logger.info(log_data) - #request_rotation(endpoint, new_cert, message, commit) + request_rotation(endpoint, new_cert, message, commit) else: log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" print(log_data) @@ -398,7 +398,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes log_data = f"[+] Rotating {endpoint.dnsname} in {region} to {endpoint.certificate.replaced[0].name}" print(log_data) current_app.logger.info(log_data) - #request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) status = SUCCESS_METRIC_STATUS else: status = FAILURE_METRIC_STATUS From 6d05af1790de4524111d6a2a4af1abb05425445b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:29:51 +0000 Subject: [PATCH 3/5] Bump pyjks from 19.0.0 to 20.0.0 Bumps [pyjks](https://github.com/kurtbrose/pyjks) from 19.0.0 to 20.0.0. - [Release notes](https://github.com/kurtbrose/pyjks/releases) - [Changelog](https://github.com/kurtbrose/pyjks/blob/master/CHANGELOG.md) - [Commits](https://github.com/kurtbrose/pyjks/compare/v19.0.0...v20.0.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index db255de3..14e54b59 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -70,7 +70,7 @@ pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-mod pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx -pyjks==19.0.0 # via -r requirements.txt +pyjks==20.0.0 # via -r requirements.txt pyjwt==1.7.1 # via -r requirements.txt pynacl==1.3.0 # via -r requirements.txt, paramiko pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests diff --git a/requirements.txt b/requirements.txt index f8d553d4..83585828 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,7 @@ pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via cffi pycryptodomex==3.9.7 # via pyjks -pyjks==19.0.0 # via -r requirements.in +pyjks==20.0.0 # via -r requirements.in pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests From 70985f4ff510d2601300b68e28bbf749f26a3dad Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 14 May 2020 22:37:30 -0700 Subject: [PATCH 4/5] revised system arch --- lemur/certificates/cli.py | 56 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 553ecf1b..2e3fbd0c 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -285,6 +285,17 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c ) +def request_rotation_region(region, endpoint, new_cert, message, commit, log_data): + if region in endpoint.dnsname: + log_data["message"] = "Rotating endpoint in region" + #request_rotation(endpoint, new_cert, message, commit) + else: + log_data["message"] = "Skipping rotation, region mismatch" + + print(log_data) + current_app.logger.info(log_data) + + @manager.option( "-e", "--endpoint", @@ -343,43 +354,38 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes print("[+] Starting endpoint rotation.") status = FAILURE_METRIC_STATUS + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "region": region, + } + try: old_cert = validate_certificate(old_certificate_name) new_cert = validate_certificate(new_certificate_name) endpoint = validate_endpoint(endpoint_name) if endpoint and new_cert: - if region in endpoint.dnsname: - log_data = f"[+] Rotating endpoint in region: {endpoint.dnsname} to certificate {new_cert.name}" - print(log_data) - current_app.logger.info(log_data) - request_rotation(endpoint, new_cert, message, commit) - else: - log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" - print(log_data) - current_app.logger.info(log_data) + log_data["endpoint"] = endpoint.dnsname + log_data["certificate"] = new_cert.name + request_rotation_region(region, endpoint, new_cert, message, commit, log_data) elif old_cert and new_cert: - log_data = f"[+] Rotating all endpoints in {region} from {old_cert.name} to {new_cert.name}" + log_data["certificate"] = new_cert.name + log_data["certificate_old"] = old_cert.name + log_data["message"] = "Rotating endpoint from old to new cert" print(log_data) current_app.logger.info(log_data) for endpoint in old_cert.endpoints: - if region in endpoint.dnsname: - log_data = f"[+] Rotating {endpoint.dnsname} in {region}" - print(log_data) - current_app.logger.info(log_data) - request_rotation(endpoint, new_cert, message, commit) - else: - log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" - print(log_data) - current_app.logger.info(log_data) + log_data["endpoint"] = endpoint.dnsname + request_rotation_region(region, endpoint, new_cert, message, commit, log_data) else: - log_data = f"[+] Rotating all endpoints that have new certificates available in {region}" + log_data["message"] = "Rotating all endpoints that have new certificates available" print(log_data) current_app.logger.info(log_data) for endpoint in endpoint_service.get_all_pending_rotation(): + log_data["endpoint"] = endpoint.dnsname if region not in endpoint.dnsname: - log_data = f"[+] Skipping rotation of {endpoint.dnsname} since not in {region}" + log_data["message"] = "Skipping rotation, region mismatch" print(log_data) current_app.logger.info(log_data) metrics.send( @@ -395,15 +401,15 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes ) if len(endpoint.certificate.replaced) == 1: - log_data = f"[+] Rotating {endpoint.dnsname} in {region} to {endpoint.certificate.replaced[0].name}" + log_data["certificate"] = endpoint.certificate.replaced[0].name + log_data["message"] = "Rotating all endpoints in region" print(log_data) current_app.logger.info(log_data) - request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + #request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) status = SUCCESS_METRIC_STATUS else: status = FAILURE_METRIC_STATUS - log_data = f"[!] Failed to rotate endpoint {endpoint.dnsname} reason: " \ - f"Multiple replacement certificates found." + log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found" print(log_data) current_app.logger.info(log_data) From 09016fd2eea7ccf357d4ef6e39f334a3ff079073 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 22 May 2020 16:04:39 -0700 Subject: [PATCH 5/5] cleaning up the code after more local testing --- lemur/certificates/cli.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 5a58d1ae..6367e1cd 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -286,10 +286,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c ) -def request_rotation_region(region, endpoint, new_cert, message, commit, log_data): +def request_rotation_region(endpoint, new_cert, message, commit, log_data, region): if region in endpoint.dnsname: log_data["message"] = "Rotating endpoint in region" - #request_rotation(endpoint, new_cert, message, commit) + request_rotation(endpoint, new_cert, message, commit) else: log_data["message"] = "Skipping rotation, region mismatch" @@ -334,7 +334,6 @@ def request_rotation_region(region, endpoint, new_cert, message, commit, log_dat "-r", "--region", dest="region", - action="store_true", required=True, help="Region in which to rotate the endpoint.", ) @@ -368,7 +367,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes if endpoint and new_cert: log_data["endpoint"] = endpoint.dnsname log_data["certificate"] = new_cert.name - request_rotation_region(region, endpoint, new_cert, message, commit, log_data) + request_rotation_region(endpoint, new_cert, message, commit, log_data, region) elif old_cert and new_cert: log_data["certificate"] = new_cert.name @@ -378,12 +377,14 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes current_app.logger.info(log_data) for endpoint in old_cert.endpoints: log_data["endpoint"] = endpoint.dnsname - request_rotation_region(region, endpoint, new_cert, message, commit, log_data) + request_rotation_region(endpoint, new_cert, message, commit, log_data, region) + else: log_data["message"] = "Rotating all endpoints that have new certificates available" print(log_data) current_app.logger.info(log_data) - for endpoint in endpoint_service.get_all_pending_rotation(): + all_pending_rotation_endpoints = endpoint_service.get_all_pending_rotation() + for endpoint in all_pending_rotation_endpoints: log_data["endpoint"] = endpoint.dnsname if region not in endpoint.dnsname: log_data["message"] = "Skipping rotation, region mismatch" @@ -397,7 +398,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes "region": region, "old_certificate_name": str(old_cert), "new_certificate_name": str(endpoint.certificate.replaced[0].name), - "endpoint_name": str(endpoint.name), + "endpoint_name": str(endpoint.dnsname), }, ) @@ -406,7 +407,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes log_data["message"] = "Rotating all endpoints in region" print(log_data) current_app.logger.info(log_data) - #request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) status = SUCCESS_METRIC_STATUS else: status = FAILURE_METRIC_STATUS