diff --git a/.travis.yml b/.travis.yml index f38555a0..f9db8d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -dist: xenial +dist: bionic node_js: - "6.2.0" @@ -47,4 +47,7 @@ after_success: notifications: email: - ccastrapel@netflix.com + recipients: + - lemur@netflix.com + on_success: never + on_failure: always diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ea8d23b7..67b792f8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,87 @@ Changelog ========= +0.8.0 - `2020-11-13` +~~~~~~~~~~~~~~ + +This release comes after more than two years and contains many interesting new features and improvements. +In addition to multiple new plugins, such as ACME-http01, ADCS, PowerDNS, UltraDNS, Entrust, SNS, many of Lemur's existing +flows have improved. + +In the future, we plan to do frequent releases. + + +Summary of notable changes: + +- AWS S3 plugin: added delete, get methods, and support for uploading/deleting acme tokens +- ACME plugin: + - revamp of the plugin + - support for http01 domain validation, via S3 and SFTP as destination for the acme token + - support for CNAME delegated domain validation + - store-acme-account-details +- PowerDNS plugin +- UltraDNS plugin +- ADCS plugin +- SNS plugin +- Entrust plugin +- Rotation: + - respecting keyType and extensions + - region-by-region rotation option + - default to auto-rotate when cert attached to endpoint + - default to 1y validity during rotation for multi-year browser-trusted certs +- Certificate: search_by_name, and important performance improvements +- UI + - reducing the EC curve options to the relevant ones + - edit option for notifications, destinations and sources + - showing 13 month validity as default + - option to hide certs expired since 3month + - faster Permalink (no search involved) + - commonName Auto Added as DNS in the UI + - improved search and cert lookup +- celery tasks instead of crone, for better logging and monitoring +- countless bugfixes + - group-lookup-fix-referral + - url_context_path + - duplicate notification + - digicert-time-bug-fix + - improved-csr-support + - fix-cryptography-intermediate-ca + - enhanced logging + - vault-k8s-auth + - cfssl-key-fix + - cert-sync-endpoint-find-by-hash + - nlb-naming-bug + - fix_vault_api_v2_append + - aid_openid_roles_provider_integration + - rewrite-java-keystore-use-pyjks + - vault_kv2 + + +To see the full list of changes, you can run + + $ git log --merges --first-parent master --pretty=format:"%h %<(10,trunc)%aN %C(white)%<(15)%ar%Creset %C(red bold)%<(15)%D%Creset %s" | grep -v "depend" + + +Special thanks to all who contributed to this release, notably: + +- `peschmae `_ +- `sirferl `_ +- `lukasmrtvy `_ +- `intgr `_ +- `kush-bavishi `_ +- `alwaysjolley `_ +- `jplana `_ +- `explody `_ +- `titouanc `_ +- `jramosf `_ + + +Upgrading +--------- + +.. note:: This release will need a migration change. Please follow the `documentation `_ to upgrade Lemur. + + 0.7 - `2018-05-07` ~~~~~~~~~~~~~~ diff --git a/docs/administration.rst b/docs/administration.rst index 00da0c8a..3ef484be 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -28,6 +28,13 @@ Basic Configuration LOG_FILE = "/logs/lemur/lemur-test.log" +.. data:: LOG_UPGRADE_FILE + :noindex: + + :: + + LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log" + .. data:: DEBUG :noindex: @@ -155,17 +162,12 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s='] -.. data:: PUBLIC_CA_AUTHORITY_NAMES - :noindex: - A list of public issuers which would be checked against to determine whether limit of max validity of 397 days - should be applied to the certificate. Configure public CA authority names in this list to enforce validity check. - This is an optional setting. Using this will allow the sanity check as mentioned. The name check is a case-insensitive - string comparision. .. data:: PUBLIC_CA_MAX_VALIDITY_DAYS :noindex: - Use this config to override the limit of 397 days of validity for certificates issued by public issuers configured - using PUBLIC_CA_AUTHORITY_NAMES. Below example overrides the default validity of 397 days and sets it to 365 days. + Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities. + The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity + of 397 days and sets it to 365 days. :: @@ -175,8 +177,8 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c .. data:: DEFAULT_VALIDITY_DAYS :noindex: Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which - is not listed in PUBLIC_CA_AUTHORITY_NAMES will be using this value as default validity to be displayed on UI. Please - note that this config is used for cert issuance only through Lemur UI. Below example overrides the default validity + is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please + note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity of 365 days and sets it to 1095 days (3 years). :: @@ -274,7 +276,7 @@ Certificates marked as inactive will **not** be notified of upcoming expiration. silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or 30, 15, 2 days before expiration if no intervals are set. -Lemur supports sending certification expiration notifications through SES and SMTP. +Lemur supports sending certificate expiration notifications through SES and SMTP. .. data:: LEMUR_EMAIL_SENDER @@ -290,6 +292,25 @@ Lemur supports sending certification expiration notifications through SES and SM you can send any mail. See: `Verifying Email Address in Amazon SES `_ +.. data:: LEMUR_SES_SOURCE_ARN + :noindex: + + Specifies an ARN to use as the SourceArn when sending emails via SES. + + .. note:: + This parameter is only required if you're using a sending authorization with SES. + See: `Using sending authorization with Amazon SES `_ + + +.. data:: LEMUR_SES_REGION + :noindex: + + Specifies a region for sending emails via SES. + + .. note:: + This parameter defaults to us-east-1 and is only required if you wish to use a different region. + + .. data:: LEMUR_EMAIL :noindex: @@ -669,6 +690,20 @@ If you are not using a metric provider you do not need to configure any of these Plugin Specific Options ----------------------- +ACME Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. data:: ACME_DNS_PROVIDER_TYPES + :noindex: + + Dictionary of ACME DNS Providers and their requirements. + +.. data:: ACME_ENABLE_DELEGATED_CNAME + :noindex: + + Enables delegated DNS domain validation using CNAMES. When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges. + + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1441,7 +1476,7 @@ Slack Adds support for slack notifications. -AWS +AWS (Source) ---- :Authors: @@ -1454,7 +1489,7 @@ AWS Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment. -AWS +AWS (Destination) ---- :Authors: @@ -1467,6 +1502,19 @@ AWS Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment. +AWS (SNS Notification) +----- + +:Authors: + Jasmine Schladen +:Type: + Notification +:Description: + Adds support for SNS notifications. SNS notifications (like other notification plugins) are currently only supported + for certificate expiration. Configuration requires a region, account number, and SNS topic name; these elements + are then combined to build the topic ARN. Lemur must have access to publish messages to the specified SNS topic. + + Kubernetes ---------- diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index 8af5e1c8..c2a8c48a 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -215,18 +215,21 @@ Notification ------------ Lemur includes the ability to create Email notifications by **default**. These notifications -currently come in the form of expiration notices. Lemur periodically checks certifications expiration dates and +currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and determines if a given certificate is eligible for notification. There are currently only two parameters used to determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number of days the current date (UTC) is from that expiration date. -There are currently two objects that available for notification plugins the first is `NotficationPlugin`. This is the base object for -any notification within Lemur. Currently the only support notification type is an certificate expiration notification. If you +Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable. +Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email. + +There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for +any notification within Lemur. Currently the only supported notification type is a certificate expiration notification. If you are trying to create a new notification type (audit, failed logins, etc.) this would be the object to base your plugin on. You would also then need to build additional code to trigger the new notification type. -The second is `ExpirationNotificationPlugin`, this object inherits from `NotificationPlugin` object. -You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, HipChat, Jira, etc.). It adds default options that are required by +The second is `ExpirationNotificationPlugin`, which inherits from the `NotificationPlugin` object. +You will most likely want to base your plugin on this object if you want to add new channels for expiration notices (HipChat, Jira, etc.). It adds default options that are required by all expiration notifications (interval, unit). This interface expects for the child to define the following function:: def send(self, notification_type, message, targets, options, **kwargs): diff --git a/docs/production/index.rst b/docs/production/index.rst index c6f561ca..57800bd8 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -554,4 +554,123 @@ Using `python-jwt` converting an existing private key in PEM format is quite eas {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/"} -The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key. \ No newline at end of file +The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key. + +OpenSSH +======= + +OpenSSH (also known as OpenBSD Secure Shell) is a suite of secure networking utilities based on the Secure Shell (SSH) protocol, which provides a secure channel over an unsecured network in a client–server architecture. + +Using a PKI with OpenSSH means you can sign a key for a user and it can log into any server that trust the CA. + +Using a CA avoids TOFU or synchronize a list of server public keys to `known_hosts` files. + +This is useful when you're managing large number of machines or for an immutable infrastructure. + +Add first OpenSSH authority +--------------------------- + +To start issuing OpenSSH, you need to create an OpenSSH authority. To do this, visit +Authorities -> Create. Set the applicable attributes: + +- Name : OpenSSH +- Common Name: example.net + +Then click "More Options" and change the plugin value to "OpenSSH". + +Just click to "Create" button to add this authority. + +.. note:: OpenSSH do not support sub CA feature. + +Add a server certificate +------------------------- + +Now visit Certificates -> Create to add a server certificate. Set the applicable attributes: + +- Common Name: server.example.net + +Then click "More Options" and set the Certificate Template to "Server Certificate". + +This step is important, a certificat for a server and for a client is not exactly the same thing. +In this case "Common Name" and all Subject Alternate Names with type DNSName will be added in the certificate. + +Finally click on "Create" button. + +Add a client certificate +------------------------ + +Now visit Certificates -> Create to add a client certificate. Set the applicable attributes: + +- Common Name: example.net + +Then click "More Options" and set the Certificate Template to "Client Certificate". + +In this case the name of the creator is used as principal (in this documentation we assume that this certificate is created by the user "lemur"). + +Finally click on "Create" button. + +Configure OpenSSH server +------------------------ + +Connect to the server.example.net server to correctly configure the OpenSSH server with the CA created previously. + +First of all add the CA chain, private and public certificates: + +- Create file `/etc/ssh/ca.pub` and copy the "CHAIN" content of the *server certificate* (everything in one line). +- Create file `/etc/ssh/ssh_host_key` and copy "PRIVATE KEY" content. +- Create file `/etc/ssh/ssh_host_key.pub` and copy "PUBLIC CERTIFICATE" content (everything in one line). + +Set the appropriate right: + +.. code-block:: bash + + chmod 600 /etc/ssh/ca.pub /etc/ssh/ssh_host_key + chmod 644 /etc/ssh/ssh_host_key.pub + chown root: /etc/ssh/ca.pub /etc/ssh/ssh_host_key /etc/ssh/ssh_host_key.pub + +Then change OpenSSH server configuration to use these files. Edit `/etc/ssh/sshd_config` and add:: + + TrustedUserCAKeys /etc/ssh/ca.pub + HostKey /etc/ssh/ssh_host_key + HostCertificate /etc/ssh/ssh_host_key.pub + +You can remove all other `HostKey` lines. + +Finally restart OpenSSH. + +.. note:: By default the server public certificate is sign for 2 weeks. You must update the `/etc/ssh/ssh_host_key.pub` file before this delay. You can use the config's parameter OPENSSH_VALID_INTERVAL_SERVER to change this behavor (unit is number of day). + +Configure the OpenSSH client +---------------------------- + +Now you can configure the user's computer. + +First of all add private and public certificates: + +- Create file `~/.ssh/key` and copy "PRIVATE KEY" content. +- Create file `~/.ssh/key.pub` and copy "PUBLIC CERTIFICATE" content of the *client certicate* (everything in one line). + +Set the appropriate right: + +.. code-block:: bash + + chmod 600 ~/.ssh/key.pub ~/.ssh/key + +To avoid TOFU, edite the `~/.ssh/known_hosts` file and add a new line (all in one line): + +- @cert-authority \*example.net +- the "CHAIN" content + +Now you can connect to server with (here 'lemur' is the principal name and must exists on the server): + +.. code-block:: bash + + ssh lemur@server.example.net -i ~/.ssh/key + +With this configuration you don't have any line like:: + + Warning: Permanently added 'server.example.net,192.168.0.1' (RSA) to the list of known hosts. + +And you don't have to enter any password. + +.. note:: By default the client public certificate is sign for 1 day. You must update the `.ssh/key.pub` everyday. You can use the config's parameter OPENSSH_VALID_INTERVAL_CLIENT to change this behavor (unit is number of day). diff --git a/gulp/build.js b/gulp/build.js index eed59503..5aca8094 100644 --- a/gulp/build.js +++ b/gulp/build.js @@ -237,7 +237,7 @@ gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){ .forEach(function(file){ return gulp.src(file) .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/'))) - .pipe(gulpif(urlContextPathExists, replace('angular/', argv.urlContextPath + '/angular/'))) + .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/'))) .pipe(gulp.dest(function(file){ return file.base; })) @@ -256,10 +256,9 @@ gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], functi var manifest = gulp.src("lemur/static/dist/rev-manifest.json"); var urlContextPathExists = argv.urlContextPath ? true : false; return gulp.src( "lemur/static/dist/index.html") - .pipe(gulpif(urlContextPathExists, revReplace({prefix: argv.urlContextPath + '/', manifest: manifest}, revReplace({manifest: manifest})))) .pipe(gulp.dest('lemur/static/dist')); }) gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']); -gulp.task('package', ['addUrlContextPath', 'package:strip']); \ No newline at end of file +gulp.task('package', ['addUrlContextPath', 'package:strip']); diff --git a/lemur/__about__.py b/lemur/__about__.py index 766d3668..0926ef33 100644 --- a/lemur/__about__.py +++ b/lemur/__about__.py @@ -15,7 +15,7 @@ __title__ = "lemur" __summary__ = "Certificate management and orchestration service" __uri__ = "https://github.com/Netflix/lemur" -__version__ = "0.7.0" +__version__ = "0.8.0" __author__ = "The Lemur developers" __email__ = "security@netflix.com" diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 310efad1..7efa196e 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -1,12 +1,15 @@ import time import json +import arrow from flask_script import Manager from flask import current_app from lemur.extensions import sentry from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler +from lemur.plugins.lemur_aws import s3 manager = Manager( usage="Handles all ACME related tasks" @@ -84,3 +87,105 @@ def dnstest(domain, token): status = SUCCESS_METRIC_STATUS print("[+] Done with ACME Tests.") + + +@manager.option( + "-t", + "--token", + dest="token", + default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="Value of the Token", +) +@manager.option( + "-n", + "--token_name", + dest="token_name", + default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="path", +) +@manager.option( + "-p", + "--prefix", + dest="prefix", + default="test/", + required=False, + help="S3 bucket prefix", +) +@manager.option( + "-a", + "--account_number", + dest="account_number", + required=True, + help="AWS Account", +) +@manager.option( + "-b", + "--bucket_name", + dest="bucket_name", + required=True, + help="Bucket Name", +) +def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): + """ + This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it. + It mainly serves for testing purposes. + :param token: + :param token_name: + :param prefix: + :param account_number: + :param bucket_name: + :return: + """ + additional_options = [ + { + "name": "bucket", + "value": bucket_name, + "type": "str", + "required": True, + "validation": r"[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account_number, + "required": True, + "validation": r"[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + p = plugins.get("aws-s3") + p.upload_acme_token(token_name, token, additional_options) + + if not prefix.endswith("/"): + prefix + "/" + + token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number) + assert(token_res == token) + s3.delete(bucket_name, prefix + token_name, account_number=account_number) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index ed87b76c..030c7c78 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -210,7 +210,8 @@ class LdapPrincipal: self.ldap_groups = [] for group in lgroups: (dn, values) = group - self.ldap_groups.append(values["cn"][0].decode("ascii")) + if type(values) == dict: + self.ldap_groups.append(values["cn"][0].decode("ascii")) else: lgroups = self.ldap_client.search_s( self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 0e1521b3..f954ce51 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -101,7 +101,8 @@ def login_required(f): return dict(message="Token is invalid"), 403 try: - payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"]) + header_data = fetch_token_header(token) + payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"], algorithms=[header_data["alg"]]) except jwt.DecodeError: return dict(message="Token is invalid"), 403 except jwt.ExpiredSignatureError: diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index d1b41a21..94985cc9 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -8,6 +8,7 @@ """ import json +from flask import current_app from sqlalchemy.orm import relationship from sqlalchemy import ( Column, @@ -17,7 +18,7 @@ from sqlalchemy import ( func, ForeignKey, DateTime, - PassiveDefault, + DefaultClause, Boolean, ) from sqlalchemy.dialects.postgresql import JSON @@ -38,7 +39,7 @@ class Authority(db.Model): plugin_name = Column(String(64)) description = Column(Text) options = Column(JSON) - date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False) + date_created = Column(DateTime, DefaultClause(func.now()), nullable=False) roles = relationship( "Role", secondary=roles_authorities, @@ -92,11 +93,25 @@ class Authority(db.Model): if not self.options: return None - for option in json.loads(self.options): - if "name" in option and option["name"] == 'cab_compliant': - return option["value"] + options_array = json.loads(self.options) + if isinstance(options_array, list): + for option in options_array: + if "name" in option and option["name"] == 'cab_compliant': + return option["value"] return None + @property + def max_issuance_days(self): + if self.is_cab_compliant: + return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) + + @property + def default_validity_days(self): + if self.is_cab_compliant: + return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) + + return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365) # 1 year default + def __repr__(self): return "Authority(name={name})".format(name=self.name) diff --git a/lemur/authorities/schemas.py b/lemur/authorities/schemas.py index 6c48a183..225e8cd1 100644 --- a/lemur/authorities/schemas.py +++ b/lemur/authorities/schemas.py @@ -7,7 +7,7 @@ """ from flask import current_app -from marshmallow import fields, validates_schema, pre_load +from marshmallow import fields, validates_schema, pre_load, post_dump from marshmallow import validate from marshmallow.exceptions import ValidationError @@ -24,6 +24,7 @@ from lemur.common import validators, missing from lemur.common.fields import ArrowDateTime from lemur.constants import CERTIFICATE_KEY_TYPES +from lemur.plugins.base import plugins class AuthorityInputSchema(LemurInputSchema): @@ -111,8 +112,6 @@ class RootAuthorityCertificateOutputSchema(LemurOutputSchema): cn = fields.String() not_after = fields.DateTime() not_before = fields.DateTime() - max_issuance_days = fields.Integer() - default_validity_days = fields.Integer() owner = fields.Email() status = fields.Boolean() user = fields.Nested(UserNestedOutputSchema) @@ -127,8 +126,16 @@ class AuthorityOutputSchema(LemurOutputSchema): active = fields.Boolean() options = fields.Dict() roles = fields.List(fields.Nested(AssociatedRoleSchema)) + max_issuance_days = fields.Integer() + default_validity_days = fields.Integer() authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema) + @post_dump + def handle_auth_certificate(self, cert): + # Plugins may need to modify the cert object before returning it to the user + plugin = plugins.get(cert['plugin']['slug']) + plugin.wrap_auth_certificate(cert['authority_certificate']) + class AuthorityNestedOutputSchema(LemurOutputSchema): __envelope__ = False @@ -138,8 +145,10 @@ class AuthorityNestedOutputSchema(LemurOutputSchema): owner = fields.Email() plugin = fields.Nested(PluginOutputSchema) active = fields.Boolean() - authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["max_issuance_days", "default_validity_days"]) + authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["not_after", "not_before"]) is_cab_compliant = fields.Boolean() + max_issuance_days = fields.Integer() + default_validity_days = fields.Integer() authority_update_schema = AuthorityUpdateSchema() diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index b883dee0..f23948be 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -735,3 +735,45 @@ def automatically_enable_autorotate(): }) cert.rotation = True database.update(cert) + + +@manager.command +def deactivate_entrust_certificates(): + """ + Attempt to deactivate test certificates issued by Entrust + """ + + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Deactivating Entrust certificates" + } + + certificates = get_all_valid_certs(['entrust-issuer']) + entrust_plugin = plugins.get('entrust-issuer') + for cert in certificates: + try: + response = entrust_plugin.deactivate_certificate(cert) + if response == 200: + cert.status = "revoked" + else: + cert.status = "unknown" + + log_data["valid"] = cert.status + log_data["certificate_name"] = cert.name + log_data["certificate_id"] = cert.id + metrics.send( + "certificate_deactivate", + "counter", + 1, + metric_tags={"status": log_data["valid"], + "certificate_name": log_data["certificate_name"], + "certificate_id": log_data["certificate_id"]}, + ) + current_app.logger.info(log_data) + + database.update(cert) + + except Exception as e: + current_app.logger.info(log_data) + sentry.captureException() + current_app.logger.exception(e) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 60442de2..94e3a42e 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -16,7 +16,7 @@ from sqlalchemy import ( Integer, ForeignKey, String, - PassiveDefault, + DefaultClause, func, Column, Text, @@ -138,7 +138,7 @@ class Certificate(db.Model): not_after = Column(ArrowType) not_after_ix = Index("ix_certificates_not_after", not_after.desc()) - date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False) + date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False) signing_algorithm = Column(String(128)) status = Column(String(128)) @@ -184,7 +184,6 @@ class Certificate(db.Model): "PendingCertificate", secondary=pending_cert_replacement_associations, backref="pending_replace", - viewonly=True, ) logs = relationship("Log", backref="certificate") @@ -317,20 +316,6 @@ class Certificate(db.Model): def validity_range(self): return self.not_after - self.not_before - @property - def max_issuance_days(self): - public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", []) - if self.name.lower() in [ca.lower() for ca in public_CA]: - return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) - - @property - def default_validity_days(self): - public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", []) - if self.name.lower() in [ca.lower() for ca in public_CA]: - return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) - - return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365) # 1 year default - @property def subject(self): return self.parsed_cert.subject diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index cc0a607e..6fb2c15f 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -38,6 +38,7 @@ from lemur.schemas import ( AssociatedRotationPolicySchema, ) from lemur.users.schemas import UserNestedOutputSchema +from lemur.plugins.base import plugins class CertificateSchema(LemurInputSchema): @@ -324,6 +325,8 @@ class CertificateOutputSchema(LemurOutputSchema): notifications = fields.Nested(NotificationNestedOutputSchema, many=True) replaces = fields.Nested(CertificateNestedOutputSchema, many=True) authority = fields.Nested(AuthorityNestedOutputSchema) + # if this certificate is an authority, the authority informations are in root_authority + root_authority = fields.Nested(AuthorityNestedOutputSchema) dns_provider = fields.Nested(DnsProvidersNestedOutputSchema) roles = fields.Nested(RoleNestedOutputSchema, many=True) endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[]) @@ -340,6 +343,8 @@ class CertificateOutputSchema(LemurOutputSchema): @post_dump def handle_subject_details(self, data): + subject_details = ["country", "state", "location", "organization", "organizational_unit"] + # Remove subject details if authority is CA/Browser Forum compliant. The code will use default set of values in that case. # If CA/Browser Forum compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below # condition checks for 'not False' ==> 'True or None' @@ -347,11 +352,23 @@ class CertificateOutputSchema(LemurOutputSchema): is_cab_compliant = data.get("authority").get("isCabCompliant") if is_cab_compliant is not False: - data.pop("country", None) - data.pop("state", None) - data.pop("location", None) - data.pop("organization", None) - data.pop("organizational_unit", None) + for field in subject_details: + data.pop(field, None) + + # Removing subject fields if None, else it complains in de-serialization + for field in subject_details: + if field in data and data[field] is None: + data.pop(field) + + @post_dump + def handle_certificate(self, cert): + # Plugins may need to modify the cert object before returning it to the user + if cert['root_authority'] and cert['authority'] is None: + # this certificate is an authority + cert['authority'] = cert['root_authority'] + del cert['root_authority'] + plugin = plugins.get(cert['authority']['plugin']['slug']) + plugin.wrap_certificate(cert) class CertificateShortOutputSchema(LemurOutputSchema): diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 6d1bd2ac..df8450d3 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -6,11 +6,13 @@ .. moduleauthor:: Kevin Glisson """ import arrow +import re from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from flask import current_app from sqlalchemy import func, or_, not_, cast, Integer +from sqlalchemy.sql.expression import false, true from lemur import database from lemur.authorities.models import Authority @@ -85,6 +87,16 @@ def get_by_attributes(conditions): return database.find_all(query, Certificate, conditions).all() +def get_by_root_authority(id): + """ + Retrieves certificate by its root_authority's id. + + :param id: + :return: + """ + return database.get(Certificate, id, field="root_authority_id") + + def delete(cert_id): """ Delete's a certificate. @@ -105,7 +117,7 @@ def get_all_certs(): def get_all_valid_certs(authority_plugin_name): """ - Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names + Retrieves all valid (not expired & not revoked) certificates within Lemur, for the given authority plugin names ignored if no authority_plugin_name provided. Note that depending on the DB size retrieving all certificates might an expensive operation @@ -116,11 +128,12 @@ def get_all_valid_certs(authority_plugin_name): return ( Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter( Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter( - Authority.plugin_name.in_(authority_plugin_name)).all() + Authority.plugin_name.in_(authority_plugin_name)).filter(Certificate.revoked.is_(False)).all() ) else: return ( - Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all() + Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter( + Certificate.revoked.is_(False)).all() ) @@ -148,7 +161,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate(): """ return ( Certificate.query.filter(Certificate.endpoints.any()) - .filter(Certificate.rotation == False) + .filter(Certificate.rotation == false()) .filter(Certificate.not_after >= arrow.now()) .filter(not_(Certificate.replaced.any())) .all() # noqa @@ -203,9 +216,9 @@ def get_all_pending_reissue(): :return: """ return ( - Certificate.query.filter(Certificate.rotation == True) + Certificate.query.filter(Certificate.rotation == true()) .filter(not_(Certificate.replaced.any())) - .filter(Certificate.in_rotation_window == True) + .filter(Certificate.in_rotation_window == true()) .all() ) # noqa @@ -359,7 +372,12 @@ def create(**kwargs): try: cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) except Exception: - current_app.logger.error("Exception minting certificate", exc_info=True) + log_data = { + "message": "Exception minting certificate", + "issuer": kwargs["authority"].name, + "cn": kwargs["common_name"], + } + current_app.logger.error(log_data, exc_info=True) sentry.captureException() raise kwargs["body"] = cert_body @@ -518,7 +536,7 @@ def render(args): ) if current_app.config.get("ALLOW_CERT_DELETION", False): - query = query.filter(Certificate.deleted == False) # noqa + query = query.filter(Certificate.deleted == false()) result = database.sort_and_page(query, Certificate, args) return result @@ -554,20 +572,21 @@ def query_common_name(common_name, args): :return: """ owner = args.pop("owner") - if not owner: - owner = "%" - # only not expired certificates current_time = arrow.utcnow() - result = ( - Certificate.query.filter(Certificate.cn.ilike(common_name)) - .filter(Certificate.owner.ilike(owner)) - .filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) - .all() - ) + query = Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\ + .filter(not_(Certificate.revoked))\ + .filter(not_(Certificate.replaced.any())) # ignore rotated certificates to avoid duplicates - return result + if owner: + query = query.filter(Certificate.owner.ilike(owner)) + + if common_name != "%": + # if common_name is a wildcard ('%'), no need to include it in the query + query = query.filter(Certificate.cn.ilike(common_name)) + + return query.all() def create_csr(**csr_config): @@ -772,6 +791,19 @@ def reissue_certificate(certificate, replace=None, user=None): if replace: primitives["replaces"] = [certificate] + # Modify description to include the certificate ID being reissued and mention that this is created by Lemur + # as part of reissue + reissue_message_prefix = "Reissued by Lemur for cert ID " + reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)") + if primitives["description"]: + match = reissue_message.search(primitives["description"]) + if match: + primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id)) + else: + primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}" + else: + primitives["description"] = f"{reissue_message_prefix}{certificate.id}" + new_cert = create(**primitives) return new_cert diff --git a/lemur/certificates/utils.py b/lemur/certificates/utils.py index e642e058..18f1c967 100644 --- a/lemur/certificates/utils.py +++ b/lemur/certificates/utils.py @@ -82,4 +82,4 @@ def get_key_type_from_csr(data): raise Exception("Unsupported key type") except NotImplemented: - raise NotImplemented() + raise NotImplementedError diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 18746636..d4f1860c 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -675,6 +675,16 @@ class CertificatePrivateKey(AuthenticatedResource): return dict(message="You are not authorized to view this key"), 403 log_service.create(g.current_user, "key_view", certificate=cert) + + # Plugins may need to modify the cert object before returning it to the user + if cert.root_authority: + # this certificate is an authority + plugin_name = cert.root_authority.plugin_name + else: + plugin_name = cert.authority.plugin_name + plugin = plugins.get(plugin_name) + plugin.wrap_private_key(cert) + response = make_response(jsonify(key=cert.private_key), 200) response.headers["cache-control"] = "private, max-age=0, no-cache, no-store" response.headers["pragma"] = "no-cache" @@ -1155,6 +1165,7 @@ class NotificationCertificatesList(AuthenticatedResource): ) parser.add_argument("creator", type=str, location="args") parser.add_argument("show", type=str, location="args") + parser.add_argument("showExpired", type=int, location="args") args = parser.parse_args() args["notification_id"] = notification_id diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a490b13b..f9d58bd9 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -759,7 +759,7 @@ def check_revoked(): log_data = { "function": function, - "message": "check if any certificates are revoked revoked", + "message": "check if any valid certificate is revoked", "task_id": task_id, } @@ -842,3 +842,39 @@ def enable_autorotate_for_certs_attached_to_endpoint(): cli_certificate.automatically_enable_autorotate() metrics.send(f"{function}.success", "counter", 1) return log_data + + +@celery.task(soft_time_limit=3600) +def deactivate_entrust_test_certificates(): + """ + This celery task attempts to deactivate all not yet deactivated Entrust certificates, and should only run in TEST + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "message": "deactivate entrust certificates", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + + current_app.logger.debug(log_data) + try: + cli_certificate.deactivate_entrust_certificates() + except SoftTimeLimitExceeded: + log_data["message"] = "Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return + + metrics.send(f"{function}.success", "counter", 1) + return log_data diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index b9c88e49..d7b37292 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -95,9 +95,11 @@ def organization(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[ - 0 - ].value.strip() + o = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME) + if not o: + return None + + return o[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get organization! {0}".format(e)) @@ -110,9 +112,11 @@ def organizational_unit(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)[ - 0 - ].value.strip() + ou = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME) + if not ou: + return None + + return ou[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get organizational unit! {0}".format(e)) @@ -125,9 +129,11 @@ def country(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)[ - 0 - ].value.strip() + c = cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME) + if not c: + return None + + return c[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get country! {0}".format(e)) @@ -140,9 +146,11 @@ def state(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)[ - 0 - ].value.strip() + s = cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME) + if not s: + return None + + return s[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get state! {0}".format(e)) @@ -155,9 +163,11 @@ def location(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)[ - 0 - ].value.strip() + loc = cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME) + if not loc: + return None + + return loc[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get location! {0}".format(e)) diff --git a/lemur/destinations/schemas.py b/lemur/destinations/schemas.py index cc46ecd4..22187a09 100644 --- a/lemur/destinations/schemas.py +++ b/lemur/destinations/schemas.py @@ -31,6 +31,9 @@ class DestinationOutputSchema(LemurOutputSchema): def fill_object(self, data): if data: data["plugin"]["pluginOptions"] = data["options"] + for option in data["plugin"]["pluginOptions"]: + if "export-plugin" in option["type"]: + option["value"]["pluginOptions"] = option["value"]["plugin_options"] return data diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 92162f4b..7bae57f0 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -41,12 +41,14 @@ def create(label, plugin_name, options, description=None): return database.create(destination) -def update(destination_id, label, options, description): +def update(destination_id, label, plugin_name, options, description): """ Updates an existing destination. :param destination_id: Lemur assigned ID :param label: Destination common name + :param plugin_name: + :param options: :param description: :rtype : Destination :return: @@ -54,6 +56,11 @@ def update(destination_id, label, options, description): destination = get(destination_id) destination.label = label + destination.plugin_name = plugin_name + # remove any sub-plugin objects before try to save the json options + for option in options: + if "plugin" in option["type"]: + del option["value"]["plugin_object"] destination.options = options destination.description = description diff --git a/lemur/destinations/views.py b/lemur/destinations/views.py index 0b0559fe..072ff34e 100644 --- a/lemur/destinations/views.py +++ b/lemur/destinations/views.py @@ -338,6 +338,7 @@ class Destinations(AuthenticatedResource): return service.update( destination_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], ) diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index b14e0339..2b8d7008 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -3,9 +3,9 @@ from flask_script import Manager import sys from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins.lemur_acme.acme_handlers import AcmeDnsHandler from lemur.dns_providers.service import get_all_dns_providers, set_domains from lemur.extensions import metrics, sentry -from lemur.plugins.base import plugins manager = Manager( usage="Iterates through all DNS providers and sets DNS zones in the database." @@ -19,7 +19,7 @@ def get_all_zones(): """ print("[+] Starting dns provider zone lookup and configuration.") dns_providers = get_all_dns_providers() - acme_plugin = plugins.get("acme-issuer") + acme_dns_handler = AcmeDnsHandler() function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { @@ -29,7 +29,7 @@ def get_all_zones(): for dns_provider in dns_providers: try: - zones = acme_plugin.get_all_zones(dns_provider) + zones = acme_dns_handler.get_all_zones(dns_provider) set_domains(dns_provider, zones) except Exception as e: print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e)) diff --git a/lemur/factory.py b/lemur/factory.py index 0563d873..edea571a 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -10,7 +10,7 @@ """ import os -import imp +import importlib import errno import pkg_resources import socket @@ -73,8 +73,9 @@ def from_file(file_path, silent=False): :param file_path: :param silent: """ - d = imp.new_module("config") - d.__file__ = file_path + module_spec = importlib.util.spec_from_file_location("config", file_path) + d = importlib.util.module_from_spec(module_spec) + try: with open(file_path) as config_file: exec( # nosec: config file safe diff --git a/lemur/logs/models.py b/lemur/logs/models.py index 07a2ded3..30cc204a 100644 --- a/lemur/logs/models.py +++ b/lemur/logs/models.py @@ -7,7 +7,7 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, ForeignKey, PassiveDefault, func, Enum +from sqlalchemy import Column, Integer, ForeignKey, DefaultClause, func, Enum from sqlalchemy_utils.types.arrow import ArrowType @@ -29,5 +29,5 @@ class Log(db.Model): ), nullable=False, ) - logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False) + logged_at = Column(ArrowType(), DefaultClause(func.now()), nullable=False) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) diff --git a/lemur/manage.py b/lemur/manage.py index e53f8bd6..bff97535 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -120,6 +120,7 @@ METRIC_PROVIDERS = [] LOG_LEVEL = "DEBUG" LOG_FILE = "lemur.log" +LOG_UPGRADE_FILE = "db_upgrade.log" # Database diff --git a/lemur/migrations/env.py b/lemur/migrations/env.py index 3acefc3a..91fa5fcb 100644 --- a/lemur/migrations/env.py +++ b/lemur/migrations/env.py @@ -20,8 +20,9 @@ fileConfig(config.config_file_name) # target_metadata = mymodel.Base.metadata from flask import current_app +db_url_escaped = current_app.config.get('SQLALCHEMY_DATABASE_URI').replace('%', '%%') config.set_main_option( - "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI") + "sqlalchemy.url", db_url_escaped ) target_metadata = current_app.extensions["migrate"].db.metadata diff --git a/lemur/migrations/versions/1db4f82bc780_.py b/lemur/migrations/versions/1db4f82bc780_.py index e6fb47f0..00b83ceb 100644 --- a/lemur/migrations/versions/1db4f82bc780_.py +++ b/lemur/migrations/versions/1db4f82bc780_.py @@ -10,11 +10,21 @@ Create Date: 2018-08-03 12:56:44.565230 revision = "1db4f82bc780" down_revision = "3adfdd6598df" -import logging - from alembic import op -log = logging.getLogger(__name__) +from flask import current_app +from logging import Formatter, FileHandler, getLogger + +log = getLogger(__name__) +handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log")) +handler.setFormatter( + Formatter( + "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]" + ) +) +handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.addHandler(handler) def upgrade(): diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 669c934f..c4213f49 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -7,8 +7,9 @@ the rest of the keys, the certificate body is parsed to determine the exact key_type information. Each individual DB change is explicitly committed, and the respective -log is added to a file named db_upgrade.log in the current working -directory. Any error encountered while parsing a certificate will +log is added to a file configured in LOG_UPGRADE_FILE or, by default, +to a file named db_upgrade.log in the current working directory. +Any error encountered while parsing a certificate will also be logged along with the certificate ID. If faced with any issue while running this upgrade, there is no harm in re-running the upgrade. Each run processes only rows for which key_type information is not yet @@ -31,15 +32,28 @@ down_revision = '434c29e40511' from alembic import op from sqlalchemy.sql import text -from lemur.common import utils import time import datetime +from flask import current_app -log_file = open('db_upgrade.log', 'a') +from logging import Formatter, FileHandler, getLogger + +from lemur.common import utils + +log = getLogger(__name__) +handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log")) +handler.setFormatter( + Formatter( + "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]" + ) +) +handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.addHandler(handler) def upgrade(): - log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) + log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) start_time = time.time() # Update RSA keys using the key length information @@ -50,8 +64,7 @@ def upgrade(): # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. update_key_type() - log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) - log_file.close() + log.info("--- Total %s seconds ---\n" % (time.time() - start_time)) def downgrade(): @@ -61,6 +74,7 @@ def downgrade(): "update certificates set key_type=null where not_after > CURRENT_DATE - 32" ) op.execute(stmt) + commit() """ @@ -69,18 +83,18 @@ def downgrade(): def update_key_type_rsa(bits): - log_file.write("Processing certificate with key type RSA %s\n" % bits) + log.info("Processing certificate with key type RSA %s\n" % bits) stmt = text( f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" ) - log_file.write("Query: %s\n" % stmt) + log.info("Query: %s\n" % stmt) start_time = time.time() op.execute(stmt) commit() - log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + log.info("--- %s seconds ---\n" % (time.time() - start_time)) def update_key_type(): @@ -95,9 +109,9 @@ def update_key_type(): try: cert_key_type = utils.get_key_type_from_certificate(body) except ValueError as e: - log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) + log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) else: - log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) stmt = text( "update certificates set key_type=:key_type where id=:id" ) @@ -106,7 +120,7 @@ def update_key_type(): commit() - log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + log.info("--- %s seconds ---\n" % (time.time() - start_time)) def commit(): diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 82db7b6e..75d227b1 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -8,6 +8,7 @@ .. moduleauthor:: Kevin Glisson """ +import sys from collections import defaultdict from datetime import timedelta from itertools import groupby @@ -15,6 +16,7 @@ from itertools import groupby import arrow from flask import current_app from sqlalchemy import and_ +from sqlalchemy.sql.expression import false, true from lemur import database from lemur.certificates.models import Certificate @@ -29,7 +31,7 @@ from lemur.plugins.utils import get_plugin_option def get_certificates(exclude=None): """ - Finds all certificates that are eligible for notifications. + Finds all certificates that are eligible for expiration notifications. :param exclude: :return: """ @@ -39,9 +41,10 @@ def get_certificates(exclude=None): q = ( database.db.session.query(Certificate) .filter(Certificate.not_after <= max) - .filter(Certificate.notify == True) - .filter(Certificate.expired == False) - ) # noqa + .filter(Certificate.notify == true()) + .filter(Certificate.expired == false()) + .filter(Certificate.revoked == false()) + ) exclude_conditions = [] if exclude: @@ -61,7 +64,8 @@ def get_certificates(exclude=None): def get_eligible_certificates(exclude=None): """ - Finds all certificates that are eligible for certificate expiration. + Finds all certificates that are eligible for certificate expiration notification. + Returns the set of all eligible certificates, grouped by owner, with a list of applicable notifications. :param exclude: :return: """ @@ -86,21 +90,31 @@ def get_eligible_certificates(exclude=None): return certificates -def send_notification(event_type, data, targets, notification): +def send_plugin_notification(event_type, data, recipients, notification): """ Executes the plugin and handles failure. :param event_type: :param data: - :param targets: + :param recipients: :param notification: :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": f"Sending expiration notification for to recipients {recipients}", + "notification_type": "expiration", + "certificate_targets": recipients, + } status = FAILURE_METRIC_STATUS try: - notification.plugin.send(event_type, data, targets, notification.options) + current_app.logger.debug(log_data) + notification.plugin.send(event_type, data, recipients, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: + log_data["message"] = f"Unable to send {event_type} notification to recipients {recipients}" + current_app.logger.error(log_data, exc_info=True) sentry.captureException() metrics.send( @@ -124,11 +138,11 @@ def send_expiration_notifications(exclude): # security team gets all security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") - security_data = [] for owner, notification_group in get_eligible_certificates(exclude=exclude).items(): for notification_label, certificates in notification_group.items(): notification_data = [] + security_data = [] notification = certificates[0][0] @@ -140,36 +154,27 @@ def send_expiration_notifications(exclude): notification_data.append(cert_data) security_data.append(cert_data) - if send_notification( - "expiration", notification_data, [owner], notification + if send_default_notification( + "expiration", notification_data, [owner], notification.options ): success += 1 else: failure += 1 - notification_recipient = get_plugin_option( - "recipients", notification.options - ) - if notification_recipient: - notification_recipient = notification_recipient.split(",") - # removing owner and security_email from notification_recipient - notification_recipient = [i for i in notification_recipient if i not in security_email and i != owner] + recipients = notification.plugin.filter_recipients(notification.options, security_email + [owner]) - if ( - notification_recipient + if send_plugin_notification( + "expiration", + notification_data, + recipients, + notification, ): - if send_notification( - "expiration", - notification_data, - notification_recipient, - notification, - ): - success += 1 - else: - failure += 1 + success += 1 + else: + failure += 1 - if send_notification( - "expiration", security_data, security_email, notification + if send_default_notification( + "expiration", security_data, security_email, notification.options ): success += 1 else: @@ -178,107 +183,86 @@ def send_expiration_notifications(exclude): return success, failure -def send_rotation_notification(certificate, notification_plugin=None): +def send_default_notification(notification_type, data, targets, notification_options=None): """ - Sends a report to certificate owners when their certificate has been - rotated. + Sends a report to the specified target via the default notification plugin. Applicable for any notification_type. + At present, "default" means email, as the other notification plugins do not support dynamically configured targets. - :param certificate: - :param notification_plugin: + :param notification_type: + :param data: + :param targets: + :param notification_options: :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": f"Sending notification for certificate data {data}", + "notification_type": notification_type, + } status = FAILURE_METRIC_STATUS - if not notification_plugin: - notification_plugin = plugins.get( - current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN") - ) - - data = certificate_notification_output_schema.dump(certificate).data + notification_plugin = plugins.get( + current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification") + ) try: - notification_plugin.send("rotation", data, [data["owner"]]) + current_app.logger.debug(log_data) + # we need the notification.options here because the email templates utilize the interval/unit info + notification_plugin.send(notification_type, data, targets, notification_options) status = SUCCESS_METRIC_STATUS except Exception as e: - current_app.logger.error( - "Unable to send notification to {}.".format(data["owner"]), exc_info=True - ) + log_data["message"] = f"Unable to send {notification_type} notification for certificate data {data} " \ + f"to target {targets}" + current_app.logger.error(log_data, exc_info=True) sentry.captureException() metrics.send( "notification", "counter", 1, - metric_tags={"status": status, "event_type": "rotation"}, + metric_tags={"status": status, "event_type": notification_type}, ) if status == SUCCESS_METRIC_STATUS: return True +def send_rotation_notification(certificate): + data = certificate_notification_output_schema.dump(certificate).data + return send_default_notification("rotation", data, [data["owner"]]) + + def send_pending_failure_notification( - pending_cert, notify_owner=True, notify_security=True, notification_plugin=None + pending_cert, notify_owner=True, notify_security=True ): """ Sends a report to certificate owners when their pending certificate failed to be created. :param pending_cert: - :param notification_plugin: + :param notify_owner: + :param notify_security: :return: """ - status = FAILURE_METRIC_STATUS - - if not notification_plugin: - notification_plugin = plugins.get( - current_app.config.get( - "LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification" - ) - ) data = pending_certificate_output_schema.dump(pending_cert).data data["security_email"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") + notify_owner_success = False if notify_owner: - try: - notification_plugin.send("failed", data, [data["owner"]], pending_cert) - status = SUCCESS_METRIC_STATUS - except Exception as e: - current_app.logger.error( - "Unable to send pending failure notification to {}.".format( - data["owner"] - ), - exc_info=True, - ) - sentry.captureException() + notify_owner_success = send_default_notification("failed", data, [data["owner"]], pending_cert) + notify_security_success = False if notify_security: - try: - notification_plugin.send( - "failed", data, data["security_email"], pending_cert - ) - status = SUCCESS_METRIC_STATUS - except Exception as e: - current_app.logger.error( - "Unable to send pending failure notification to " - "{}.".format(data["security_email"]), - exc_info=True, - ) - sentry.captureException() + notify_security_success = send_default_notification("failed", data, data["security_email"], pending_cert) - metrics.send( - "notification", - "counter", - 1, - metric_tags={"status": status, "event_type": "rotation"}, - ) - - if status == SUCCESS_METRIC_STATUS: - return True + return notify_owner_success or notify_security_success def needs_notification(certificate): """ - Determine if notifications for a given certificate should - currently be sent + Determine if notifications for a given certificate should currently be sent. + For each notification configured for the cert, verifies it is active, properly configured, + and that the configured expiration period is currently met. :param certificate: :return: @@ -290,7 +274,7 @@ def needs_notification(certificate): for notification in certificate.notifications: if not notification.active or not notification.options: - return + continue interval = get_plugin_option("interval", notification.options) unit = get_plugin_option("unit", notification.options) @@ -306,9 +290,8 @@ def needs_notification(certificate): else: raise Exception( - "Invalid base unit for expiration interval: {0}".format(unit) + f"Invalid base unit for expiration interval: {unit}" ) - if days == interval: notifications.append(notification) return notifications diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index ac624d1c..5bc5f3e1 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -43,7 +43,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): "name": "recipients", "type": "str", "required": True, - "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", + "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", "helpMessage": "Comma delimited list of email addresses", "value": ",".join(recipients), }, @@ -63,7 +63,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): "name": "interval", "type": "int", "required": True, - "validation": "^\d+$", + "validation": r"^\d+$", "helpMessage": "Number of days to be alert before expiration.", "value": i, } @@ -104,12 +104,13 @@ def create(label, plugin_name, options, description, certificates): return database.create(notification) -def update(notification_id, label, options, description, active, certificates): +def update(notification_id, label, plugin_name, options, description, active, certificates): """ Updates an existing notification. :param notification_id: :param label: Notification label + :param plugin_name: :param options: :param description: :param active: @@ -120,6 +121,7 @@ def update(notification_id, label, options, description, active, certificates): notification = get(notification_id) notification.label = label + notification.plugin_name = plugin_name notification.options = options notification.description = description notification.active = active diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index cdabb4d4..f6eef655 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -340,6 +340,7 @@ class Notifications(AuthenticatedResource): return service.update( notification_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], data["active"], diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index fa6be073..ee3e5e97 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -9,7 +9,7 @@ from sqlalchemy import ( Integer, ForeignKey, String, - PassiveDefault, + DefaultClause, func, Column, Text, @@ -76,14 +76,14 @@ class PendingCertificate(db.Model): chain = Column(Text()) private_key = Column(Vault, nullable=True) - date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False) + date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False) dns_provider_id = Column( Integer, ForeignKey("dns_providers.id", ondelete="CASCADE") ) status = Column(Text(), nullable=True) last_updated = Column( - ArrowType, PassiveDefault(func.now()), onupdate=func.now(), nullable=False + ArrowType, DefaultClause(func.now()), onupdate=func.now(), nullable=False ) rotation = Column(Boolean, default=False) diff --git a/lemur/plugins/bases/issuer.py b/lemur/plugins/bases/issuer.py index f1e6aa0e..659611f3 100644 --- a/lemur/plugins/bases/issuer.py +++ b/lemur/plugins/bases/issuer.py @@ -31,3 +31,12 @@ class IssuerPlugin(Plugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): raise NotImplementedError + + def wrap_certificate(self, cert): + pass + + def wrap_auth_certificate(self, cert): + pass + + def wrap_private_key(self, cert): + pass diff --git a/lemur/plugins/bases/notification.py b/lemur/plugins/bases/notification.py index 730f68be..76aa33de 100644 --- a/lemur/plugins/bases/notification.py +++ b/lemur/plugins/bases/notification.py @@ -20,6 +20,15 @@ class NotificationPlugin(Plugin): def send(self, notification_type, message, targets, options, **kwargs): raise NotImplementedError + def filter_recipients(self, options, excluded_recipients): + """ + Given a set of options (which should include configured recipient info), filters out recipients that + we do NOT want to notify. + + For any notification types where recipients can't be dynamically modified, this returns an empty list. + """ + return [] + class ExpirationNotificationPlugin(NotificationPlugin): """ @@ -33,7 +42,7 @@ class ExpirationNotificationPlugin(NotificationPlugin): "name": "interval", "type": "int", "required": True, - "validation": "^\d+$", + "validation": r"^\d+$", "helpMessage": "Number of days to be alert before expiration.", }, { @@ -50,5 +59,5 @@ class ExpirationNotificationPlugin(NotificationPlugin): def options(self): return self.default_options + self.additional_options - def send(self, notification_type, message, targets, options, **kwargs): + def send(self, notification_type, message, excluded_targets, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py new file mode 100644 index 00000000..c1ab5281 --- /dev/null +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -0,0 +1,521 @@ +""" +.. module: lemur.plugins.lemur_acme.plugin + :platform: Unix + :synopsis: This module contains handlers for certain acme related tasks. It needed to be refactored to avoid circular imports + :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + + Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py + +.. moduleauthor:: Kevin Glisson +.. moduleauthor:: Mikhail Khodorovskiy +.. moduleauthor:: Curtis Castrapel +.. moduleauthor:: Mathias Petermann +""" +import datetime +import json +import time + +import OpenSSL.crypto +import josepy as jose +import dns.resolver +from acme import challenges, errors, messages +from acme.client import BackwardsCompatibleClientV2, ClientNetwork +from acme.errors import TimeoutError +from acme.messages import Error as AcmeError +from flask import current_app + +from lemur.common.utils import generate_private_key +from lemur.dns_providers import service as dns_provider_service +from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration +from lemur.extensions import metrics, sentry + +from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns +from lemur.authorities import service as authorities_service +from retrying import retry + + +class AuthorizationRecord(object): + def __init__(self, domain, target_domain, authz, dns_challenge, change_id): + self.domain = domain + self.target_domain = target_domain + self.authz = authz + self.dns_challenge = dns_challenge + self.change_id = change_id + + +class AcmeHandler(object): + + def reuse_account(self, authority): + if not authority.options: + raise InvalidAuthority("Invalid authority. Options not set") + existing_key = False + existing_regr = False + + for option in json.loads(authority.options): + if option["name"] == "acme_private_key" and option["value"]: + existing_key = True + if option["name"] == "acme_regr" and option["value"]: + existing_regr = True + + if not existing_key and current_app.config.get("ACME_PRIVATE_KEY"): + existing_key = True + + if not existing_regr and current_app.config.get("ACME_REGR"): + existing_regr = True + + if existing_key and existing_regr: + return True + else: + return False + + def strip_wildcard(self, host): + """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" + prefix = "*." + if host.startswith(prefix): + return host[len(prefix):], True + return host, False + + def maybe_add_extension(self, host, dns_provider_options): + if dns_provider_options and dns_provider_options.get( + "acme_challenge_extension" + ): + host = host + dns_provider_options.get("acme_challenge_extension") + return host + + def request_certificate(self, acme_client, authorizations, order): + for authorization in authorizations: + for authz in authorization.authz: + authorization_resource, _ = acme_client.poll(authz) + + deadline = datetime.datetime.now() + datetime.timedelta(seconds=360) + + try: + orderr = acme_client.poll_and_finalize(order, deadline) + + except (AcmeError, TimeoutError): + sentry.captureException(extra={"order_url": str(order.uri)}) + metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri}) + current_app.logger.error( + f"Unable to resolve Acme order: {order.uri}", exc_info=True + ) + raise + except errors.ValidationError: + if order.fullchain_pem: + orderr = order + else: + raise + + metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri}) + current_app.logger.info( + f"Successfully resolved Acme order: {order.uri}", exc_info=True + ) + + pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem) + + current_app.logger.debug( + "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) + ) + return pem_certificate, pem_certificate_chain + + def extract_cert_and_chain(self, fullchain_pem): + pem_certificate = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, fullchain_pem + ), + ).decode() + + if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ + and datetime.datetime.now() < datetime.datetime.strptime( + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") + else: + pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() + + return pem_certificate, pem_certificate_chain + + @retry(stop_max_attempt_number=5, wait_fixed=5000) + def setup_acme_client(self, authority): + if not authority.options: + raise InvalidAuthority("Invalid authority. Options not set") + options = {} + + for option in json.loads(authority.options): + options[option["name"]] = option.get("value") + email = options.get("email", current_app.config.get("ACME_EMAIL")) + tel = options.get("telephone", current_app.config.get("ACME_TEL")) + directory_url = options.get( + "acme_url", current_app.config.get("ACME_DIRECTORY_URL") + ) + + existing_key = options.get( + "acme_private_key", current_app.config.get("ACME_PRIVATE_KEY") + ) + existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR")) + + if existing_key and existing_regr: + current_app.logger.debug("Reusing existing ACME account") + # Reuse the same account for each certificate issuance + key = jose.JWK.json_loads(existing_key) + regr = messages.RegistrationResource.json_loads(existing_regr) + current_app.logger.debug( + "Connecting with directory at {0}".format(directory_url) + ) + net = ClientNetwork(key, account=regr) + client = BackwardsCompatibleClientV2(net, key, directory_url) + return client, {} + else: + # Create an account for each certificate issuance + key = jose.JWKRSA(key=generate_private_key("RSA2048")) + + current_app.logger.debug("Creating a new ACME account") + current_app.logger.debug( + "Connecting with directory at {0}".format(directory_url) + ) + + net = ClientNetwork(key, account=None, timeout=3600) + client = BackwardsCompatibleClientV2(net, key, directory_url) + registration = client.new_account_and_tos( + messages.NewRegistration.from_data(email=email) + ) + + # if store_account is checked, add the private_key and registration resources to the options + if options['store_account']: + new_options = json.loads(authority.options) + # the key returned by fields_to_partial_json is missing the key type, so we add it manually + key_dict = key.fields_to_partial_json() + key_dict["kty"] = "RSA" + acme_private_key = { + "name": "acme_private_key", + "value": json.dumps(key_dict) + } + new_options.append(acme_private_key) + + acme_regr = { + "name": "acme_regr", + "value": json.dumps({"body": {}, "uri": registration.uri}) + } + new_options.append(acme_regr) + + authorities_service.update_options(authority.id, options=json.dumps(new_options)) + + current_app.logger.debug("Connected: {0}".format(registration.uri)) + + return client, registration + + def get_domains(self, options): + """ + Fetches all domains currently requested + :param options: + :return: + """ + current_app.logger.debug("Fetching domains") + + domains = [options["common_name"]] + if options.get("extensions"): + for dns_name in options["extensions"]["sub_alt_names"]["names"]: + if dns_name.value not in domains: + domains.append(dns_name.value) + + current_app.logger.debug("Got these domains: {0}".format(domains)) + return domains + + def revoke_certificate(self, certificate): + if not self.reuse_account(certificate.authority): + raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") + acme_client, _ = self.acme.setup_acme_client(certificate.authority) + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, certificate.body)) + + try: + acme_client.revoke(fullchain_com, 0) # revocation reason = 0 + except (errors.ConflictError, errors.ClientError, errors.Error) as e: + # Certificate already revoked. + current_app.logger.error("Certificate revocation failed with message: " + e.detail) + metrics.send("acme_revoke_certificate_failure", "counter", 1) + return False + + current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) + metrics.send("acme_revoke_certificate_success", "counter", 1) + return True + + +class AcmeDnsHandler(AcmeHandler): + + def __init__(self): + self.dns_providers_for_domain = {} + try: + self.all_dns_providers = dns_provider_service.get_all_dns_providers() + except Exception as e: + metrics.send("AcmeHandler_init_error", "counter", 1) + sentry.captureException() + current_app.logger.error(f"Unable to fetch DNS Providers: {e}") + self.all_dns_providers = [] + + def get_all_zones(self, dns_provider): + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + return dns_provider_plugin.get_zones(account_number=account_number) + + def get_dns_challenges(self, host, authorizations): + """Get dns challenges for provided domain""" + + domain_to_validate, is_wildcard = self.strip_wildcard(host) + dns_challenges = [] + for authz in authorizations: + if not authz.body.identifier.value.lower() == domain_to_validate.lower(): + continue + if is_wildcard and not authz.body.wildcard: + continue + if not is_wildcard and authz.body.wildcard: + continue + for combo in authz.body.challenges: + if isinstance(combo.chall, challenges.DNS01): + dns_challenges.append(combo) + + return dns_challenges + + def get_dns_provider(self, type): + provider_types = { + "cloudflare": cloudflare, + "dyn": dyn, + "route53": route53, + "ultradns": ultradns, + "powerdns": powerdns + } + provider = provider_types.get(type) + if not provider: + raise UnknownProvider("No such DNS provider: {}".format(type)) + return provider + + def start_dns_challenge( + self, + acme_client, + account_number, + domain, + target_domain, + dns_provider, + order, + dns_provider_options, + ): + current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.") + + change_ids = [] + dns_challenges = self.get_dns_challenges(domain, order.authorizations) + host_to_validate, _ = self.strip_wildcard(target_domain) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) + + if not dns_challenges: + sentry.captureException() + metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) + raise Exception("Unable to determine DNS challenges from authorizations") + + for dns_challenge in dns_challenges: + + # Only prepend '_acme-challenge' if not using CNAME redirection + if domain == target_domain: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate) + + change_id = dns_provider.create_txt_record( + host_to_validate, + dns_challenge.validation(acme_client.client.net.key), + account_number, + ) + change_ids.append(change_id) + + return AuthorizationRecord( + domain, target_domain, order.authorizations, dns_challenges, change_ids + ) + + def complete_dns_challenge(self, acme_client, authz_record): + current_app.logger.debug( + "Finalizing DNS challenge for {0}".format( + authz_record.authz[0].body.identifier.value + ) + ) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) + if not dns_providers: + metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) + raise Exception( + "No DNS providers found for domain: {}".format(authz_record.target_domain) + ) + + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + for change_id in authz_record.change_id: + try: + dns_provider_plugin.wait_for_dns_change( + change_id, account_number=account_number + ) + except Exception: + metrics.send("complete_dns_challenge_error", "counter", 1) + sentry.captureException() + current_app.logger.debug( + f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " + f"{account_number}", + exc_info=True, + ) + raise + + for dns_challenge in authz_record.dns_challenge: + response = dns_challenge.response(acme_client.client.net.key) + + verified = response.simple_verify( + dns_challenge.chall, + authz_record.target_domain, + acme_client.client.net.key.public_key(), + ) + + if not verified: + metrics.send("complete_dns_challenge_verification_error", "counter", 1) + raise ValueError("Failed verification") + + time.sleep(5) + res = acme_client.answer_challenge(dns_challenge, response) + current_app.logger.debug(f"answer_challenge response: {res}") + + def get_authorizations(self, acme_client, order, order_info): + authorizations = [] + + for domain in order_info.domains: + + # If CNAME exists, set host to the target address + target_domain = domain + if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): + cname_result, _ = self.strip_wildcard(domain) + cname_result = challenges.DNS01().validation_domain_name(cname_result) + cname_result = self.get_cname(cname_result) + if cname_result: + target_domain = cname_result + self.autodetect_dns_providers(target_domain) + + if not self.dns_providers_for_domain.get(target_domain): + metrics.send( + "get_authorizations_no_dns_provider_for_domain", "counter", 1 + ) + raise Exception("No DNS providers found for domain: {}".format(target_domain)) + + for dns_provider in self.dns_providers_for_domain[target_domain]: + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + authz_record = self.start_dns_challenge( + acme_client, + account_number, + domain, + target_domain, + dns_provider_plugin, + order, + dns_provider.options, + ) + authorizations.append(authz_record) + return authorizations + + def autodetect_dns_providers(self, domain): + """ + Get DNS providers associated with a domain when it has not been provided for certificate creation. + :param domain: + :return: dns_providers: List of DNS providers that have the correct zone. + """ + self.dns_providers_for_domain[domain] = [] + match_length = 0 + for dns_provider in self.all_dns_providers: + if not dns_provider.domains: + continue + for name in dns_provider.domains: + if name == domain or domain.endswith("." + name): + if len(name) > match_length: + self.dns_providers_for_domain[domain] = [dns_provider] + match_length = len(name) + elif len(name) == match_length: + self.dns_providers_for_domain[domain].append(dns_provider) + + return self.dns_providers_for_domain + + def finalize_authorizations(self, acme_client, authorizations): + for authz_record in authorizations: + self.complete_dns_challenge(acme_client, authz_record) + for authz_record in authorizations: + dns_challenges = authz_record.dns_challenge + for dns_challenge in dns_challenges: + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_plugin = self.get_dns_provider( + dns_provider.provider_type + ) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) + if authz_record.domain == authz_record.target_domain: + host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate) + dns_provider_plugin.delete_txt_record( + authz_record.change_id, + account_number, + host_to_validate, + dns_challenge.validation(acme_client.client.net.key), + ) + + return authorizations + + def cleanup_dns_challenges(self, acme_client, authorizations): + """ + Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called + on an exception + + :param acme_client: + :param account_number: + :param dns_provider: + :param authorizations: + :param dns_provider_options: + :return: + """ + for authz_record in authorizations: + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_challenges = authz_record.dns_challenge + host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) + host_to_validate = self.maybe_add_extension( + host_to_validate, dns_provider_options + ) + + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + for dns_challenge in dns_challenges: + if authz_record.domain == authz_record.target_domain: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate) + try: + dns_provider_plugin.delete_txt_record( + authz_record.change_id, + account_number, + host_to_validate, + dns_challenge.validation(acme_client.client.net.key), + ) + except Exception as e: + # If this fails, it's most likely because the record doesn't exist (It was already cleaned up) + # or we're not authorized to modify it. + metrics.send("cleanup_dns_challenges_error", "counter", 1) + sentry.captureException() + pass + + def get_cname(self, domain): + """ + :param domain: Domain name to look up a CNAME for. + :return: First CNAME target or False if no CNAME record exists. + """ + try: + result = dns.resolver.query(domain, 'CNAME') + if len(result) > 0: + return str(result[0].target).rstrip('.') + except dns.exception.DNSException: + return False diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py new file mode 100644 index 00000000..538ec236 --- /dev/null +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -0,0 +1,260 @@ +""" +.. module: lemur.plugins.lemur_acme.plugin + :platform: Unix + :synopsis: This module contains the different challenge types for ACME implementations + :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Mathias Petermann +""" +import datetime +import json + +from acme import challenges +from acme.messages import errors, STATUS_VALID, ERROR_CODES +from flask import current_app + +from lemur.authorizations import service as authorization_service +from lemur.exceptions import LemurException, InvalidConfiguration +from lemur.plugins.base import plugins +from lemur.destinations import service as destination_service +from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler + + +class AcmeChallengeMissmatchError(LemurException): + pass + + +class AcmeChallenge(object): + """ + This is the base class, all ACME challenges will need to extend, allowing for future extendability + """ + + def create_certificate(self, csr, issuer_options): + """ + Create the new certificate, using the provided CSR and issuer_options. + Right now this is basically a copy of the create_certificate methods in the AcmeHandlers, but should be cleaned + and tried to make use of the deploy and cleanup methods + + :param csr: + :param issuer_options: + :return: + """ + pass + + def deploy(self, challenge, acme_client, validation_target): + """ + In here the challenge validation is fetched and deployed somewhere that it can be validated by the provider + + :param self: + :param challenge: the challenge object, must match for the challenge implementation + :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on + :param validation_target: an identifier for the validation target, e.g. the name of a DNS provider + """ + raise NotImplementedError + + def cleanup(self, challenge, acme_client, validation_target): + """ + Ideally the challenge should be cleaned up, after the validation is done + :param challenge: Needed to identify the challenge to be removed + :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on + :param validation_target: Needed to remove the validation + """ + raise NotImplementedError + + +class AcmeHttpChallenge(AcmeChallenge): + challengeType = challenges.HTTP01 + + def create_certificate(self, csr, issuer_options): + """ + Creates an ACME certificate using the HTTP-01 challenge. + + :param csr: + :param issuer_options: + :return: :raise Exception: + """ + self.acme = AcmeHandler() + authority = issuer_options.get("authority") + acme_client, registration = self.acme.setup_acme_client(authority) + + orderr = acme_client.new_order(csr) + + chall = [] + deployed_challenges = [] + all_pre_validated = True + for authz in orderr.authorizations: + # Choosing challenge. + # check if authorizations is already in a valid state + if authz.body.status != STATUS_VALID: + all_pre_validated = False + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. + if isinstance(i.chall, challenges.HTTP01): + chall.append(i) + else: + current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value)) + + if len(chall) == 0 and not all_pre_validated: + raise Exception('HTTP-01 challenge was not offered by the CA server at {}'.format(orderr.uri)) + elif not all_pre_validated: + validation_target = None + for option in json.loads(issuer_options["authority"].options): + if option["name"] == "tokenDestination": + validation_target = option["value"] + + if validation_target is None: + raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') + + for challenge in chall: + try: + response = self.deploy(challenge, acme_client, validation_target) + deployed_challenges.append(challenge.chall.path) + acme_client.answer_challenge(challenge, response) + except Exception as e: + current_app.logger.error(e) + raise Exception('Failure while trying to deploy token to configure destination. See logs for more information') + + current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") + + try: + finalized_orderr = acme_client.poll_and_finalize(orderr, + datetime.datetime.now() + datetime.timedelta(seconds=90)) + except errors.ValidationError as validationError: + for authz in validationError.failed_authzrs: + for chall in authz.body.challenges: + if chall.error: + current_app.logger.error( + "ValidationError occured of type {}, with message {}".format(chall.error.typ, + ERROR_CODES[chall.error.code])) + raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.') + + pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem) + + if len(deployed_challenges) != 0: + for token_path in deployed_challenges: + self.cleanup(token_path, validation_target) + + # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate + return pem_certificate, pem_certificate_chain, None + + def deploy(self, challenge, acme_client, validation_target): + + if not isinstance(challenge.chall, challenges.HTTP01): + raise AcmeChallengeMissmatchError( + 'The provided challenge is not of type HTTP01, but instead of type {}'.format( + challenge.__class__.__name__)) + + destination = destination_service.get(validation_target) + + if destination is None: + raise Exception( + 'Couldn\'t find the destination with name {}. Cant complete HTTP01 challenge'.format(validation_target)) + + destination_plugin = plugins.get(destination.plugin_name) + + response, validation = challenge.response_and_validation(acme_client.net.key) + + destination_plugin.upload_acme_token(challenge.chall.path, validation, destination.options) + current_app.logger.info("Uploaded HTTP-01 challenge token.") + + return response + + def cleanup(self, token_path, validation_target): + destination = destination_service.get(validation_target) + + if destination is None: + current_app.logger.info( + 'Couldn\'t find the destination with name {}, won\'t cleanup the challenge'.format(validation_target)) + + destination_plugin = plugins.get(destination.plugin_name) + + destination_plugin.delete_acme_token(token_path, destination.options) + current_app.logger.info("Cleaned up HTTP-01 challenge token.") + + +class AcmeDnsChallenge(AcmeChallenge): + challengeType = challenges.DNS01 + + def create_certificate(self, csr, issuer_options): + """ + Creates an ACME certificate. + + :param csr: + :param issuer_options: + :return: :raise Exception: + """ + self.acme = AcmeDnsHandler() + authority = issuer_options.get("authority") + create_immediately = issuer_options.get("create_immediately", False) + acme_client, registration = self.acme.setup_acme_client(authority) + dns_provider = issuer_options.get("dns_provider", {}) + + if dns_provider: + dns_provider_options = dns_provider.options + credentials = json.loads(dns_provider.credentials) + current_app.logger.debug( + "Using DNS provider: {0}".format(dns_provider.provider_type) + ) + dns_provider_plugin = __import__( + dns_provider.provider_type, globals(), locals(), [], 1 + ) + account_number = credentials.get("account_id") + provider_type = dns_provider.provider_type + if provider_type == "route53" and not account_number: + error = "Route53 DNS Provider {} does not have an account number configured.".format( + dns_provider.name + ) + current_app.logger.error(error) + raise InvalidConfiguration(error) + else: + dns_provider = {} + dns_provider_options = None + account_number = None + provider_type = None + + domains = self.acme.get_domains(issuer_options) + if not create_immediately: + # Create pending authorizations that we'll need to do the creation + dns_authorization = authorization_service.create( + account_number, domains, provider_type + ) + # Return id of the DNS Authorization + return None, None, dns_authorization.id + + authorizations = self.acme.get_authorizations( + acme_client, + account_number, + domains, + dns_provider_plugin, + dns_provider_options, + ) + self.acme.finalize_authorizations( + acme_client, + account_number, + dns_provider_plugin, + authorizations, + dns_provider_options, + ) + pem_certificate, pem_certificate_chain = self.acme.request_certificate( + acme_client, authorizations, csr + ) + # TODO add external ID (if possible) + return pem_certificate, pem_certificate_chain, None + + def deploy(self, challenge, acme_client, validation_target): + pass + + def cleanup(self, authorizations, acme_client, validation_target): + """ + Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called + on an exception + + :param authorizations: all the authorizations to be cleaned up + :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on + :param validation_target: Unused right now + :return: + """ + acme = AcmeDnsHandler() + acme.cleanup_dns_challenges(acme_client, authorizations) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 8bc1485f..4763a2fa 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -11,432 +11,27 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Curtis Castrapel """ -import datetime -import json -import time -import OpenSSL.crypto -import josepy as jose -from acme import challenges, errors, messages -from acme.client import BackwardsCompatibleClientV2, ClientNetwork -from acme.errors import PollError, TimeoutError, WildcardUnsupportedError +from acme.errors import PollError, WildcardUnsupportedError from acme.messages import Error as AcmeError from botocore.exceptions import ClientError from flask import current_app - from lemur.authorizations import service as authorization_service -from lemur.common.utils import generate_private_key from lemur.dns_providers import service as dns_provider_service -from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider +from lemur.exceptions import InvalidConfiguration from lemur.extensions import metrics, sentry + from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin -from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns -from lemur.authorities import service as authorities_service -from retrying import retry - - -class AuthorizationRecord(object): - def __init__(self, host, authz, dns_challenge, change_id): - self.host = host - self.authz = authz - self.dns_challenge = dns_challenge - self.change_id = change_id - - -class AcmeHandler(object): - def __init__(self): - self.dns_providers_for_domain = {} - try: - self.all_dns_providers = dns_provider_service.get_all_dns_providers() - except Exception as e: - metrics.send("AcmeHandler_init_error", "counter", 1) - sentry.captureException() - current_app.logger.error(f"Unable to fetch DNS Providers: {e}") - self.all_dns_providers = [] - - def get_dns_challenges(self, host, authorizations): - """Get dns challenges for provided domain""" - - domain_to_validate, is_wildcard = self.strip_wildcard(host) - dns_challenges = [] - for authz in authorizations: - if not authz.body.identifier.value.lower() == domain_to_validate.lower(): - continue - if is_wildcard and not authz.body.wildcard: - continue - if not is_wildcard and authz.body.wildcard: - continue - for combo in authz.body.challenges: - if isinstance(combo.chall, challenges.DNS01): - dns_challenges.append(combo) - - return dns_challenges - - def strip_wildcard(self, host): - """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" - prefix = "*." - if host.startswith(prefix): - return host[len(prefix):], True - return host, False - - def maybe_add_extension(self, host, dns_provider_options): - if dns_provider_options and dns_provider_options.get( - "acme_challenge_extension" - ): - host = host + dns_provider_options.get("acme_challenge_extension") - return host - - def start_dns_challenge( - self, - acme_client, - account_number, - host, - dns_provider, - order, - dns_provider_options, - ): - current_app.logger.debug("Starting DNS challenge for {0}".format(host)) - - change_ids = [] - dns_challenges = self.get_dns_challenges(host, order.authorizations) - host_to_validate, _ = self.strip_wildcard(host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) - - if not dns_challenges: - sentry.captureException() - metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) - raise Exception("Unable to determine DNS challenges from authorizations") - - for dns_challenge in dns_challenges: - change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key), - account_number, - ) - change_ids.append(change_id) - - return AuthorizationRecord( - host, order.authorizations, dns_challenges, change_ids - ) - - def complete_dns_challenge(self, acme_client, authz_record): - current_app.logger.debug( - "Finalizing DNS challenge for {0}".format( - authz_record.authz[0].body.identifier.value - ) - ) - dns_providers = self.dns_providers_for_domain.get(authz_record.host) - if not dns_providers: - metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) - raise Exception( - "No DNS providers found for domain: {}".format(authz_record.host) - ) - - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - for change_id in authz_record.change_id: - try: - dns_provider_plugin.wait_for_dns_change( - change_id, account_number=account_number - ) - except Exception: - metrics.send("complete_dns_challenge_error", "counter", 1) - sentry.captureException() - current_app.logger.debug( - f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " - f"{account_number}", - exc_info=True, - ) - raise - - for dns_challenge in authz_record.dns_challenge: - response = dns_challenge.response(acme_client.client.net.key) - - verified = response.simple_verify( - dns_challenge.chall, - authz_record.host, - acme_client.client.net.key.public_key(), - ) - - if not verified: - metrics.send("complete_dns_challenge_verification_error", "counter", 1) - raise ValueError("Failed verification") - - time.sleep(5) - res = acme_client.answer_challenge(dns_challenge, response) - current_app.logger.debug(f"answer_challenge response: {res}") - - def request_certificate(self, acme_client, authorizations, order): - for authorization in authorizations: - for authz in authorization.authz: - authorization_resource, _ = acme_client.poll(authz) - - deadline = datetime.datetime.now() + datetime.timedelta(seconds=360) - - try: - orderr = acme_client.poll_and_finalize(order, deadline) - - except (AcmeError, TimeoutError): - sentry.captureException(extra={"order_url": str(order.uri)}) - metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri}) - current_app.logger.error( - f"Unable to resolve Acme order: {order.uri}", exc_info=True - ) - raise - except errors.ValidationError: - if order.fullchain_pem: - orderr = order - else: - raise - - metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri}) - current_app.logger.info( - f"Successfully resolved Acme order: {order.uri}", exc_info=True - ) - - pem_certificate = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem - ), - ).decode() - - if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ - and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): - pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") - else: - pem_certificate_chain = orderr.fullchain_pem[ - len(pem_certificate) : # noqa - ].lstrip() - - current_app.logger.debug( - "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) - ) - return pem_certificate, pem_certificate_chain - - @retry(stop_max_attempt_number=5, wait_fixed=5000) - def setup_acme_client(self, authority): - if not authority.options: - raise InvalidAuthority("Invalid authority. Options not set") - options = {} - - for option in json.loads(authority.options): - options[option["name"]] = option.get("value") - email = options.get("email", current_app.config.get("ACME_EMAIL")) - tel = options.get("telephone", current_app.config.get("ACME_TEL")) - directory_url = options.get( - "acme_url", current_app.config.get("ACME_DIRECTORY_URL") - ) - - existing_key = options.get( - "acme_private_key", current_app.config.get("ACME_PRIVATE_KEY") - ) - existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR")) - - if existing_key and existing_regr: - current_app.logger.debug("Reusing existing ACME account") - # Reuse the same account for each certificate issuance - key = jose.JWK.json_loads(existing_key) - regr = messages.RegistrationResource.json_loads(existing_regr) - current_app.logger.debug( - "Connecting with directory at {0}".format(directory_url) - ) - net = ClientNetwork(key, account=regr) - client = BackwardsCompatibleClientV2(net, key, directory_url) - return client, {} - else: - # Create an account for each certificate issuance - key = jose.JWKRSA(key=generate_private_key("RSA2048")) - - current_app.logger.debug("Creating a new ACME account") - current_app.logger.debug( - "Connecting with directory at {0}".format(directory_url) - ) - - net = ClientNetwork(key, account=None, timeout=3600) - client = BackwardsCompatibleClientV2(net, key, directory_url) - registration = client.new_account_and_tos( - messages.NewRegistration.from_data(email=email) - ) - - # if store_account is checked, add the private_key and registration resources to the options - if options['store_account']: - new_options = json.loads(authority.options) - # the key returned by fields_to_partial_json is missing the key type, so we add it manually - key_dict = key.fields_to_partial_json() - key_dict["kty"] = "RSA" - acme_private_key = { - "name": "acme_private_key", - "value": json.dumps(key_dict) - } - new_options.append(acme_private_key) - - acme_regr = { - "name": "acme_regr", - "value": json.dumps({"body": {}, "uri": registration.uri}) - } - new_options.append(acme_regr) - - authorities_service.update_options(authority.id, options=json.dumps(new_options)) - - current_app.logger.debug("Connected: {0}".format(registration.uri)) - - return client, registration - - def get_domains(self, options): - """ - Fetches all domains currently requested - :param options: - :return: - """ - current_app.logger.debug("Fetching domains") - - domains = [options["common_name"]] - if options.get("extensions"): - for dns_name in options["extensions"]["sub_alt_names"]["names"]: - if dns_name.value not in domains: - domains.append(dns_name.value) - - current_app.logger.debug("Got these domains: {0}".format(domains)) - return domains - - def get_authorizations(self, acme_client, order, order_info): - authorizations = [] - - for domain in order_info.domains: - if not self.dns_providers_for_domain.get(domain): - metrics.send( - "get_authorizations_no_dns_provider_for_domain", "counter", 1 - ) - raise Exception("No DNS providers found for domain: {}".format(domain)) - for dns_provider in self.dns_providers_for_domain[domain]: - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - authz_record = self.start_dns_challenge( - acme_client, - account_number, - domain, - dns_provider_plugin, - order, - dns_provider.options, - ) - authorizations.append(authz_record) - return authorizations - - def autodetect_dns_providers(self, domain): - """ - Get DNS providers associated with a domain when it has not been provided for certificate creation. - :param domain: - :return: dns_providers: List of DNS providers that have the correct zone. - """ - self.dns_providers_for_domain[domain] = [] - match_length = 0 - for dns_provider in self.all_dns_providers: - if not dns_provider.domains: - continue - for name in dns_provider.domains: - if name == domain or domain.endswith("." + name): - if len(name) > match_length: - self.dns_providers_for_domain[domain] = [dns_provider] - match_length = len(name) - elif len(name) == match_length: - self.dns_providers_for_domain[domain].append(dns_provider) - - return self.dns_providers_for_domain - - def finalize_authorizations(self, acme_client, authorizations): - for authz_record in authorizations: - self.complete_dns_challenge(acme_client, authz_record) - for authz_record in authorizations: - dns_challenges = authz_record.dns_challenge - for dns_challenge in dns_challenges: - dns_providers = self.dns_providers_for_domain.get(authz_record.host) - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_plugin = self.get_dns_provider( - dns_provider.provider_type - ) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - host_to_validate, _ = self.strip_wildcard(authz_record.host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) - dns_provider_plugin.delete_txt_record( - authz_record.change_id, - account_number, - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key), - ) - - return authorizations - - def cleanup_dns_challenges(self, acme_client, authorizations): - """ - Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called - on an exception - - :param acme_client: - :param account_number: - :param dns_provider: - :param authorizations: - :param dns_provider_options: - :return: - """ - for authz_record in authorizations: - dns_providers = self.dns_providers_for_domain.get(authz_record.host) - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_challenges = authz_record.dns_challenge - host_to_validate, _ = self.strip_wildcard(authz_record.host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - for dns_challenge in dns_challenges: - try: - dns_provider_plugin.delete_txt_record( - authz_record.change_id, - account_number, - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key), - ) - except Exception as e: - # If this fails, it's most likely because the record doesn't exist (It was already cleaned up) - # or we're not authorized to modify it. - metrics.send("cleanup_dns_challenges_error", "counter", 1) - sentry.captureException() - pass - - def get_dns_provider(self, type): - provider_types = { - "cloudflare": cloudflare, - "dyn": dyn, - "route53": route53, - "ultradns": ultradns, - "powerdns": powerdns - } - provider = provider_types.get(type) - if not provider: - raise UnknownProvider("No such DNS provider: {}".format(type)) - return provider +from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler +from lemur.plugins.lemur_acme.challenge_types import AcmeHttpChallenge, AcmeDnsChallenge class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" slug = "acme-issuer" description = ( - "Enables the creation of certificates via ACME CAs (including Let's Encrypt)" + "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the DNS-01 challenge" ) version = acme.VERSION @@ -448,7 +43,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "name": "acme_url", "type": "str", "required": True, - "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", "helpMessage": "Must be a valid web url starting with http[s]://", }, { @@ -461,7 +56,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "name": "email", "type": "str", "default": "", - "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", "helpMessage": "Email to use", }, { @@ -483,30 +78,8 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) - def get_dns_provider(self, type): - self.acme = AcmeHandler() - - provider_types = { - "cloudflare": cloudflare, - "dyn": dyn, - "route53": route53, - "ultradns": ultradns, - "powerdns": powerdns - } - provider = provider_types.get(type) - if not provider: - raise UnknownProvider("No such DNS provider: {}".format(type)) - return provider - - def get_all_zones(self, dns_provider): - self.acme = AcmeHandler() - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - return dns_provider_plugin.get_zones(account_number=account_number) - def get_ordered_certificate(self, pending_cert): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -552,7 +125,8 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() + self.acme_dns_challenge = AcmeDnsChallenge() pending = [] certs = [] for pending_cert in pending_certs: @@ -649,76 +223,22 @@ class ACMEIssuerPlugin(IssuerPlugin): } ) # Ensure DNS records get deleted - self.acme.cleanup_dns_challenges( - entry["acme_client"], entry["authorizations"] + self.acme_dns_challenge.cleanup( + entry["authorizations"], entry["acme_client"] ) return certs def create_certificate(self, csr, issuer_options): """ - Creates an ACME certificate. + Creates an ACME certificate using the DNS-01 challenge. :param csr: :param issuer_options: :return: :raise Exception: """ - self.acme = AcmeHandler() - authority = issuer_options.get("authority") - create_immediately = issuer_options.get("create_immediately", False) - acme_client, registration = self.acme.setup_acme_client(authority) - dns_provider = issuer_options.get("dns_provider", {}) + acme_dns_challenge = AcmeDnsChallenge() - if dns_provider: - dns_provider_options = dns_provider.options - credentials = json.loads(dns_provider.credentials) - current_app.logger.debug( - "Using DNS provider: {0}".format(dns_provider.provider_type) - ) - dns_provider_plugin = __import__( - dns_provider.provider_type, globals(), locals(), [], 1 - ) - account_number = credentials.get("account_id") - provider_type = dns_provider.provider_type - if provider_type == "route53" and not account_number: - error = "Route53 DNS Provider {} does not have an account number configured.".format( - dns_provider.name - ) - current_app.logger.error(error) - raise InvalidConfiguration(error) - else: - dns_provider = {} - dns_provider_options = None - account_number = None - provider_type = None - - domains = self.acme.get_domains(issuer_options) - if not create_immediately: - # Create pending authorizations that we'll need to do the creation - dns_authorization = authorization_service.create( - account_number, domains, provider_type - ) - # Return id of the DNS Authorization - return None, None, dns_authorization.id - - authorizations = self.acme.get_authorizations( - acme_client, - account_number, - domains, - dns_provider_plugin, - dns_provider_options, - ) - self.acme.finalize_authorizations( - acme_client, - account_number, - dns_provider_plugin, - authorizations, - dns_provider_options, - ) - pem_certificate, pem_certificate_chain = self.acme.request_certificate( - acme_client, authorizations, csr - ) - # TODO add external ID (if possible) - return pem_certificate, pem_certificate_chain, None + return acme_dns_challenge.create_certificate(csr, issuer_options) @staticmethod def create_authority(options): @@ -746,3 +266,108 @@ class ACMEIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. pass + + def revoke_certificate(self, certificate, comments): + self.acme = AcmeDnsHandler() + return self.acme.revoke_certificate(certificate) + + +class ACMEHttpIssuerPlugin(IssuerPlugin): + title = "Acme HTTP-01" + slug = "acme-http-issuer" + description = ( + "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the HTTP-01 challenge" + ) + version = acme.VERSION + + author = "Netflix" + author_url = "https://github.com/netflix/lemur.git" + + options = [ + { + "name": "acme_url", + "type": "str", + "required": True, + "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "helpMessage": "Must be a valid web url starting with http[s]://", + }, + { + "name": "telephone", + "type": "str", + "default": "", + "helpMessage": "Telephone to use", + }, + { + "name": "email", + "type": "str", + "default": "", + "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "helpMessage": "Email to use", + }, + { + "name": "certificate", + "type": "textarea", + "default": "", + "validation": "/^-----BEGIN CERTIFICATE-----/", + "helpMessage": "Certificate to use", + }, + { + "name": "store_account", + "type": "bool", + "required": False, + "helpMessage": "Disable to create a new account for each ACME request", + "default": False, + }, + { + "name": "tokenDestination", + "type": "destinationSelect", + "required": True, + "helpMessage": "The destination to use to deploy the token.", + }, + ] + + def __init__(self, *args, **kwargs): + super(ACMEHttpIssuerPlugin, self).__init__(*args, **kwargs) + + def create_certificate(self, csr, issuer_options): + """ + Creates an ACME certificate using the HTTP-01 challenge. + + :param csr: + :param issuer_options: + :return: :raise Exception: + """ + acme_http_challenge = AcmeHttpChallenge() + + return acme_http_challenge.create_certificate(csr, issuer_options) + + @staticmethod + def create_authority(options): + """ + Creates an authority, this authority is then used by Lemur to allow a user + to specify which Certificate Authority they want to sign their certificate. + + :param options: + :return: + """ + role = {"username": "", "password": "", "name": "acme"} + plugin_options = options.get("plugin", {}).get("plugin_options") + if not plugin_options: + error = "Invalid options for lemur_acme plugin: {}".format(options) + current_app.logger.error(error) + raise InvalidConfiguration(error) + # Define static acme_root based off configuration variable by default. However, if user has passed a + # certificate, use this certificate as the root. + acme_root = current_app.config.get("ACME_ROOT") + for option in plugin_options: + if option.get("name") == "certificate": + acme_root = option.get("value") + return acme_root, "", [role] + + def cancel_ordered_certificate(self, pending_cert, **kwargs): + # Needed to override issuer function. + pass + + def revoke_certificate(self, certificate, comments): + self.acme = AcmeHandler() + return self.acme.revoke_certificate(certificate) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py similarity index 71% rename from lemur/plugins/lemur_acme/tests/test_acme.py rename to lemur/plugins/lemur_acme/tests/test_acme_dns.py index ab246563..f0e7dbfa 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -3,16 +3,18 @@ from unittest.mock import patch, Mock import josepy as jose from cryptography.x509 import DNSName +from flask import Flask from lemur.plugins.lemur_acme import plugin +from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord from lemur.common.utils import generate_private_key from mock import MagicMock -class TestAcme(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") +class TestAcmeDns(unittest.TestCase): + @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") def setUp(self, mock_dns_provider_service): self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeHandler() + self.acme = plugin.AcmeDnsHandler() mock_dns_provider = Mock() mock_dns_provider.name = "cloudflare" mock_dns_provider.credentials = "{}" @@ -22,6 +24,16 @@ class TestAcme(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) def test_get_dns_challenges(self, mock_len): assert mock_len @@ -39,36 +51,19 @@ class TestAcme(unittest.TestCase): result = yield self.acme.get_dns_challenges(host, mock_authz) self.assertEqual(result, mock_entry) - def test_strip_wildcard(self): - expected = ("example.com", False) - result = self.acme.strip_wildcard("example.com") - self.assertEqual(expected, result) - - expected = ("example.com", True) - result = self.acme.strip_wildcard("*.example.com") - self.assertEqual(expected, result) - - def test_authz_record(self): - a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") - self.assertEqual(type(a), plugin.AuthorizationRecord) - @patch("acme.client.Client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_get_dns_challenges, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_acme ): assert mock_len mock_order = Mock() - mock_app.logger.debug = Mock() mock_authz = Mock() mock_authz.body.resolved_combinations = [] mock_entry = MagicMock() - from acme import challenges - c = challenges.DNS01() - mock_entry.chall = TestAcme.test_complete_dns_challenge_fail + mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail mock_authz.body.resolved_combinations.append(mock_entry) mock_acme.request_domain_challenges = Mock(return_value=mock_authz) mock_dns_provider = Mock() @@ -79,16 +74,15 @@ class TestAcme(unittest.TestCase): iterator = iter(values) iterable.__iter__.return_value = iterator result = self.acme.start_dns_challenge( - mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} + mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {} ) - self.assertEqual(type(result), plugin.AuthorizationRecord) + self.assertEqual(type(result), AuthorizationRecord) @patch("acme.client.Client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") @patch("time.sleep") def test_complete_dns_challenge_success( - self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_sleep, mock_wait_for_dns_change, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -97,7 +91,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True) mock_authz.authz = [] - mock_authz.host = "www.test.com" + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) @@ -109,39 +103,38 @@ class TestAcme(unittest.TestCase): self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) + mock_dns_challenge = Mock() + response = Mock() + response.simple_verify = Mock(return_value=False) + mock_dns_challenge.response = Mock(return_value=response) + mock_authz = Mock() - mock_authz.dns_challenge.response = Mock() - mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False) - mock_authz.authz = [] - mock_authz.host = "www.test.com" + mock_authz.dns_challenge = [] + mock_authz.dns_challenge.append(mock_dns_challenge) + + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" + mock_authz.authz = [] mock_authz.authz.append(mock_authz_record) mock_authz.change_id = [] mock_authz.change_id.append("123") - mock_authz.dns_challenge = [] - dns_challenge = Mock() - mock_authz.dns_challenge.append(dns_challenge) - self.assertRaises( - ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz) - ) + with self.assertRaises(ValueError): + self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") - @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") def test_request_certificate( self, - mock_current_app, mock_get_dns_challenges, mock_jose, mock_crypto, @@ -158,7 +151,6 @@ class TestAcme(unittest.TestCase): mock_acme.fetch_chain = Mock(return_value="mock_chain") mock_crypto.dump_certificate = Mock(return_value=b"chain") mock_order = Mock() - mock_current_app.config = {} self.acme.request_certificate(mock_acme, [], mock_order) def test_setup_acme_client_fail(self): @@ -167,10 +159,9 @@ class TestAcme(unittest.TestCase): with self.assertRaises(Exception): self.acme.setup_acme_client(mock_authority) - @patch("lemur.plugins.lemur_acme.plugin.jose.JWK.json_loads") - @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_setup_acme_client_success_load_account_from_authority(self, mock_current_app, mock_acme, mock_key_json_load): + @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads") + @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") + def test_setup_acme_client_success_load_account_from_authority(self, mock_acme, mock_key_json_load): mock_authority = Mock() mock_authority.id = 2 mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ @@ -179,7 +170,6 @@ class TestAcme(unittest.TestCase): '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]' mock_client = Mock() mock_acme.return_value = mock_client - mock_current_app.config = {} mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048")) @@ -189,11 +179,10 @@ class TestAcme(unittest.TestCase): assert result_client assert not result_registration - @patch("lemur.plugins.lemur_acme.plugin.jose.JWKRSA.fields_to_partial_json") - @patch("lemur.plugins.lemur_acme.plugin.authorities_service") - @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_setup_acme_client_success_store_new_account(self, mock_current_app, mock_acme, mock_authorities_service, + @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWKRSA.fields_to_partial_json") + @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") + @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") + def test_setup_acme_client_success_store_new_account(self, mock_acme, mock_authorities_service, mock_key_generation): mock_authority = Mock() mock_authority.id = 2 @@ -206,7 +195,6 @@ class TestAcme(unittest.TestCase): mock_client.agree_to_tos = Mock(return_value=True) mock_client.new_account_and_tos.return_value = mock_registration mock_acme.return_value = mock_client - mock_current_app.config = {} mock_key_generation.return_value = {"n": "PwIOkViO"} @@ -219,10 +207,9 @@ class TestAcme(unittest.TestCase): '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]') - @patch("lemur.plugins.lemur_acme.plugin.authorities_service") - @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_setup_acme_client_success(self, mock_current_app, mock_acme, mock_authorities_service): + @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") + @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") + def test_setup_acme_client_success(self, mock_acme, mock_authorities_service): mock_authority = Mock() mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ '{"name": "store_account", "value": false}]' @@ -232,20 +219,17 @@ class TestAcme(unittest.TestCase): mock_client.register = mock_registration mock_client.agree_to_tos = Mock(return_value=True) mock_acme.return_value = mock_client - mock_current_app.config = {} result_client, result_registration = self.acme.setup_acme_client(mock_authority) mock_authorities_service.update_options.assert_not_called() assert result_client assert result_registration - @patch('lemur.plugins.lemur_acme.plugin.current_app') - def test_get_domains_single(self, mock_current_app): + def test_get_domains_single(self): options = {"common_name": "test.netflix.net"} result = self.acme.get_domains(options) self.assertEqual(result, [options["common_name"]]) - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_get_domains_multiple(self, mock_current_app): + def test_get_domains_multiple(self): options = { "common_name": "test.netflix.net", "extensions": { @@ -257,8 +241,7 @@ class TestAcme(unittest.TestCase): result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] ) - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_get_domains_san(self, mock_current_app): + def test_get_domains_san(self): options = { "common_name": "test.netflix.net", "extensions": { @@ -270,10 +253,62 @@ class TestAcme(unittest.TestCase): result, [options["common_name"], "test2.netflix.net"] ) - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", - return_value="test", - ) + def test_create_authority(self): + options = { + "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} + } + acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options) + self.assertEqual(acme_root, "123") + self.assertEqual(b, "") + self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}]) + + @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") + def test_get_dns_provider(self, mock_dns_provider_service): + provider = plugin.AcmeDnsHandler() + route53 = provider.get_dns_provider("route53") + assert route53 + cloudflare = provider.get_dns_provider("cloudflare") + assert cloudflare + dyn = provider.get_dns_provider("dyn") + assert dyn + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.challenge_types.authorization_service") + def test_create_certificate( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_acme, + ): + provider = plugin.ACMEIssuerPlugin() + mock_authority = Mock() + + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + + mock_dns_provider = Mock() + mock_dns_provider.credentials = '{"account_id": 1}' + mock_dns_provider.provider_type = "route53" + mock_dns_provider_service.get.return_value = mock_dns_provider + + issuer_options = { + "authority": mock_authority, + "dns_provider": mock_dns_provider, + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + result = provider.create_certificate(csr, issuer_options) + assert result + + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", return_value="test") def test_get_authorizations(self, mock_start_dns_challenge): mock_order = Mock() mock_order.body.identifiers = [] @@ -288,7 +323,7 @@ class TestAcme(unittest.TestCase): self.assertEqual(result, ["test"]) @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge", + "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.complete_dns_challenge", return_value="test", ) def test_finalize_authorizations(self, mock_complete_dns_challenge): @@ -306,51 +341,21 @@ class TestAcme(unittest.TestCase): result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) self.assertEqual(result, mock_authz) - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_create_authority(self, mock_current_app): - mock_current_app.config = Mock() - options = { - "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} - } - acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options) - self.assertEqual(acme_root, "123") - self.assertEqual(b, "") - self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}]) - - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.dyn.current_app") - @patch("lemur.plugins.lemur_acme.cloudflare.current_app") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - def test_get_dns_provider( - self, - mock_dns_provider_service, - mock_current_app_cloudflare, - mock_current_app_dyn, - mock_current_app, - ): - provider = plugin.ACMEIssuerPlugin() - route53 = provider.get_dns_provider("route53") - assert route53 - cloudflare = provider.get_dns_provider("cloudflare") - assert cloudflare - dyn = provider.get_dns_provider("dyn") - assert dyn - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") def test_get_ordered_certificate( self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations, + mock_dns_provider_service_p, mock_dns_provider_service, mock_authorization_service, - mock_current_app, mock_acme, ): mock_client = Mock() @@ -368,20 +373,20 @@ class TestAcme(unittest.TestCase): ) @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") def test_get_ordered_certificates( self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations, mock_dns_provider_service, + mock_dns_provider_service_p, mock_authorization_service, - mock_current_app, mock_acme, ): mock_client = Mock() @@ -406,41 +411,3 @@ class TestAcme(unittest.TestCase): result[1]["cert"], {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, ) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - def test_create_certificate( - self, - mock_authorization_service, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_current_app, - mock_dns_provider_service, - mock_acme, - ): - provider = plugin.ACMEIssuerPlugin() - mock_authority = Mock() - - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - - mock_dns_provider = Mock() - mock_dns_provider.credentials = '{"account_id": 1}' - mock_dns_provider.provider_type = "route53" - mock_dns_provider_service.get.return_value = mock_dns_provider - - issuer_options = { - "authority": mock_authority, - "dns_provider": mock_dns_provider, - "common_name": "test.netflix.net", - } - csr = "123" - mock_request_certificate.return_value = ("pem_certificate", "chain") - result = provider.create_certificate(csr, issuer_options) - assert result diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py new file mode 100644 index 00000000..cfc18c83 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -0,0 +1,112 @@ +import unittest +from unittest.mock import patch, Mock + +from flask import Flask +from cryptography.x509 import DNSName +from lemur.plugins.lemur_acme import acme_handlers + + +class TestAcmeHandler(unittest.TestCase): + def setUp(self): + self.acme = acme_handlers.AcmeHandler() + + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + + def test_strip_wildcard(self): + expected = ("example.com", False) + result = self.acme.strip_wildcard("example.com") + self.assertEqual(expected, result) + + expected = ("example.com", True) + result = self.acme.strip_wildcard("*.example.com") + self.assertEqual(expected, result) + + def test_authz_record(self): + a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id") + self.assertEqual(type(a), acme_handlers.AuthorizationRecord) + + def test_setup_acme_client_fail(self): + mock_authority = Mock() + mock_authority.options = [] + with self.assertRaises(Exception): + self.acme.setup_acme_client(mock_authority) + + def test_reuse_account_not_defined(self): + mock_authority = Mock() + mock_authority.options = [] + with self.assertRaises(Exception): + self.acme.reuse_account(mock_authority) + + def test_reuse_account_from_authority(self): + mock_authority = Mock() + mock_authority.options = '[{"name": "acme_private_key", "value": "PRIVATE_KEY"}, {"name": "acme_regr", "value": "ACME_REGR"}]' + + self.assertTrue(self.acme.reuse_account(mock_authority)) + + @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") + def test_reuse_account_from_config(self, mock_current_app): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_current_app.config = {"ACME_PRIVATE_KEY": "PRIVATE_KEY", "ACME_REGR": "ACME_REGR"} + + self.assertTrue(self.acme.reuse_account(mock_authority)) + + def test_reuse_account_no_configuration(self): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + + self.assertFalse(self.acme.reuse_account(mock_authority)) + + @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") + @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") + def test_setup_acme_client_success(self, mock_acme, mock_authorities_service): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ + '{"name": "store_account", "value": false}]' + mock_client = Mock() + mock_registration = Mock() + mock_registration.uri = "http://test.com" + mock_client.register = mock_registration + mock_client.agree_to_tos = Mock(return_value=True) + mock_acme.return_value = mock_client + result_client, result_registration = self.acme.setup_acme_client(mock_authority) + mock_authorities_service.update_options.assert_not_called() + assert result_client + assert result_registration + + def test_get_domains_single(self): + options = {"common_name": "test.netflix.net"} + result = self.acme.get_domains(options) + self.assertEqual(result, [options["common_name"]]) + + def test_get_domains_multiple(self): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] + ) + + def test_get_domains_san(self): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net"] + ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py new file mode 100644 index 00000000..5a546165 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -0,0 +1,171 @@ +import unittest +from unittest.mock import patch, Mock + +from flask import Flask +from acme import challenges +from lemur.plugins.lemur_acme import plugin + + +class TestAcmeHttp(unittest.TestCase): + + def setUp(self): + self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() + self.acme = plugin.AcmeHandler() + + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + + def test_create_authority(self): + options = { + "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} + } + acme_root, b, role = self.ACMEHttpIssuerPlugin.create_authority(options) + self.assertEqual(acme_root, "123") + self.assertEqual(b, "") + self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}]) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate( + self, + mock_authorization_service, + mock_request_certificate, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (Mock(), "Anything-goes") + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_client.answer_challenge.return_value = True + + mock_finalized_order = Mock() + mock_finalized_order.fullchain_pem = "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n" + mock_client.poll_and_finalize.return_value = mock_finalized_order + + mock_acme.return_value = (mock_client, "") + + mock_destination = Mock() + mock_destination.label = "mock-sftp-destination" + mock_destination.plugin_name = "SFTPDestinationPlugin" + mock_destination_service.get.return_value = mock_destination + + mock_destination_plugin = Mock() + mock_destination_plugin.upload_acme_token.return_value = True + mock_plugin_manager_get.return_value = mock_destination_plugin + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options) + + self.assertEqual(pem_certificate, "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n") + self.assertEqual(pem_certificate_chain, + "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n") + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate_missing_destination_token( + self, + mock_authorization_service, + mock_request_certificate, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + mock_destination = Mock() + mock_destination.label = "mock-sftp-destination" + mock_destination.plugin_name = "SFTPDestinationPlugin" + mock_destination_service.get_by_label.return_value = mock_destination + + mock_destination_plugin = Mock() + mock_destination_plugin.upload_acme_token.return_value = True + mock_plugin_manager_get.return_value = mock_destination_plugin + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + with self.assertRaisesRegex(Exception, "No token_destination configured"): + provider.create_certificate(csr, issuer_options) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate_missing_http_challenge( + self, + mock_authorization_service, + mock_request_certificate, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.DNS01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + with self.assertRaisesRegex(Exception, "HTTP-01 challenge was not offered"): + provider.create_certificate(csr, issuer_options) diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 37e4968e..cf850970 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,5 +1,7 @@ import unittest from unittest.mock import patch, Mock + +from flask import Flask from lemur.plugins.lemur_acme import plugin, powerdns @@ -17,6 +19,16 @@ class TestPowerdns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.powerdns.current_app") def test_get_zones(self, mock_current_app): account_number = "1234567890" diff --git a/lemur/plugins/lemur_acme/tests/test_ultradns.py b/lemur/plugins/lemur_acme/tests/test_ultradns.py index f1d61e68..7616459e 100644 --- a/lemur/plugins/lemur_acme/tests/test_ultradns.py +++ b/lemur/plugins/lemur_acme/tests/test_ultradns.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, Mock +from flask import Flask from lemur.plugins.lemur_acme import plugin, ultradns from requests.models import Response @@ -19,6 +20,16 @@ class TestUltradns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.ultradns.requests") @patch("lemur.plugins.lemur_acme.ultradns.current_app") def test_ultradns_get_token(self, mock_current_app, mock_requests): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8692348a..fcc2e0cf 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,13 +32,15 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ + +import sys from acme.errors import ClientError from flask import current_app -from lemur.extensions import sentry, metrics -from lemur.plugins import lemur_aws as aws +from lemur.extensions import sentry, metrics +from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin -from lemur.plugins.lemur_aws import iam, s3, elb, ec2 +from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns def get_region_from_dns(dns): @@ -406,3 +408,120 @@ class S3DestinationPlugin(ExportDestinationPlugin): self.get_option("encrypt", options), account_number=self.get_option("accountNumber", options), ) + + def upload_acme_token(self, token_path, token, options, **kwargs): + """ + This is called from the acme http challenge + :param self: + :param token_path: + :param token: + :param options: + :param kwargs: + :return: + """ + current_app.logger.debug("S3 destination plugin is started to upload HTTP-01 challenge") + + function = f"{__name__}.{sys._getframe().f_code.co_name}" + + account_number = self.get_option("accountNumber", options) + bucket_name = self.get_option("bucket", options) + prefix = self.get_option("prefix", options) + region = self.get_option("region", options) + filename = token_path.split("/")[-1] + if not prefix.endswith("/"): + prefix + "/" + + response = s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) + res = "Success" if response else "Failure" + log_data = { + "function": function, + "message": "upload acme token challenge", + "result": res, + "bucket_name": bucket_name, + "filename": filename + } + current_app.logger.info(log_data) + metrics.send(f"{function}", "counter", 1, metric_tags={"result": res, + "bucket_name": bucket_name, + "filename": filename}) + return response + + def delete_acme_token(self, token_path, options, **kwargs): + + current_app.logger.debug("S3 destination plugin is started to delete HTTP-01 challenge") + + function = f"{__name__}.{sys._getframe().f_code.co_name}" + + account_number = self.get_option("accountNumber", options) + bucket_name = self.get_option("bucket", options) + prefix = self.get_option("prefix", options) + filename = token_path.split("/")[-1] + response = s3.delete(bucket_name=bucket_name, + prefixed_object_name=prefix + filename, + account_number=account_number) + res = "Success" if response else "Failure" + log_data = { + "function": function, + "message": "delete acme token challenge", + "result": res, + "bucket_name": bucket_name, + "filename": filename + } + current_app.logger.info(log_data) + metrics.send(f"{function}", "counter", 1, metric_tags={"result": res, + "bucket_name": bucket_name, + "filename": filename}) + return response + + +class SNSNotificationPlugin(ExpirationNotificationPlugin): + title = "AWS SNS" + slug = "aws-sns" + description = "Sends notifications to AWS SNS" + version = aws.VERSION + + author = "Jasmine Schladen " + author_url = "https://github.com/Netflix/lemur" + + additional_options = [ + { + "name": "accountNumber", + "type": "str", + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access the SNS topic", + }, + { + "name": "region", + "type": "str", + "required": True, + "validation": "[0-9a-z\\-]{1,25}", + "helpMessage": "Region in which the SNS topic is located, e.g. \"us-east-1\"", + }, + { + "name": "topicName", + "type": "str", + "required": True, + # base topic name is 1-256 characters (alphanumeric plus underscore and hyphen) + "validation": "^[a-zA-Z0-9_\\-]{1,256}$", + "helpMessage": "The name of the topic to use for expiration notifications", + } + ] + + def send(self, notification_type, message, excluded_targets, options, **kwargs): + """ + While we receive a `targets` parameter here, it is unused, as the SNS topic is pre-configured in the + plugin configuration, and can't reasonably be changed dynamically. + """ + + topic_arn = f"arn:aws:sns:{self.get_option('region', options)}:" \ + f"{self.get_option('accountNumber', options)}:" \ + f"{self.get_option('topicName', options)}" + + current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}") + sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 43faa28f..1b0831b3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -6,12 +6,15 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from botocore.exceptions import ClientError from flask import current_app +from lemur.extensions import sentry + from .sts import sts_client @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ @@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs): ServerSideEncryption="AES256", ) else: - bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + try: + bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + return True + except ClientError: + sentry.captureException() + return False + + +@sts_client("s3", service_type="client") +def delete(bucket_name, prefixed_object_name, **kwargs): + """ + Use STS to delete an object + """ + try: + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name) + current_app.logger.debug(f"Delete data from S3." + f"Bucket: {bucket_name}," + f"Prefix: {prefixed_object_name}," + f"Status_code: {response}") + return response['ResponseMetadata']['HTTPStatusCode'] < 300 + except ClientError: + sentry.captureException() + return False + + +@sts_client("s3", service_type="client") +def get(bucket_name, prefixed_object_name, **kwargs): + """ + Use STS to get an object + """ + try: + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name) + current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," + f"object_name: {prefixed_object_name}") + return response['Body'].read().decode("utf-8") + except ClientError: + sentry.captureException() + return None diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py new file mode 100644 index 00000000..14109c11 --- /dev/null +++ b/lemur/plugins/lemur_aws/sns.py @@ -0,0 +1,58 @@ +""" +.. module: lemur.plugins.lemur_aws.sts + :platform: Unix + :copyright: (c) 2020 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Jasmine Schladen +""" +import json + +import arrow +import boto3 +from flask import current_app + + +def publish(topic_arn, certificates, notification_type, **kwargs): + sns_client = boto3.client("sns", **kwargs) + message_ids = {} + subject = "Lemur: {0} Notification".format(notification_type.capitalize()) + for certificate in certificates: + message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject) + + return message_ids + + +def publish_single(sns_client, topic_arn, certificate, notification_type, subject): + response = sns_client.publish( + TopicArn=topic_arn, + Message=format_message(certificate, notification_type), + Subject=subject, + ) + + response_code = response["ResponseMetadata"]["HTTPStatusCode"] + if response_code != 200: + raise Exception(f"Failed to publish {notification_type} notification to SNS topic {topic_arn}. " + f"SNS response: {response_code} {response}") + + current_app.logger.info(f"AWS SNS message published to topic [{topic_arn}] with message ID {response['MessageId']}") + current_app.logger.debug(f"AWS SNS message published to topic [{topic_arn}]: [{response}]") + + return response["MessageId"] + + +def create_certificate_url(name): + return "https://{hostname}/#/certificates/{name}".format( + hostname=current_app.config.get("LEMUR_HOSTNAME"), name=name + ) + + +def format_message(certificate, notification_type): + json_message = { + "notification_type": notification_type, + "certificate_name": certificate["name"], + "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), # 2047-12-31T22:00:00 + "endpoints_detected": len(certificate["endpoints"]), + "owner": certificate["owner"], + "details": create_certificate_url(certificate["name"]) + } + return json.dumps(json_message) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index dbad7b02..e032cf02 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -1,5 +1,88 @@ +import boto3 +from moto import mock_sts, mock_s3 + + def test_get_certificates(app): from lemur.plugins.base import plugins p = plugins.get("aws-s3") assert p + + +@mock_sts() +@mock_s3() +def test_upload_acme_token(app): + from lemur.plugins.base import plugins + from lemur.plugins.lemur_aws.s3 import get + + bucket = "public-bucket" + account = "123456789012" + prefix = "some-path/more-path/" + token_content = "Challenge" + token_name = "TOKEN" + token_path = ".well-known/acme-challenge/" + token_name + + additional_options = [ + { + "name": "bucket", + "value": bucket, + "type": "str", + "required": True, + "validation": r"[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account, + "required": True, + "validation": r"[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + p = plugins.get("aws-s3") + + response = p.upload_acme_token(token_path=token_path, + token_content=token_content, + token=token_content, + options=additional_options) + assert response + + response = get(bucket_name=bucket, + prefixed_object_name=prefix + token_name, + encrypt=False, + account_number=account) + + # put data, and getting the same data + assert (response == token_content) + + response = p.delete_acme_token(token_path=token_path, + options=additional_options, + account_number=account) + assert response diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py new file mode 100644 index 00000000..7d0fa843 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -0,0 +1,41 @@ +import boto3 +from moto import mock_sts, mock_s3 + + +@mock_sts() +@mock_s3() +def test_put_delete_s3_object(app): + from lemur.plugins.lemur_aws.s3 import put, delete, get + + bucket = "public-bucket" + region = "us-east-1" + account = "123456789012" + path = "some-path/foo" + data = "dummy data" + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + + put(bucket_name=bucket, + region_name=region, + prefix=path, + data=data, + encrypt=False, + account_number=account, + region=region) + + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) + + # put data, and getting the same data + assert (response == data) + + response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account) + + # attempting to get thccle wrong data + assert (response is None) + + delete(bucket_name=bucket, prefixed_object_name=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) + + # delete data, and getting the same data + assert (response is None) diff --git a/lemur/plugins/lemur_aws/tests/test_sns.py b/lemur/plugins/lemur_aws/tests/test_sns.py new file mode 100644 index 00000000..59ef30f2 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_sns.py @@ -0,0 +1,123 @@ +import json +from datetime import timedelta + +import arrow +import boto3 +from moto import mock_sns, mock_sqs, mock_ses + +from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.plugins.lemur_aws.sns import format_message +from lemur.plugins.lemur_aws.sns import publish +from lemur.tests.factories import NotificationFactory, CertificateFactory +from lemur.tests.test_messaging import verify_sender_email + + +@mock_sns() +def test_format(certificate, endpoint): + data = [certificate_notification_output_schema.dump(certificate).data] + + for certificate in data: + expected_message = { + "notification_type": "expiration", + "certificate_name": certificate["name"], + "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), + "endpoints_detected": 0, + "owner": certificate["owner"], + "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) + } + assert expected_message == json.loads(format_message(certificate, "expiration")) + + +@mock_sns() +@mock_sqs() +def create_and_subscribe_to_topic(): + sns_client = boto3.client("sns", region_name="us-east-1") + topic_arn = sns_client.create_topic(Name='lemursnstest')["TopicArn"] + + sqs_client = boto3.client("sqs", region_name="us-east-1") + queue = sqs_client.create_queue(QueueName="lemursnstestqueue") + queue_url = queue["QueueUrl"] + queue_arn = sqs_client.get_queue_attributes(QueueUrl=queue_url)["Attributes"]["QueueArn"] + sns_client.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn) + + return [topic_arn, sqs_client, queue_url] + + +@mock_sns() +@mock_sqs() +def test_publish(certificate, endpoint): + data = [certificate_notification_output_schema.dump(certificate).data] + + topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic() + + message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1") + assert len(message_ids) == len(data) + received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] + + for certificate in data: + expected_message_id = message_ids[certificate["name"]] + actual_message = next( + (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None) + actual_json = json.loads(actual_message["Body"]) + assert actual_json["Message"] == format_message(certificate, "expiration") + assert actual_json["Subject"] == "Lemur: Expiration Notification" + + +def get_options(): + return [ + {"name": "interval", "value": 10}, + {"name": "unit", "value": "days"}, + {"name": "region", "value": "us-east-1"}, + {"name": "accountNumber", "value": "123456789012"}, + {"name": "topicName", "value": "lemursnstest"}, + ] + + +@mock_sns() +@mock_sqs() +@mock_ses() # because email notifications are also sent +def test_send_expiration_notification(): + from lemur.notifications.messaging import send_expiration_notifications + + verify_sender_email() # emails are sent to owner and security; SNS only used for configured notification + topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic() + + notification = NotificationFactory(plugin_name="aws-sns") + notification.options = get_options() + + now = arrow.utcnow() + in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future + + certificate = CertificateFactory() + certificate.not_after = in_ten_days + certificate.notifications.append(notification) + + assert send_expiration_notifications([]) == (3, 0) # owner, SNS, and security + + received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] + assert len(received_messages) == 1 + expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration") + actual_message = json.loads(received_messages[0]["Body"])["Message"] + assert actual_message == expected_message + + +# Currently disabled as the SNS plugin doesn't support this type of notification +# def test_send_rotation_notification(endpoint, source_plugin): +# from lemur.notifications.messaging import send_rotation_notification +# from lemur.deployment.service import rotate_certificate +# +# notification = NotificationFactory(plugin_name="aws-sns") +# notification.options = get_options() +# +# new_certificate = CertificateFactory() +# rotate_certificate(endpoint, new_certificate) +# assert endpoint.certificate == new_certificate +# +# assert send_rotation_notification(new_certificate) + + +# Currently disabled as the SNS plugin doesn't support this type of notification +# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin): +# from lemur.notifications.messaging import send_pending_failure_notification +# +# assert send_pending_failure_notification(pending_certificate) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index f28279a6..d8e88fa3 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -37,7 +37,13 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ + log_data = { + "reason": (r.reason if r.reason else ""), + "status_code": r.status_code, + "url": (r.url if r.url else ""), + } metrics.send("digicert_status_code_{}".format(r.status_code), "counter", 1) + current_app.logger.info(log_data) def signature_hash(signing_algorithm): @@ -171,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "organization": { "name": options["organization"], @@ -204,7 +210,7 @@ def handle_response(response): :return: """ if response.status_code > 399: - raise Exception(response.json()["errors"][0]["message"]) + raise Exception("DigiCert rejected request with the error:" + response.json()["errors"][0]["message"]) return response.json() @@ -215,13 +221,20 @@ def handle_cis_response(response): :param response: :return: """ - if response.status_code > 399: - raise Exception(response.text) + if response.status_code == 404: + raise Exception("DigiCert: order not in issued state") + elif response.status_code == 406: + raise Exception("DigiCert: wrong header request format") + elif response.status_code > 399: + raise Exception("DigiCert rejected request with the error:" + response.text) - return response.json() + if response.url.endswith("download"): + return response.content + else: + return response.json() -@retry(stop_max_attempt_number=10, wait_fixed=10000) +@retry(stop_max_attempt_number=10, wait_fixed=1000) def get_certificate_id(session, base_url, order_id): """Retrieve certificate order id from Digicert API.""" order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id) @@ -232,17 +245,15 @@ def get_certificate_id(session, base_url, order_id): return response_data["certificate"]["id"] -@retry(stop_max_attempt_number=10, wait_fixed=10000) +@retry(stop_max_attempt_number=10, wait_fixed=1000) def get_cis_certificate(session, base_url, order_id): """Retrieve certificate order id from Digicert API, including the chain""" certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id) session.headers.update({"Accept": "application/x-pkcs7-certificates"}) response = session.get(certificate_url) + response_content = handle_cis_response(response) - if response.status_code == 404: - raise Exception("Order not in issued state.") - - cert_chain_pem = convert_pkcs7_bytes_to_pem(response.content) + cert_chain_pem = convert_pkcs7_bytes_to_pem(response_content) if len(cert_chain_pem) < 3: raise Exception("Missing the certificate chain") return cert_chain_pem diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 34dcef71..fd07ea2b 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -123,7 +123,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } @@ -159,7 +159,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 241aa1b0..041b27ec 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -17,16 +17,19 @@ from lemur.plugins.bases import ExpirationNotificationPlugin from lemur.plugins import lemur_email as email from lemur.plugins.lemur_email.templates.config import env +from lemur.plugins.utils import get_plugin_option -def render_html(template_name, message): +def render_html(template_name, options, certificates): """ Renders the html for our email notification. :param template_name: - :param message: + :param options: + :param certificates: :return: """ + message = {"options": options, "certificates": certificates} template = env.get_template("{}.html".format(template_name)) return template.render( dict(message=message, hostname=current_app.config.get("LEMUR_HOSTNAME")) @@ -35,7 +38,7 @@ def render_html(template_name, message): def send_via_smtp(subject, body, targets): """ - Attempts to deliver email notification via SES service. + Attempts to deliver email notification via SMTP. :param subject: :param body: @@ -52,21 +55,26 @@ def send_via_smtp(subject, body, targets): def send_via_ses(subject, body, targets): """ - Attempts to deliver email notification via SMTP. + Attempts to deliver email notification via SES service. :param subject: :param body: :param targets: :return: """ - client = boto3.client("ses", region_name="us-east-1") - client.send_email( - Source=current_app.config.get("LEMUR_EMAIL"), - Destination={"ToAddresses": targets}, - Message={ + ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1") + client = boto3.client("ses", region_name=ses_region) + source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN") + args = { + "Source": current_app.config.get("LEMUR_EMAIL"), + "Destination": {"ToAddresses": targets}, + "Message": { "Subject": {"Data": subject, "Charset": "UTF-8"}, "Body": {"Html": {"Data": body, "Charset": "UTF-8"}}, }, - ) + } + if source_arn: + args["SourceArn"] = source_arn + client.send_email(**args) class EmailNotificationPlugin(ExpirationNotificationPlugin): @@ -83,7 +91,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): "name": "recipients", "type": "str", "required": True, - "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", + "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", "helpMessage": "Comma delimited list of email addresses", } ] @@ -100,8 +108,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): subject = "Lemur: {0} Notification".format(notification_type.capitalize()) - data = {"options": options, "certificates": message} - body = render_html(notification_type, data) + body = render_html(notification_type, options, message) s_type = current_app.config.get("LEMUR_EMAIL_SENDER", "ses").lower() @@ -110,3 +117,13 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): elif s_type == "smtp": send_via_smtp(subject, body, targets) + + @staticmethod + def filter_recipients(options, excluded_recipients, **kwargs): + notification_recipients = get_plugin_option("recipients", options) + if notification_recipients: + notification_recipients = notification_recipients.split(",") + # removing owner and security_email from notification_recipient + notification_recipients = [i for i in notification_recipients if i not in excluded_recipients] + + return notification_recipients diff --git a/lemur/plugins/lemur_email/templates/rotation.html b/lemur/plugins/lemur_email/templates/rotation.html index 521eb327..9ce7ff33 100644 --- a/lemur/plugins/lemur_email/templates/rotation.html +++ b/lemur/plugins/lemur_email/templates/rotation.html @@ -83,12 +83,12 @@ - {{ certificate.name }} + {{ message.certificates.name }}
-
{{ certificate.owner }} -
{{ certificate.validityEnd | time }} - Details +
{{ message.certificates.owner }} +
{{ message.certificates.validityEnd | time }} + Details @@ -110,12 +110,12 @@ - {{ certificate.replacedBy[0].name }} + {{ message.certificates.name }}
-
{{ certificate.replacedBy[0].owner }} -
{{ certificate.replacedBy[0].validityEnd | time }} - Details +
{{ message.certificates.owner }} +
{{ message.certificates.validityEnd | time }} + Details @@ -133,7 +133,7 @@ - {% for endpoint in certificate.endpoints %} + {% for endpoint in message.certificates.endpoints %} diff --git a/lemur/plugins/lemur_email/tests/test_email.py b/lemur/plugins/lemur_email/tests/test_email.py index 43168cab..fd4dc575 100644 --- a/lemur/plugins/lemur_email/tests/test_email.py +++ b/lemur/plugins/lemur_email/tests/test_email.py @@ -1,36 +1,90 @@ import os -from lemur.plugins.lemur_email.templates.config import env +from datetime import timedelta +import arrow +from moto import mock_ses + +from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.plugins.lemur_email.plugin import render_html from lemur.tests.factories import CertificateFactory +from lemur.tests.test_messaging import verify_sender_email dir_path = os.path.dirname(os.path.realpath(__file__)) -def test_render(certificate, endpoint): - from lemur.certificates.schemas import certificate_notification_output_schema +def get_options(): + return [ + {"name": "interval", "value": 10}, + {"name": "unit", "value": "days"}, + {"name": "recipients", "value": "person1@example.com,person2@example.com"}, + ] + + +def test_render_expiration(certificate, endpoint): new_cert = CertificateFactory() new_cert.replaces.append(certificate) - data = { - "certificates": [certificate_notification_output_schema.dump(certificate).data], - "options": [ - {"name": "interval", "value": 10}, - {"name": "unit", "value": "days"}, - ], - } + assert render_html("expiration", get_options(), [certificate_notification_output_schema.dump(certificate).data]) - template = env.get_template("{}.html".format("expiration")) - - body = template.render(dict(message=data, hostname="lemur.test.example.com")) - - template = env.get_template("{}.html".format("rotation")) +def test_render_rotation(certificate, endpoint): certificate.endpoints.append(endpoint) - body = template.render( - dict( - certificate=certificate_notification_output_schema.dump(certificate).data, - hostname="lemur.test.example.com", - ) - ) + assert render_html("rotation", get_options(), certificate_notification_output_schema.dump(certificate).data) + + +def test_render_rotation_failure(pending_certificate): + assert render_html("failed", get_options(), certificate_notification_output_schema.dump(pending_certificate).data) + + +@mock_ses +def test_send_expiration_notification(): + from lemur.notifications.messaging import send_expiration_notifications + from lemur.tests.factories import CertificateFactory + from lemur.tests.factories import NotificationFactory + + now = arrow.utcnow() + in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future + certificate = CertificateFactory() + notification = NotificationFactory(plugin_name="email-notification") + + certificate.not_after = in_ten_days + certificate.notifications.append(notification) + certificate.notifications[0].options = get_options() + + verify_sender_email() + assert send_expiration_notifications([]) == (3, 0) # owner, recipients (only counted as 1), and security + + +@mock_ses +def test_send_rotation_notification(endpoint, source_plugin): + from lemur.notifications.messaging import send_rotation_notification + from lemur.deployment.service import rotate_certificate + + new_certificate = CertificateFactory() + rotate_certificate(endpoint, new_certificate) + assert endpoint.certificate == new_certificate + + verify_sender_email() + assert send_rotation_notification(new_certificate) + + +@mock_ses +def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin): + from lemur.notifications.messaging import send_pending_failure_notification + + verify_sender_email() + assert send_pending_failure_notification(pending_certificate) + + +def test_filter_recipients(certificate, endpoint): + from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin + + options = [{"name": "recipients", "value": "security@example.com,bob@example.com,joe@example.com"}] + assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@example.com", "bob@example.com", + "joe@example.com"] + assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com"]) == ["bob@example.com", + "joe@example.com"] + assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com", "bob@example.com", + "joe@example.com"]) == [] diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 515e2400..d3324db0 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -1,9 +1,9 @@ - import arrow import requests import json import sys from flask import current_app +from retrying import retry from lemur.plugins import lemur_entrust as entrust from lemur.plugins.bases import IssuerPlugin, SourcePlugin @@ -20,7 +20,13 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ + log_data = { + "reason": (r.reason if r.reason else ""), + "status_code": r.status_code, + "url": (r.url if r.url else ""), + } metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1) + current_app.logger.info(log_data) def determine_end_date(end_date): @@ -72,7 +78,6 @@ def process_options(options): "eku": "SERVER_AND_CLIENT_AUTH", "certType": product_type, "certExpiryDate": validity_end, - # "keyType": "RSA", Entrust complaining about this parameter "tracking": tracking_data } return data @@ -81,7 +86,7 @@ def process_options(options): def handle_response(my_response): """ Helper function for parsing responses from the Entrust API. - :param content: + :param my_response: :return: :raise Exception: """ msg = { @@ -94,22 +99,47 @@ def handle_response(my_response): } try: - d = json.loads(my_response.content) + data = json.loads(my_response.content) except ValueError: # catch an empty jason object here - d = {'response': 'No detailed message'} - s = my_response.status_code - if s > 399: - raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}") + data = {'response': 'No detailed message'} + status_code = my_response.status_code + if status_code > 399: + raise Exception(f"ENTRUST error: {msg.get(status_code, status_code)}\n{data['errors']}") log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "message": "Response", - "status": s, - "response": d + "status": status_code, + "response": data } current_app.logger.info(log_data) - return d + if data == {'response': 'No detailed message'}: + # status if no data + return status_code + else: + # return data from the response + return data + + +@retry(stop_max_attempt_number=3, wait_fixed=5000) +def order_and_download_certificate(session, url, data): + """ + Helper function to place a certificacte order and download it + :param session: + :param url: Entrust endpoint url + :param data: CSR, and the required order details, such as validity length + :return: the cert chain + :raise Exception: + """ + try: + response = session.post(url, json=data, timeout=(15, 40)) + except requests.exceptions.Timeout: + raise Exception("Timeout for POST") + except requests.exceptions.RequestException as e: + raise Exception(f"Error for POST {e}") + + return handle_response(response) class EntrustIssuerPlugin(IssuerPlugin): @@ -167,14 +197,8 @@ class EntrustIssuerPlugin(IssuerPlugin): data = process_options(issuer_options) data["csr"] = csr - try: - response = self.session.post(url, json=data, timeout=(15, 40)) - except requests.exceptions.Timeout: - raise Exception("Timeout for POST") - except requests.exceptions.RequestException as e: - raise Exception(f"Error for POST {e}") + response_dict = order_and_download_certificate(self.session, url, data) - response_dict = handle_response(response) external_id = response_dict['trackingId'] cert = response_dict['endEntityCert'] if len(response_dict['chainCerts']) < 2: @@ -189,6 +213,7 @@ class EntrustIssuerPlugin(IssuerPlugin): return cert, chain, external_id + @retry(stop_max_attempt_number=3, wait_fixed=1000) def revoke_certificate(self, certificate, comments): """Revoke an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") @@ -205,6 +230,7 @@ class EntrustIssuerPlugin(IssuerPlugin): metrics.send("entrust_revoke_certificate", "counter", 1) return handle_response(response) + @retry(stop_max_attempt_number=3, wait_fixed=1000) def deactivate_certificate(self, certificate): """Deactivates an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") @@ -233,7 +259,7 @@ class EntrustIssuerPlugin(IssuerPlugin): def get_ordered_certificate(self, order_id): raise NotImplementedError("Not implemented\n", self, order_id) - def canceled_ordered_certificate(self, pending_cert, **kwargs): + def cancel_ordered_certificate(self, pending_cert, **kwargs): raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs) diff --git a/lemur/plugins/lemur_openssh/__init__.py b/lemur/plugins/lemur_openssh/__init__.py new file mode 100644 index 00000000..f8afd7e3 --- /dev/null +++ b/lemur/plugins/lemur_openssh/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__("pkg_resources").get_distribution(__name__).version +except Exception as e: + VERSION = "unknown" diff --git a/lemur/plugins/lemur_openssh/plugin.py b/lemur/plugins/lemur_openssh/plugin.py new file mode 100644 index 00000000..6a02abe0 --- /dev/null +++ b/lemur/plugins/lemur_openssh/plugin.py @@ -0,0 +1,152 @@ +""" +.. module: lemur.plugins.lemur_openssh.plugin + :platform: Unix + :copyright: (c) 2020 by Emmanuel Garette, see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Emmanuel Garette +""" +import subprocess +from os import unlink + +from flask import current_app +from cryptography.hazmat.primitives import serialization +from datetime import datetime, timedelta + +from lemur.utils import mktempfile +from lemur.plugins import lemur_openssh as openssh +from lemur.common.utils import parse_private_key, parse_certificate +from lemur.plugins.lemur_cryptography.plugin import CryptographyIssuerPlugin +from lemur.certificates.service import get_by_root_authority + + +def run_process(command): + """ + Runs a given command with pOpen and wraps some + error handling around it. + :param command: + :return: + """ + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + current_app.logger.debug(" ".join(command)) + stdout, stderr = p.communicate() + + if p.returncode != 0: + current_app.logger.error(stderr.decode()) + raise Exception(stderr.decode()) + + +def split_cert(body): + """ + To display certificate in Lemur website, we have to split + certificate in several line + :param body: certificate + :retur: splitted certificate + """ + length = 65 + return '\n'.join([body[i:i + length] for i in range(0, len(body), length)]) + + +def sign_certificate(common_name, public_key, authority_private_key, user, extensions, not_before, not_after): + with mktempfile() as issuer_tmp: + cmd = ['ssh-keygen', '-s', issuer_tmp] + with open(issuer_tmp, 'w') as i: + i.writelines(authority_private_key) + if 'extendedKeyUsage' in extensions and extensions['extendedKeyUsage'].get('useClientAuthentication'): + valid_interval = current_app.config.get("OPENSSH_VALID_INTERVAL_CLIENT", 1) # 1 day by default + cmd.extend(['-I', user['username'] + ' user key', + '-n', user['username']]) + else: + valid_interval = current_app.config.get("OPENSSH_VALID_INTERVAL_SERVER", 14) # 2 weeks by default + domains = {common_name} + for name in extensions['subAltNames']['names']: + if name['nameType'] == 'DNSName': + domains.add(name['value']) + cmd.extend(['-I', common_name + ' host key', + '-n', ','.join(domains), + '-h']) + # something like 20201024 + ssh_not_before = datetime.fromisoformat(not_before).strftime("%Y%m%d") + cert_not_after = datetime.fromisoformat(not_after).strftime("%Y%m%d") + ssh_not_after = (datetime.now() + timedelta(days=valid_interval)).strftime("%Y%m%d") + ssh_not_after = min(ssh_not_after, cert_not_after) + cmd.extend(['-V', ssh_not_before + ':' + ssh_not_after]) + with mktempfile() as cert_tmp: + with open(cert_tmp, 'w') as f: + f.write(public_key) + + cmd.append(cert_tmp) + run_process(cmd) + pub = cert_tmp + '-cert.pub' + with open(pub, 'r') as p: + body = split_cert(p.read()) + unlink(pub) + return body + + +class OpenSSHIssuerPlugin(CryptographyIssuerPlugin): + """This issuer plugins is base in Cryptography plugin + Certificates and authorities are x509 certificates created by Cryptography plugin. + Those certificates are converted to OpenSSH format when people get them. + """ + title = "OpenSSH" + slug = "openssh-issuer" + description = "Enables the creation and signing OpenSSH keys" + version = openssh.VERSION + + author = "Emmanuel Garette" + author_url = "http://gnunux.info" + + def create_authority(self, options): + # OpenSSH do not support parent's authoriy + if options.get("parent"): + raise Exception('cannot create authority with a parent for OpenSSH plugin') + # create a x509 certificat + cert_pem, private_key, chain_cert_pem, roles = super().create_authority(options) + return cert_pem, private_key, chain_cert_pem, roles + + def wrap_certificate(self, cert): + # get public_key in OpenSSH format + public_key = parse_certificate(cert['body']).public_key().public_bytes( + encoding=serialization.Encoding.OpenSSH, + format=serialization.PublicFormat.OpenSSH, + ).decode() + public_key += ' ' + cert['user']['email'] + # sign it with authority private key + authority = get_by_root_authority(cert['authority']['id']) + authority_private_key = authority.private_key + cert['body'] = sign_certificate( + cert['common_name'], + public_key, + authority_private_key, + cert['user'], + cert['extensions'], + cert['not_before'], + cert['not_after'] + ) + # convert chain in OpenSSH format + if cert['chain']: + chain_cert = {'body': cert['chain'], 'cn': authority.cn} + self.wrap_auth_certificate(chain_cert) + cert['chain'] = chain_cert['body'] + # OpenSSH do not support csr + cert['csr'] = None + + @staticmethod + def wrap_auth_certificate(auth_cert): + # convert chain in OpenSSH format + chain_key = parse_certificate(auth_cert['body']).public_key().public_bytes( + encoding=serialization.Encoding.OpenSSH, + format=serialization.PublicFormat.OpenSSH, + ).decode() + chain_key += ' root@' + auth_cert['cn'] + auth_cert['body'] = split_cert(chain_key) + + @staticmethod + def wrap_private_key(cert): + # convert private_key in OpenSSH format + cert.private_key = parse_private_key(cert.private_key).private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.OpenSSH, + encryption_algorithm=serialization.NoEncryption(), + ) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 66784048..75d49e2b 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,8 +16,10 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ +from os import path import paramiko +from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError from flask import current_app from lemur.plugins import lemur_sftp @@ -47,7 +49,7 @@ class SFTPDestinationPlugin(DestinationPlugin): "type": "int", "required": True, "helpMessage": "The SFTP port, default is 22.", - "validation": "^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})", + "validation": r"^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})", "default": "22", }, { @@ -95,33 +97,15 @@ class SFTPDestinationPlugin(DestinationPlugin): }, ] - def upload(self, name, body, private_key, cert_chain, options, **kwargs): - - current_app.logger.debug("SFTP destination plugin is started") - - cn = common_name(parse_certificate(body)) + def open_sftp_connection(self, options): host = self.get_option("host", options) port = self.get_option("port", options) user = self.get_option("user", options) password = self.get_option("password", options) ssh_priv_key = self.get_option("privateKeyPath", options) ssh_priv_key_pass = self.get_option("privateKeyPass", options) - dst_path = self.get_option("destinationPath", options) - export_format = self.get_option("exportFormat", options) - # prepare files for upload - files = {cn + ".key": private_key, cn + ".pem": body} - - if cert_chain: - if export_format == "NGINX": - # assemble body + chain in the single file - files[cn + ".pem"] += "\n" + cert_chain - - elif export_format == "Apache": - # store chain in the separate file - files[cn + ".ca.bundle.pem"] = cert_chain - - # upload files + # delete files try: current_app.logger.debug( "Connecting to {0}@{1}:{2}".format(user, host, port) @@ -145,50 +129,170 @@ class SFTPDestinationPlugin(DestinationPlugin): current_app.logger.error( "No password or private key provided. Can't proceed" ) - raise paramiko.ssh_exception.AuthenticationException + raise AuthenticationException # open the sftp session inside the ssh connection - sftp = ssh.open_sftp() + return ssh.open_sftp(), ssh - # make sure that the destination path exist - try: - current_app.logger.debug("Creating {0}".format(dst_path)) - sftp.mkdir(dst_path) - except IOError: - current_app.logger.debug("{0} already exist, resuming".format(dst_path)) - try: - dst_path_cn = dst_path + "/" + cn - current_app.logger.debug("Creating {0}".format(dst_path_cn)) - sftp.mkdir(dst_path_cn) - except IOError: - current_app.logger.debug( - "{0} already exist, resuming".format(dst_path_cn) - ) + except AuthenticationException as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.") + except NoValidConnectionsError as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + raise NoValidConnectionsError("Couldn't connect to {0}, possible timeout or invalid hostname") - # upload certificate files to the sftp destination - for filename, data in files.items(): + # this is called when using this as a default destination plugin + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + + current_app.logger.debug("SFTP destination plugin is started") + + cn = common_name(parse_certificate(body)) + dst_path = self.get_option("destinationPath", options) + dst_path_cn = dst_path + "/" + cn + export_format = self.get_option("exportFormat", options) + + # prepare files for upload + files = {cn + ".key": private_key, cn + ".pem": body} + + if cert_chain: + if export_format == "NGINX": + # assemble body + chain in the single file + files[cn + ".pem"] += "\n" + cert_chain + + elif export_format == "Apache": + # store chain in the separate file + files[cn + ".ca.bundle.pem"] = cert_chain + + self.upload_file(dst_path_cn, files, options) + + # this is called from the acme http challenge + def upload_acme_token(self, token_path, token, options, **kwargs): + + current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") + + dst_path = self.get_option("destinationPath", options) + + _, filename = path.split(token_path) + + # prepare files for upload + files = {filename: token} + + self.upload_file(dst_path, files, options) + + # this is called from the acme http challenge + def delete_acme_token(self, token_path, options, **kwargs): + dst_path = self.get_option("destinationPath", options) + + _, filename = path.split(token_path) + + # prepare files for upload + files = {filename: None} + + self.delete_file(dst_path, files, options) + + # here the file is deleted + def delete_file(self, dst_path, files, options): + + try: + # open the ssh and sftp sessions + sftp, ssh = self.open_sftp_connection(options) + + # delete files + for filename, _ in files.items(): current_app.logger.debug( - "Uploading {0} to {1}".format(filename, dst_path_cn) + "Deleting {0} from {1}".format(filename, dst_path) ) try: - with sftp.open(dst_path_cn + "/" + filename, "w") as f: - f.write(data) - except (PermissionError) as permerror: + sftp.remove(path.join(dst_path, filename)) + except PermissionError as permerror: if permerror.errno == 13: current_app.logger.debug( - "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn) + "Deleting {0} from {1} returned Permission Denied Error, making file writable and retrying".format( + filename, dst_path) ) - sftp.chmod(dst_path_cn + "/" + filename, 0o600) - with sftp.open(dst_path_cn + "/" + filename, "w") as f: - f.write(data) - # read only for owner, -r-------- - sftp.chmod(dst_path_cn + "/" + filename, 0o400) + sftp.chmod(path.join(dst_path, filename), 0o600) + sftp.remove(path.join(dst_path, filename)) ssh.close() - + except (AuthenticationException, NoValidConnectionsError) as e: + raise e except Exception as e: current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) try: ssh.close() except BaseException: pass + + # here the file is uploaded for real, this helps to keep this class DRY + def upload_file(self, dst_path, files, options): + + try: + # open the ssh and sftp sessions + sftp, ssh = self.open_sftp_connection(options) + + # split the path into it's segments, so we can create it recursively + allparts = [] + path_copy = dst_path + while True: + parts = path.split(path_copy) + if parts[0] == path_copy: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path_copy: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path_copy = parts[0] + allparts.insert(0, parts[1]) + + # make sure that the destination path exists, recursively + remote_path = allparts[0] + for part in allparts: + try: + if part != "/" and part != "": + remote_path = path.join(remote_path, part) + sftp.stat(remote_path) + except IOError: + current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path)) + try: + sftp.mkdir(remote_path) + except IOError as ioerror: + current_app.logger.debug( + "Couldn't create {0}, error message: {1}".format(remote_path, ioerror)) + + # upload certificate files to the sftp destination + for filename, data in files.items(): + current_app.logger.debug( + "Uploading {0} to {1}".format(filename, dst_path) + ) + try: + with sftp.open(path.join(dst_path, filename), "w") as f: + f.write(data) + except PermissionError as permerror: + if permerror.errno == 13: + current_app.logger.debug( + "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format( + filename, dst_path) + ) + sftp.chmod(path.join(dst_path, filename), 0o600) + with sftp.open(path.join(dst_path, filename), "w") as f: + f.write(data) + # most likely the upload user isn't the webuser, -rw-r--r-- + sftp.chmod(path.join(dst_path, filename), 0o644) + + ssh.close() + + except (AuthenticationException, NoValidConnectionsError) as e: + raise e + except Exception as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + try: + ssh.close() + except BaseException: + pass + message = '' + if hasattr(e, 'errors'): + for _, error in e.errors.items(): + message = error.strerror + raise Exception( + 'Couldn\'t upload file to {}, error message: {}'.format(self.get_option("host", options), message)) diff --git a/lemur/plugins/lemur_sftp/tests/test_sftp.py b/lemur/plugins/lemur_sftp/tests/test_sftp.py new file mode 100644 index 00000000..e30a1ac9 --- /dev/null +++ b/lemur/plugins/lemur_sftp/tests/test_sftp.py @@ -0,0 +1,144 @@ +import unittest +from unittest.mock import patch, Mock, MagicMock, mock_open + +from flask import Flask +from lemur.plugins.lemur_sftp import plugin +from paramiko.ssh_exception import AuthenticationException + + +class TestSftp(unittest.TestCase): + def setUp(self): + self.sftp_destination = plugin.SFTPDestinationPlugin() + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_sftp') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + + def test_failing_ssh_connection(self): + dst_path = '/var/non-existent' + files = {'first-file': 'data'} + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}] + + with self.assertRaises(AuthenticationException): + self.sftp_destination.upload_file(dst_path, files, options) + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_upload_file_single_with_password(self, mock_paramiko): + dst_path = '/var/non-existent' + files = {'first-file': 'data'} + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}] + + mock_sftp = Mock() + mock_sftp.open = mock_open() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.upload_file(dst_path, files, options) + + mock_sftp.open.assert_called_once_with('/var/non-existent/first-file', 'w') + handle = mock_sftp.open() + handle.write.assert_called_once_with('data') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + password='test_password') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_upload_file_multiple_with_key(self, mock_paramiko): + dst_path = '/var/non-existent' + files = {'first-file': 'data', 'second-file': 'data2'} + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'privateKeyPath', 'value': '/var/id_rsa'}, + {'name': 'privateKeyPass', 'value': 'ssh-key-password'}] + + mock_sftp = Mock() + mock_sftp.open = mock_open() + + mock_paramiko.RSAKey.from_private_key_file.return_value = 'ssh-rsa test-key' + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.upload_file(dst_path, files, options) + + mock_sftp.open.assert_called_with('/var/non-existent/second-file', 'w') + handle = mock_sftp.open() + handle.write.assert_called_with('data2') + mock_ssh.close.assert_called_once() + + mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/var/id_rsa', 'ssh-key-password') + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + pkey='ssh-rsa test-key') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_upload_acme_token(self, mock_paramiko): + token_path = './well-known/acme-challenge/some-token-path' + token = 'token-data' + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}, + {'name': 'destinationPath', 'value': '/var/destination-path'}] + + mock_sftp = Mock() + mock_sftp.open = mock_open() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.upload_acme_token(token_path, token, options) + + mock_sftp.open.assert_called_once_with('/var/destination-path/some-token-path', 'w') + handle = mock_sftp.open() + handle.write.assert_called_once_with('token-data') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + password='test_password') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_delete_file_with_password(self, mock_paramiko): + dst_path = '/var/non-existent' + files = {'first-file': None} + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}] + + mock_sftp = Mock() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.delete_file(dst_path, files, options) + + mock_sftp.remove.assert_called_once_with('/var/non-existent/first-file') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + password='test_password') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_delete_acme_token(self, mock_paramiko): + token_path = './well-known/acme-challenge/some-token-path' + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}, + {'name': 'destinationPath', 'value': '/var/destination-path'}] + + mock_sftp = Mock() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.delete_acme_token(token_path, options) + + mock_sftp.remove.assert_called_once_with('/var/destination-path/some-token-path') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + password='test_password') diff --git a/lemur/plugins/lemur_slack/plugin.py b/lemur/plugins/lemur_slack/plugin.py index 7569d295..3ad22bca 100644 --- a/lemur/plugins/lemur_slack/plugin.py +++ b/lemur/plugins/lemur_slack/plugin.py @@ -58,26 +58,19 @@ def create_rotation_attachments(certificate): "title": certificate["name"], "title_link": create_certificate_url(certificate["name"]), "fields": [ + {"title": "Owner", "value": certificate["owner"], "short": True}, { - {"title": "Owner", "value": certificate["owner"], "short": True}, - { - "title": "Expires", - "value": arrow.get(certificate["validityEnd"]).format( - "dddd, MMMM D, YYYY" - ), - "short": True, - }, - { - "title": "Replaced By", - "value": len(certificate["replaced"][0]["name"]), - "short": True, - }, - { - "title": "Endpoints Rotated", - "value": len(certificate["endpoints"]), - "short": True, - }, - } + "title": "Expires", + "value": arrow.get(certificate["validityEnd"]).format( + "dddd, MMMM D, YYYY" + ), + "short": True, + }, + { + "title": "Endpoints Rotated", + "value": len(certificate["endpoints"]), + "short": True, + }, ], } @@ -96,7 +89,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): "name": "webhook", "type": "str", "required": True, - "validation": "^https:\/\/hooks\.slack\.com\/services\/.+$", + "validation": r"^https:\/\/hooks\.slack\.com\/services\/.+$", "helpMessage": "The url Slack told you to use for this integration", }, { @@ -119,6 +112,9 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): """ A typical check can be performed using the notify command: `lemur notify` + + While we receive a `targets` parameter here, it is unused, as Slack webhooks do not allow + dynamic re-targeting of messages. The webhook itself specifies a channel. """ attachments = None if notification_type == "expiration": @@ -131,7 +127,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): raise Exception("Unable to create message attachments") body = { - "text": "Lemur {0} Notification".format(notification_type.capitalize()), + "text": f"Lemur {notification_type.capitalize()} Notification", "attachments": attachments, "channel": self.get_option("recipients", options), "username": self.get_option("username", options), @@ -140,8 +136,8 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): r = requests.post(self.get_option("webhook", options), json.dumps(body)) if r.status_code not in [200]: - raise Exception("Failed to send message") + raise Exception(f"Failed to send message. Slack response: {r.status_code} {body}") - current_app.logger.error( - "Slack response: {0} Message Body: {1}".format(r.status_code, body) + current_app.logger.info( + f"Slack response: {r.status_code} Message Body: {body}" ) diff --git a/lemur/plugins/lemur_slack/tests/test_slack.py b/lemur/plugins/lemur_slack/tests/test_slack.py index 86add25f..2161b28b 100644 --- a/lemur/plugins/lemur_slack/tests/test_slack.py +++ b/lemur/plugins/lemur_slack/tests/test_slack.py @@ -1,3 +1,12 @@ +from datetime import timedelta + +import arrow +from moto import mock_ses + +from lemur.tests.factories import NotificationFactory, CertificateFactory +from lemur.tests.test_messaging import verify_sender_email + + def test_formatting(certificate): from lemur.plugins.lemur_slack.plugin import create_expiration_attachments from lemur.certificates.schemas import certificate_notification_output_schema @@ -21,3 +30,52 @@ def test_formatting(certificate): } assert attachment == create_expiration_attachments(data)[0] + + +def get_options(): + return [ + {"name": "interval", "value": 10}, + {"name": "unit", "value": "days"}, + {"name": "webhook", "value": "https://slack.com/api/api.test"}, + ] + + +@mock_ses() # because email notifications are also sent +def test_send_expiration_notification(): + from lemur.notifications.messaging import send_expiration_notifications + + verify_sender_email() # emails are sent to owner and security; Slack only used for configured notification + + notification = NotificationFactory(plugin_name="slack-notification") + notification.options = get_options() + + now = arrow.utcnow() + in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future + + certificate = CertificateFactory() + certificate.not_after = in_ten_days + certificate.notifications.append(notification) + + assert send_expiration_notifications([]) == (3, 0) # owner, Slack, and security + + +# Currently disabled as the Slack plugin doesn't support this type of notification +# def test_send_rotation_notification(endpoint, source_plugin): +# from lemur.notifications.messaging import send_rotation_notification +# from lemur.deployment.service import rotate_certificate +# +# notification = NotificationFactory(plugin_name="slack-notification") +# notification.options = get_options() +# +# new_certificate = CertificateFactory() +# rotate_certificate(endpoint, new_certificate) +# assert endpoint.certificate == new_certificate +# +# assert send_rotation_notification(new_certificate, notification_plugin=notification.plugin) + + +# Currently disabled as the Slack plugin doesn't support this type of notification +# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin): +# from lemur.notifications.messaging import send_pending_failure_notification +# +# assert send_pending_failure_notification(pending_certificate, notification_plugin=plugins.get("slack-notification")) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index fafa6f5a..be0de049 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -264,13 +264,14 @@ def create(label, plugin_name, options, description=None): return database.create(source) -def update(source_id, label, options, description): +def update(source_id, label, plugin_name, options, description): """ Updates an existing source. :param source_id: Lemur assigned ID :param label: Source common name :param options: + :param plugin_name: :param description: :rtype : Source :return: @@ -278,6 +279,7 @@ def update(source_id, label, options, description): source = get(source_id) source.label = label + source.plugin_name = plugin_name source.options = options source.description = description diff --git a/lemur/sources/views.py b/lemur/sources/views.py index b74c4d80..3b4deab7 100644 --- a/lemur/sources/views.py +++ b/lemur/sources/views.py @@ -284,6 +284,7 @@ class Sources(AuthenticatedResource): return service.update( source_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], ) diff --git a/lemur/static/app/angular/authorities/authority/authority.js b/lemur/static/app/angular/authorities/authority/authority.js index a449cff5..82f38a92 100644 --- a/lemur/static/app/angular/authorities/authority/authority.js +++ b/lemur/static/app/angular/authorities/authority/authority.js @@ -34,7 +34,7 @@ angular.module('lemur') }; }) - .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster) { + .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster, DestinationService) { $scope.authority = LemurRestangular.restangularizeElement(null, {}, 'authorities'); // set the defaults AuthorityService.getDefaults($scope.authority).then(function () { @@ -52,6 +52,12 @@ angular.module('lemur') }); }); + $scope.getDestinations = function() { + return DestinationService.findDestinationsByName('').then(function(destinations) { + $scope.destinations = destinations; + }); + }; + $scope.getAuthoritiesByName = function (value) { return AuthorityService.findAuthorityByName(value).then(function (authorities) { $scope.authorities = authorities; diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index bf1ad70c..deadf598 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -20,8 +20,10 @@ Key Type
- +
@@ -64,11 +66,28 @@
+ + + + + {{$select.selected.label}} + +
+ + + + + + +
diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index d60a1a6a..c50d40ba 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -139,7 +139,7 @@
+ Default ({{certificate.authority.defaultValidityDays}} days)
diff --git a/lemur/static/app/angular/certificates/services.js b/lemur/static/app/angular/certificates/services.js index 280d6078..be19bafd 100644 --- a/lemur/static/app/angular/certificates/services.js +++ b/lemur/static/app/angular/certificates/services.js @@ -172,12 +172,12 @@ angular.module('lemur') // Minimum end date will be same as selected start date this.authority.authorityCertificate.minValidityEnd = value; - if(!this.authority.authorityCertificate || !this.authority.authorityCertificate.maxIssuanceDays) { + if(!this.authority.maxIssuanceDays) { this.authority.authorityCertificate.maxValidityEnd = this.authority.authorityCertificate.notAfter; } else { // Move max end date by maxIssuanceDays let endDate = new Date(value); - endDate.setDate(endDate.getDate() + this.authority.authorityCertificate.maxIssuanceDays); + endDate.setDate(endDate.getDate() + this.authority.maxIssuanceDays); this.authority.authorityCertificate.maxValidityEnd = endDate; } } diff --git a/lemur/static/app/angular/destinations/destination/destination.js b/lemur/static/app/angular/destinations/destination/destination.js index 21f624c8..93a7f80e 100644 --- a/lemur/static/app/angular/destinations/destination/destination.js +++ b/lemur/static/app/angular/destinations/destination/destination.js @@ -52,19 +52,19 @@ angular.module('lemur') if (plugin.slug === $scope.destination.plugin.slug) { plugin.pluginOptions = $scope.destination.plugin.pluginOptions; $scope.destination.plugin = plugin; - _.each($scope.destination.plugin.pluginOptions, function (option) { - if (option.type === 'export-plugin') { - PluginService.getByType('export').then(function (plugins) { - $scope.exportPlugins = plugins; + PluginService.getByType('export').then(function (plugins) { + $scope.exportPlugins = plugins; + _.each($scope.destination.plugin.pluginOptions, function (option) { + if (option.type === 'export-plugin') { _.each($scope.exportPlugins, function (plugin) { if (plugin.slug === option.value.slug) { plugin.pluginOptions = option.value.pluginOptions; option.value = plugin; } }); - }); - } + } + }); }); } }); diff --git a/lemur/static/app/angular/notifications/notification/notification.js b/lemur/static/app/angular/notifications/notification/notification.js index d3cfac9b..9cf88cbf 100644 --- a/lemur/static/app/angular/notifications/notification/notification.js +++ b/lemur/static/app/angular/notifications/notification/notification.js @@ -42,8 +42,8 @@ angular.module('lemur') PluginService.getByType('notification').then(function (plugins) { $scope.plugins = plugins; _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.notification.pluginName) { - plugin.pluginOptions = $scope.notification.notificationOptions; + if (plugin.slug === $scope.notification.plugin.slug) { + plugin.pluginOptions = $scope.notification.plugin.pluginOptions; $scope.notification.plugin = plugin; } }); @@ -51,16 +51,6 @@ angular.module('lemur') NotificationService.getCertificates(notification); }); - PluginService.getByType('notification').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.notification.pluginName) { - plugin.pluginOptions = $scope.notification.notificationOptions; - $scope.notification.plugin = plugin; - } - }); - }); - $scope.save = function (notification) { NotificationService.update(notification).then( function () { diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 02443701..9e8c9b33 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -27,7 +27,7 @@ angular.module('lemur') }; NotificationService.getCertificates = function (notification) { - notification.getList('certificates').then(function (certificates) { + notification.getList('certificates', {showExpired: 0}).then(function (certificates) { notification.certificates = certificates; }); }; @@ -40,7 +40,7 @@ angular.module('lemur') NotificationService.loadMoreCertificates = function (notification, page) { - notification.getList('certificates', {page: page}).then(function (certificates) { + notification.getList('certificates', {page: page, showExpired: 0}).then(function (certificates) { _.each(certificates, function (certificate) { notification.roles.push(certificate); }); diff --git a/lemur/static/app/angular/pending_certificates/services.js b/lemur/static/app/angular/pending_certificates/services.js index 9b32c1d3..7f20355b 100644 --- a/lemur/static/app/angular/pending_certificates/services.js +++ b/lemur/static/app/angular/pending_certificates/services.js @@ -152,12 +152,12 @@ angular.module('lemur') // Minimum end date will be same as selected start date this.authority.authorityCertificate.minValidityEnd = value; - if(!this.authority.authorityCertificate || !this.authority.authorityCertificate.maxIssuanceDays) { + if(!this.authority.maxIssuanceDays) { this.authority.authorityCertificate.maxValidityEnd = this.authority.authorityCertificate.notAfter; } else { // Move max end date by maxIssuanceDays let endDate = new Date(value); - endDate.setDate(endDate.getDate() + this.authority.authorityCertificate.maxIssuanceDays); + endDate.setDate(endDate.getDate() + this.authority.maxIssuanceDays); this.authority.authorityCertificate.maxValidityEnd = endDate; } } diff --git a/lemur/static/app/angular/sources/source/source.js b/lemur/static/app/angular/sources/source/source.js index 1d5c1641..8ea381f8 100644 --- a/lemur/static/app/angular/sources/source/source.js +++ b/lemur/static/app/angular/sources/source/source.js @@ -41,22 +41,14 @@ angular.module('lemur') PluginService.getByType('source').then(function (plugins) { $scope.plugins = plugins; _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.source.pluginName) { + if (plugin.slug === $scope.source.plugin.slug) { + plugin.pluginOptions = $scope.source.plugin.pluginOptions; $scope.source.plugin = plugin; } }); }); }); - PluginService.getByType('source').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.source.pluginName) { - $scope.source.plugin = plugin; - } - }); - }); - $scope.save = function (source) { SourceService.update(source).then( function () { diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index 38b8bade..3dfb5621 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -46,7 +46,7 @@ LEMUR_ALLOWED_DOMAINS = [ # Lemur currently only supports SES for sending email, this address # needs to be verified -LEMUR_EMAIL = "" +LEMUR_EMAIL = "lemur@example.com" LEMUR_SECURITY_TEAM_EMAIL = ["security@example.com"] LEMUR_HOSTNAME = "lemur.example.com" diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index c271a97e..583022eb 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -802,6 +802,7 @@ def test_reissue_certificate( assert new_cert.organization != certificate.organization # Check for default value since authority does not have cab_compliant option set assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION + assert new_cert.description.startswith(f"Reissued by Lemur for cert ID {certificate.id}") # update cab_compliant option to false for crypto_authority to maintain subject details update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]') diff --git a/lemur/tests/test_dns_providers.py b/lemur/tests/test_dns_providers.py index 83315be5..9b8fdb5a 100644 --- a/lemur/tests/test_dns_providers.py +++ b/lemur/tests/test_dns_providers.py @@ -13,7 +13,7 @@ class TestDNSProvider(unittest.TestCase): self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com')) self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com')) self.assertFalse(dnsutil.is_valid_domain('e/xample.com')) - self.assertFalse(dnsutil.is_valid_domain('exam\ple.com')) + self.assertFalse(dnsutil.is_valid_domain('exam\\ple.com')) self.assertFalse(dnsutil.is_valid_domain('