Merge branch 'master' into bugfix/group-lookup-fix-referral

lemur/acme_providers/cli.py

@@ -1,12 +1,15 @@
 import time
 import json
+import arrow
 
 from flask_script import Manager
 from flask import current_app
 
 from lemur.extensions import sentry
 from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins import plugins
 from lemur.plugins.lemur_acme.plugin import AcmeHandler
+from lemur.plugins.lemur_aws import s3
 
 manager = Manager(
     usage="Handles all ACME related tasks"
@@ -84,3 +87,105 @@ def dnstest(domain, token):
 
     status = SUCCESS_METRIC_STATUS
     print("[+] Done with ACME Tests.")
+
+
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="Value of the Token",
+)
+@manager.option(
+    "-n",
+    "--token_name",
+    dest="token_name",
+    default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="path",
+)
+@manager.option(
+    "-p",
+    "--prefix",
+    dest="prefix",
+    default="test/",
+    required=False,
+    help="S3 bucket prefix",
+)
+@manager.option(
+    "-a",
+    "--account_number",
+    dest="account_number",
+    required=True,
+    help="AWS Account",
+)
+@manager.option(
+    "-b",
+    "--bucket_name",
+    dest="bucket_name",
+    required=True,
+    help="Bucket Name",
+)
+def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
+    """
+    This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
+    It mainly serves for testing purposes.
+    :param token:
+    :param token_name:
+    :param prefix:
+    :param account_number:
+    :param bucket_name:
+    :return:
+    """
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket_name,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account_number,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    p = plugins.get("aws-s3")
+    p.upload_acme_token(token_name, token, additional_options)
+
+    if not prefix.endswith("/"):
+        prefix + "/"
+
+    token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
+    assert(token_res == token)
+    s3.delete(bucket_name, prefix + token_name, account_number=account_number)

lemur/plugins/lemur_aws/plugin.py

@@ -33,6 +33,7 @@
 .. moduleauthor:: Harm Weites <harm@weites.com>
 """
 
+import sys
 from acme.errors import ClientError
 from flask import current_app
 
@@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin):
                 account_number=self.get_option("accountNumber", options),
             )
 
+    def upload_acme_token(self, token_path, token, options, **kwargs):
+        """
+         This is called from the acme http challenge
+        :param self:
+        :param token_path:
+        :param token:
+        :param options:
+        :param kwargs:
+        :return:
+        """
+        current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
+
+        function = f"{__name__}.{sys._getframe().f_code.co_name}"
+
+        account_number = self.get_option("accountNumber", options)
+        bucket_name = self.get_option("bucket", options)
+        prefix = self.get_option("prefix", options)
+        region = self.get_option("region", options)
+        filename = token_path.split("/")[-1]
+        if not prefix.endswith("/"):
+            prefix + "/"
+
+        res = s3.put(bucket_name=bucket_name,
+                     region_name=region,
+                     prefix=prefix + filename,
+                     data=token,
+                     encrypt=False,
+                     account_number=account_number)
+        res = "Success" if res else "Failure"
+        log_data = {
+            "function": function,
+            "message": "check if any valid certificate is revoked",
+            "result": res,
+            "bucket_name": bucket_name,
+            "filename": filename
+        }
+        current_app.logger.info(log_data)
+        metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
+                                                               "bucket_name": bucket_name,
+                                                               "filename": filename})
+
 
 class SNSNotificationPlugin(ExpirationNotificationPlugin):
     title = "AWS SNS"

lemur/plugins/lemur_aws/s3.py

@@ -6,12 +6,15 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+from botocore.exceptions import ClientError
 from flask import current_app
+from lemur.extensions import sentry
+
 from .sts import sts_client
 
 
 @sts_client("s3", service_type="resource")
-def put(bucket_name, region, prefix, data, encrypt, **kwargs):
+def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
     """
     Use STS to write to an S3 bucket
     """
@@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
             ServerSideEncryption="AES256",
         )
     else:
-        bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
+        try:
+            bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
+            return True
+        except ClientError:
+            sentry.captureException()
+            return False
+
+
+@sts_client("s3", service_type="client")
+def delete(bucket_name, prefixed_object_name, **kwargs):
+    """
+    Use STS to delete an object
+    """
+    try:
+        response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
+        current_app.logger.debug(f"Delete data from S3."
+                                 f"Bucket: {bucket_name},"
+                                 f"Prefix: {prefixed_object_name},"
+                                 f"Status_code: {response}")
+        return response['ResponseMetadata']['HTTPStatusCode'] < 300
+    except ClientError:
+        sentry.captureException()
+        return False
+
+
+@sts_client("s3", service_type="client")
+def get(bucket_name, prefixed_object_name, **kwargs):
+    """
+    Use STS to get an object
+    """
+    try:
+        response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
+        current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
+                                 f"object_name: {prefixed_object_name}")
+        return response['Body'].read().decode("utf-8")
+    except ClientError:
+        sentry.captureException()
+        return None

lemur/plugins/lemur_aws/tests/test_plugin.py

@@ -1,5 +1,82 @@
+import boto3
+from moto import mock_sts, mock_s3
+
+
 def test_get_certificates(app):
     from lemur.plugins.base import plugins
 
     p = plugins.get("aws-s3")
     assert p
+
+
+@mock_sts()
+@mock_s3()
+def test_upload_acme_token(app):
+    from lemur.plugins.base import plugins
+    from lemur.plugins.lemur_aws.s3 import get
+
+    bucket = "public-bucket"
+    account = "123456789012"
+    prefix = "some-path/more-path/"
+    token_content = "Challenge"
+    token_name = "TOKEN"
+    token_path = ".well-known/acme-challenge/" + token_name
+
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=bucket)
+    p = plugins.get("aws-s3")
+
+    p.upload_acme_token(token_path=token_path,
+                        token_content=token_content,
+                        token=token_content,
+                        options=additional_options)
+
+    response = get(bucket_name=bucket,
+                   prefixed_object_name=prefix + token_name,
+                   encrypt=False,
+                   account_number=account)
+
+    # put data, and getting the same data
+    assert (response == token_content)

lemur/plugins/lemur_aws/tests/test_s3.py

@@ -0,0 +1,41 @@
+import boto3
+from moto import mock_sts, mock_s3
+
+
+@mock_sts()
+@mock_s3()
+def test_put_delete_s3_object(app):
+    from lemur.plugins.lemur_aws.s3 import put, delete, get
+
+    bucket = "public-bucket"
+    region = "us-east-1"
+    account = "123456789012"
+    path = "some-path/foo"
+    data = "dummy data"
+
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=bucket)
+
+    put(bucket_name=bucket,
+        region_name=region,
+        prefix=path,
+        data=data,
+        encrypt=False,
+        account_number=account,
+        region=region)
+
+    response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+
+    # put data, and getting the same data
+    assert (response == data)
+
+    response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
+
+    # attempting to get thccle wrong data
+    assert (response is None)
+
+    delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+    response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+
+    # delete data, and getting the same data
+    assert (response is None)

requirements-dev.txt

@@ -6,7 +6,7 @@
 #
 appdirs==1.4.3            # via virtualenv
 bleach==3.1.4             # via readme-renderer
-certifi==2020.6.20        # via requests
+certifi==2020.11.8        # via requests
 cffi==1.14.0              # via cryptography
 cfgv==3.1.0               # via pre-commit
 chardet==3.0.4            # via requests
@@ -15,9 +15,9 @@ cryptography==3.2.1       # via secretstorage
 distlib==0.3.0            # via virtualenv
 docutils==0.16            # via readme-renderer
 filelock==3.0.12          # via virtualenv
+flake8==3.8.4             # via -r requirements-dev.in
 identify==1.4.14          # via pre-commit
 idna==2.9                 # via requests
-flake8==3.8.4             # via -r requirements-dev.in
 invoke==1.4.1             # via -r requirements-dev.in
 jeepney==0.4.3            # via keyring, secretstorage
 keyring==21.2.0           # via twine

requirements-docs.txt

@@ -17,14 +17,14 @@ bcrypt==3.1.7             # via -r requirements.txt, flask-bcrypt, paramiko
 beautifulsoup4==4.9.1     # via -r requirements.txt, cloudflare
 billiard==3.6.3.0         # via -r requirements.txt, celery
 blinker==1.4              # via -r requirements.txt, flask-mail, flask-principal, raven
-boto3==1.16.9             # via -r requirements.txt
+boto3==1.16.14            # via -r requirements.txt
-botocore==1.19.9          # via -r requirements.txt, boto3, s3transfer
+botocore==1.19.14         # via -r requirements.txt, boto3, s3transfer
 celery[redis]==4.4.2      # via -r requirements.txt
-certifi==2020.6.20        # via -r requirements.txt, requests
+certifi==2020.11.8        # via -r requirements.txt, requests
 certsrv==2.1.1            # via -r requirements.txt
 cffi==1.14.0              # via -r requirements.txt, bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements.txt, requests
-click==7.1.1              # via -r requirements.txt, flask
+click==7.1.2              # via -r requirements.txt, flask
 cloudflare==2.8.13        # via -r requirements.txt
 cryptography==3.2.1       # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests
 dnspython3==1.15.0        # via -r requirements.txt

requirements-tests.txt

@@ -10,10 +10,10 @@ aws-sam-translator==1.22.0  # via cfn-lint
 aws-xray-sdk==2.5.0       # via moto
 bandit==1.6.2             # via -r requirements-tests.in
 black==20.8b1             # via -r requirements-tests.in
-boto3==1.16.9             # via aws-sam-translator, moto
+boto3==1.16.14            # via aws-sam-translator, moto
 boto==2.49.0              # via moto
-botocore==1.19.9          # via aws-xray-sdk, boto3, moto, s3transfer
+botocore==1.19.14         # via aws-xray-sdk, boto3, moto, s3transfer
-certifi==2020.6.20        # via requests
+certifi==2020.11.8        # via requests
 cffi==1.14.0              # via cryptography
 cfn-lint==0.29.5          # via moto
 chardet==3.0.4            # via requests
@@ -24,7 +24,7 @@ decorator==4.4.2          # via networkx
 docker==4.2.0             # via moto
 ecdsa==0.14.1             # via moto, python-jose, sshpubkeys
 factory-boy==3.1.0        # via -r requirements-tests.in
-faker==4.14.0             # via -r requirements-tests.in, factory-boy
+faker==4.14.2             # via -r requirements-tests.in, factory-boy
 fakeredis==1.4.4          # via -r requirements-tests.in
 flask==1.1.2              # via pytest-flask
 freezegun==1.0.0          # via -r requirements-tests.in
@@ -59,7 +59,7 @@ pycparser==2.20           # via cffi
 pyflakes==2.2.0           # via -r requirements-tests.in
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema
-pytest-flask==1.0.0       # via -r requirements-tests.in
+pytest-flask==1.1.0       # via -r requirements-tests.in
 pytest-mock==3.3.1        # via -r requirements-tests.in
 pytest==6.1.2             # via -r requirements-tests.in, pytest-flask, pytest-mock
 python-dateutil==2.8.1    # via botocore, faker, freezegun, moto

requirements.txt

@@ -15,14 +15,14 @@ bcrypt==3.1.7             # via flask-bcrypt, paramiko
 beautifulsoup4==4.9.1     # via cloudflare
 billiard==3.6.3.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.16.10            # via -r requirements.in
+boto3==1.16.14            # via -r requirements.in
-botocore==1.19.10         # via -r requirements.in, boto3, s3transfer
+botocore==1.19.14         # via -r requirements.in, boto3, s3transfer
 celery[redis]==4.4.2      # via -r requirements.in
-certifi==2020.6.20        # via -r requirements.in, requests
+certifi==2020.11.8        # via -r requirements.in, requests
 certsrv==2.1.1            # via -r requirements.in
 cffi==1.14.0              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
-click==7.1.2              # black 20.8b1 has requirement click>=7.1.2
+click==7.1.2              # via flask
 cloudflare==2.8.13        # via -r requirements.in
 cryptography==3.2.1       # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests
 dnspython3==1.15.0        # via -r requirements.in