From f846d78778a350669a3edf7152b5bf95ef1a7209 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Mon, 27 Jun 2016 15:11:46 -0700 Subject: [PATCH] S3 destination (#371) --- lemur/plugins/lemur_aws/plugin.py | 113 ++++++++++++++++++++++- lemur/plugins/lemur_aws/s3.py | 26 ++++++ lemur/plugins/lemur_aws/sts.py | 7 ++ lemur/plugins/lemur_aws/tests/test_s3.py | 11 +++ 4 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 lemur/plugins/lemur_aws/s3.py create mode 100644 lemur/plugins/lemur_aws/tests/test_s3.py diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 7eaa2d25..91bced7d 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -4,15 +4,41 @@ :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more :license: Apache, see LICENSE for more details. + Terraform example to setup the destination bucket: + resource "aws_s3_bucket" "certs_log_bucket" { + bucket = "certs-log-access-bucket" + acl = "log-delivery-write" + } + + resource "aws_s3_bucket" "certs_lemur" { + bucket = "certs-lemur" + acl = "private" + + logging { + target_bucket = "${aws_s3_bucket.certs_log_bucket.id}" + target_prefix = "log/lemur" + } + } + + The IAM role Lemur is running as should have the following actions on the destination bucket: + + "S3:PutObject", + "S3:PutObjectAcl" + + The reader should have the following actions: + "s3:GetObject" + .. moduleauthor:: Kevin Glisson +.. moduleauthor:: Mikhail Khodorovskiy +.. moduleauthor:: Harm Weites """ from flask import current_app from boto.exception import BotoServerError from lemur.plugins.bases import DestinationPlugin, SourcePlugin -from lemur.plugins.lemur_aws import iam from lemur.plugins.lemur_aws.ec2 import get_regions from lemur.plugins.lemur_aws.elb import get_all_elbs, describe_load_balancer_policies, attach_certificate +from lemur.plugins.lemur_aws import iam, s3 from lemur.plugins import lemur_aws as aws @@ -34,6 +60,7 @@ class AWSDestinationPlugin(DestinationPlugin): 'helpMessage': 'Must be a valid AWS account number!', } ] + # 'elb': { # 'name': {'type': 'name'}, # 'region': {'type': 'str'}, @@ -43,7 +70,8 @@ class AWSDestinationPlugin(DestinationPlugin): def upload(self, name, body, private_key, cert_chain, options, **kwargs): if private_key: try: - iam.upload_cert(self.get_option('accountNumber', options), name, body, private_key, cert_chain=cert_chain) + iam.upload_cert(self.get_option('accountNumber', options), name, body, private_key, + cert_chain=cert_chain) except BotoServerError as e: if e.error_code != 'EntityAlreadyExists': raise Exception(e) @@ -146,3 +174,84 @@ def format_elb_cipher_policy(policy): ciphers.append(attr['AttributeName']) return dict(name=name, ciphers=ciphers) + + +class S3DestinationPlugin(DestinationPlugin): + title = 'AWS-S3' + slug = 'aws-s3' + description = 'Allow the uploading of certificates to Amazon S3' + + author = 'Mikhail Khodorovskiy, Harm Weites ' + author_url = 'https://github.com/Netflix/lemur' + + options = [ + { + 'name': 'bucket', + 'type': 'str', + 'required': True, + 'validation': '/^$|\s+/', + 'helpMessage': 'Must be a valid S3 bucket name!', + }, + { + 'name': 'accountNumber', + 'type': 'int', + 'required': True, + 'validation': '/^[0-9]{12,12}$/', + 'helpMessage': 'A valid AWS account number with permission to access S3', + }, + { + 'name': 'region', + 'type': 'str', + 'default': 'eu-west-1', + 'required': False, + 'validation': '/^\w+-\w+-\d+$/', + 'helpMessage': 'Availability zone to use', + }, + { + 'name': 'encrypt', + 'type': 'bool', + 'required': False, + 'helpMessage': 'Availability zone to use', + 'default': True + }, + { + 'name': 'key', + 'type': 'str', + 'required': False, + 'validation': '/^$|\s+/', + 'helpMessage': 'Must be a valid S3 object key!', + }, + { + 'name': 'caKey', + 'type': 'str', + 'required': False, + 'validation': '/^$|\s+/', + 'helpMessage': 'Must be a valid S3 object key!', + }, + { + 'name': 'certKey', + 'type': 'str', + 'required': False, + 'validation': '/^$|\s+/', + 'helpMessage': 'Must be a valid S3 object key!', + } + ] + + def __init__(self, *args, **kwargs): + super(S3DestinationPlugin, self).__init__(*args, **kwargs) + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + account_number = self.get_option('accountId', options) + encrypt = self.get_option('encrypt', options) + bucket = self.get_option('bucket', options) + key = self.get_option('key', options) + ca_key = self.get_option('caKey', options) + cert_key = self.get_option('certKey', options) + + if key and ca_key and cert_key: + s3.write_to_s3(account_number, bucket, key, private_key, encrypt=encrypt) + s3.write_to_s3(account_number, bucket, ca_key, cert_chain, encrypt=encrypt) + s3.write_to_s3(account_number, bucket, cert_key, body, encrypt=encrypt) + else: + pem_body = key + '\n' + body + '\n' + cert_chain + '\n' + s3.write_to_s3(account_number, bucket, name, pem_body, encrypt=encrypt) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py new file mode 100644 index 00000000..0f445e43 --- /dev/null +++ b/lemur/plugins/lemur_aws/s3.py @@ -0,0 +1,26 @@ +""" +.. module: lemur.plugins.lemur_aws.s3 + :platform: Unix + :synopsis: Contains helper functions for interactive with AWS S3 Apis. + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from boto.s3.key import Key +from lemur.plugins.lemur_aws.sts import assume_service + + +def write_to_s3(account_number, bucket_name, key, data, encrypt=True): + """ + Use STS to write to an S3 bucket + + :param account_number: + :param bucket_name: + :param data: + """ + conn = assume_service(account_number, 's3') + b = conn.get_bucket(bucket_name, validate=False) # validate=False removes need for ListObjects permission + + k = Key(bucket=b, name=key) + k.set_contents_from_string(data, encrypt_key=encrypt) + k.set_canned_acl("bucket-owner-read") diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index ab58b776..7d392fa6 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -39,6 +39,13 @@ def assume_service(account_number, service, region='us-east-1'): aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token) + elif service in 's3': + return boto.s3.connect_to_region( + region, + aws_access_key_id=role.credentials.access_key, + aws_secret_access_key=role.credentials.secret_key, + security_token=role.credentials.session_token) + def sts_client(service, service_type='client'): def decorator(f): diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py new file mode 100644 index 00000000..fb6142bd --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -0,0 +1,11 @@ +from moto import mock_s3 +import boto + + +@mock_s3() +def test_get_name_from_arn(): + conn = boto.connect_s3() + conn.create_bucket('test') + from lemur.plugins.lemur_aws.s3 import write_to_s3 + write_to_s3('11111111111111', 'test', 'key', 'body') + assert conn.get_bucket('test').get_key('key').get_contents_as_string() == 'body'