AWS Lambda Function - S3 Bucket Automation

Table of Contents

AWS Lambda Function
#

Prerequisites
#

# I use the following S3 Buckets for this tutorial
S3 Bucket1: jkw-bucket-1
S3 Bucket1: jkw-bucket-2

Move Files to another S3 Bucket
#

The following Lambda function moves files from the “Input” folder of jkw-bucket-1 to the “Output” folder of jkw-bucket-2 and then deletes the file from the “Input folder”.

Create the following IAM Policy

{
	"Version": "2012-10-17",
	"Statement": [
		{
			"Sid": "VisualEditor0",
			"Effect": "Allow",
			"Action": [
				"s3:GetObject",
				"s3:DeleteObject"
			],
			"Resource": "arn:aws:s3:::jkw-bucket-1/Input/*"
		},
		{
			"Sid": "VisualEditor1",
			"Effect": "Allow",
			"Action": "s3:ListBucket",
			"Resource": [
				"arn:aws:s3:::jkw-bucket-1",
				"arn:aws:s3:::jkw-bucket-1/Input/*"
			]
		},
		{
			"Sid": "VisualEditor2",
			"Effect": "Allow",
			"Action": "s3:PutObject",
			"Resource": "arn:aws:s3:::jkw-bucket-2/*"
		}
	]
}

Create a new Role for Lambda

Add the new Policy to the Role

Create a new Lambda function

Link: https://eu-central-1.console.aws.amazon.com/lambda/

Add Python script

import boto3
import json

s3 = boto3.resource('s3')

def lambda_handler(event, context):
    bucket = s3.Bucket('jkw-bucket-1')
    dest_bucket = s3.Bucket('jkw-bucket-2')

    print(dest_bucket)
    print(bucket)
    
    for obj in bucket.objects.filter(Prefix='Input/', Delimiter='/'):
        dest_key = 'Output/' + obj.key.split('/', 1)[-1]
        print(dest_key)
        print('copy file ' + dest_key)
        s3.Object(dest_bucket.name, dest_key).copy_from(CopySource={'Bucket': obj.bucket_name, 'Key': obj.key})

        if obj.key != 'Input/':
            print('delete file from source bucket ' + dest_key)
            s3.Object(bucket.name, obj.key).delete()

Create a trigger for the Lambda function

Move files to folder
#

This Lambda function movies files from the “Input” to the “Output” folder in jkw-bucket-1.

IAM Policy

{
	"Version": "2012-10-17",
	"Statement": [
		{
			"Sid": "VisualEditor0",
			"Effect": "Allow",
			"Action": [
				"s3:GetObject",
				"s3:DeleteObject"
			],
			"Resource": "arn:aws:s3:::jkw-bucket-1/Input/*"
		},
		{
			"Sid": "VisualEditor1",
			"Effect": "Allow",
			"Action": "s3:ListBucket",
			"Resource": [
				"arn:aws:s3:::jkw-bucket-1",
				"arn:aws:s3:::jkw-bucket-1/Input/*"
			]
		},
		{
			"Sid": "VisualEditor2",
			"Effect": "Allow",
			"Action": "s3:PutObject",
			"Resource": "arn:aws:s3:::jkw-bucket-1/*"
		}
	]
}

Python script

import boto3
import json

s3 = boto3.resource('s3')

def lambda_handler(event, context):
    bucket = s3.Bucket('jkw-bucket-1')

    print(bucket)
    
    for obj in bucket.objects.filter(Prefix='Input/', Delimiter='/'):
        dest_key = 'Output/' + obj.key.split('/', 1)[-1]
        print(dest_key)
        print('copy file to ' + dest_key)
        s3.Object(bucket.name, dest_key).copy_from(CopySource={'Bucket': obj.bucket_name, 'Key': obj.key})

        if obj.key != 'Input/':
            print('delete file from source bucket ' + dest_key)
            s3.Object(bucket.name, obj.key).delete()

Move specific files
#

This Lambda function moves files that contain “01a” in their file name from the “Input” to the “Output” folder in jkw-bucket-1.

Python script

import boto3
import json

s3 = boto3.resource('s3')

def lambda_handler(event, context):
    bucket = s3.Bucket('jkw-bucket-1')

    print(bucket)
    
    for obj in bucket.objects.filter(Prefix='Input/', Delimiter='/'):
        if "01a" in obj.key:
            dest_key = 'Output/' + obj.key.split('/', 1)[-1]
            print(dest_key)
            print('copy file to ' + dest_key)
            s3.Object(bucket.name, dest_key).copy_from(CopySource={'Bucket': obj.bucket_name, 'Key': obj.key})
            
            if obj.key != 'Input/':
                print('delete file from source bucket ' + dest_key)
                s3.Object(bucket.name, obj.key).delete()

Move specific files (Loop Version)
#

This Lambda function moves files that contain “01a” to folder “Output-01a” and files that contain “02b” to folder “Output-02b”

Python script

import boto3
import json

s3 = boto3.resource('s3')

def lambda_handler(event, context):
    bucket = s3.Bucket('jkw-bucket-1')

    # Folder Mapping
    folder_map = {
        "01a": "Output-01a/",
        "02b": "Output-02b/"
    }

    for obj in bucket.objects.filter(Prefix='Input/', Delimiter='/'):
        for identifier, folder_name in folder_map.items():
            if identifier in obj.key:
                dest_key = folder_name + obj.key.split('/', 1)[-1]
                print(dest_key)
                print(f'copy file to {dest_key}')
                s3.Object(bucket.name, dest_key).copy_from(CopySource={'Bucket': obj.bucket_name, 'Key': obj.key})
                
                if obj.key != 'Input/':
                    print(f'delete file from source bucket {dest_key}')
                    s3.Object(bucket.name, obj.key).delete()
                break