Creation of snapshots using AWS Lambda

Question

I tried using the below script to create snapshots for instances having the tag name [Backup or backup] according to https://serverlesscode.com/post/lambda-schedule-ebs-snapshot-backups/ && https://serverlesscode.com/post/lambda-schedule-ebs-snapshot-backups-2/

import boto3
import collections
import datetime

ec = boto3.client('ec2')

def lambda_handler(event, context):
    reservations = ec.describe_instances(
        Filters=[
            {'Name': 'tag-key', 'Values': ['backup', 'Backup']},
        ]
    ).get(
        'Reservations', []
    )

instances = sum(
    [
        [i for i in r['Instances']]
        for r in reservations
    ], [])

print "Found %d instances that need backing up" % len(instances)

to_tag = collections.defaultdict(list)

for instance in instances:
    try:
        retention_days = [
            int(t.get('Value')) for t in instance['Tags']
            if t['Key'] == 'Retention'][0]
    except IndexError:
        retention_days = 7

    for dev in instance['BlockDeviceMappings']:
        if dev.get('Ebs', None) is None:
            continue
        vol_id = dev['Ebs']['VolumeId']
        print "Found EBS volume %s on instance %s" % (
            vol_id, instance['InstanceId'])

        snap = ec.create_snapshot(
            VolumeId=vol_id,
        )

        to_tag[retention_days].append(snap['SnapshotId'])

        print "Retaining snapshot %s of volume %s from instance %s for %d days" % (
            snap['SnapshotId'],
            vol_id,
            instance['InstanceId'],
            retention_days,
        )


for retention_days in to_tag.keys():
    delete_date = datetime.date.today() + datetime.timedelta(days=retention_days)
    delete_fmt = delete_date.strftime('%Y-%m-%d')
    print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt)
    ec.create_tags(
        Resources=to_tag[retention_days],
        Tags=[
            {'Key': 'DeleteOn', 'Value': delete_fmt},
        ]
    )

The code worked well but returned a null response saying:

Found 0 instances that need backing up 

I have two instances in my console running with the tag name Backup. Then, what could be the reason behind the null response?

Answer 1

I have solved it. I was assigning my tags in the wrong way till now.

Answer 2

I had a similar issue and I've solved it with a slight different filtering:

    Filters=[
        {'Name': 'tag:toBackup', 'Values': ['yes', 'Yes']},
    ]

And for each of my instances I want to backup I set a tag which Key field is set to toBackup and Value field set either to yes or Yes .

Inspired from this answer by mootmoot which point to Russell Ballestrini blog Filtering AWS resources with Boto3 .

Answer 3

In the Lambda console, navigate through Functions > Create a Lambda Function -> Configure function . Then, use the following parameters: Name, Description, Runtime. I used the Boto library that is in the AWS SDK for Python for the code below:

    # Backup all in-use volumes in all regions


import boto3

def lambda_handler(event, context):
    ec2 = boto3.client('ec2')

    # Get list of regions
    regions = ec2.describe_regions().get('Regions',[] )

    # Iterate over regions
    for region in regions:
        print "Checking region %s " % region['RegionName']
        reg=region['RegionName']

        # Connect to region
        ec2 = boto3.client('ec2', region_name=reg)

        # Get all in-use volumes in all regions 
        result = ec2.describe_volumes( Filters=[{'Name': 'status', 'Values': ['in-use']}])

        for volume in result['Volumes']:
            print "Backing up %s in %s" % (volume['VolumeId'], volume['AvailabilityZone'])

            # Create snapshot
            result = ec2.create_snapshot(VolumeId=volume['VolumeId'],Description='Created by Lambda backup function ebs-snapshots')

            # Get snapshot resource
            ec2resource = boto3.resource('ec2', region_name=reg)
            snapshot = ec2resource.Snapshot(result['SnapshotId'])

            volumename = 'N/A'

            # Find name tag for volume if it exists
            if 'Tags' in volume:
                for tags in volume['Tags']:
                    if tags["Key"] == 'Name':
                        volumename = tags["Value"]

            # Add volume name to snapshot for easier identification
            snapshot.create_tags(Tags=[{'Key': 'Name','Value': volumename}])

It will create snapshots for any volume across all regions. Moreover, it will add the name of the volume to the created snapshot name tag. So, it will be easier to identify to view the list of snapshots.

Answer 4

I got this completely working. It will tag snapshot corresponding to their Instance_ID and Instance_Name. Just copy the lambda function and tag your EC2 accordingly.

#Tag to folllow
#Retention    number of days here
#backup
#backup-monthly

import boto3
import collections
import datetime

ec = boto3.client('ec2')

def lambda_handler(event, context):
    reservations = ec.describe_instances(
        Filters=[
            {'Name': 'tag-key', 'Values': ['backup', 'Backup']},
            # Uncomment this line if need to take snaphsot of running instances only
            # {'Name': 'instance-state-name', 'Values': ['running']},
        ]
    ).get(
        'Reservations', []
    )

    instances = sum(
        [
            [i for i in r['Instances']]
            for r in reservations
        ], [])

    print "Found %d instances that need backing up" % len(instances)

    to_tag = collections.defaultdict(list)

    for instance in instances:
        try:
            retention_days = [
                int(t.get('Value')) for t in instance['Tags']
                if t['Key'] == 'Retention'][0]
        except IndexError:
            retention_days = 7

        for dev in instance['BlockDeviceMappings']:
            if dev.get('Ebs', None) is None:
                continue
            vol_id = dev['Ebs']['VolumeId']
            print "Found EBS volume %s on instance %s" % (
                vol_id, instance['InstanceId'])

            instance_id = instance['InstanceId']

            snapshot_name = 'N/A'
            if 'Tags' in instance:
                for tags in instance['Tags']:
                    if tags["Key"] == 'Name':
                        snapshot_name = tags["Value"]

            print "Tagging snapshot with Name: {} and Instance ID {}".format(snapshot_name, instance_id)

            snap = ec.create_snapshot(
                Description = 'Instance ID is {} and Snapshot taken by Lambda script'.format(instance_id),
                VolumeId = vol_id,
                TagSpecifications = [{
                    'ResourceType': 'snapshot',
                    'Tags': [{
                        'Key': 'Name',
                        'Value': snapshot_name
                    }, ]
                }, ]
                # DryRun = False
                )

            to_tag[retention_days].append(snap['SnapshotId'])

            print "Retaining snapshot %s of volume %s from instance %s for %d days" % (
                snap['SnapshotId'],
                vol_id,
                instance['InstanceId'],
                retention_days,
            )


    for retention_days in to_tag.keys():
        delete_date = datetime.date.today() + datetime.timedelta(days=retention_days)
        delete_fmt = delete_date.strftime('%Y-%m-%d')
        print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt)
        ec.create_tags(
            Resources=to_tag[retention_days],
            Tags=[
                {'Key': 'DeleteOn', 'Value': delete_fmt},
            ]
        )

Answer 5

Few things need to change. For python version 3.7, 3.6 or 3.8 you may be try below code.

import boto3
import collections
import datetime
 
ec = boto3.client('ec2')
 
def lambda_handler(event, context):
    reservations = ec.describe_instances(
        Filters=[
            {'Name': 'tag-key', 'Values': ['backup', 'Backup']},
        ]
    ).get(
        'Reservations', []
    )
 
    instances = sum(
        [
            [i for i in r['Instances']]
            for r in reservations
        ], [])
 
    print("Found {num_of_instances} instances that need backing up".format(num_of_instances=len(instances)))
 
    to_tag = collections.defaultdict(list)
 
    for instance in instances:
        try:
            retention_days = [
                int(t.get('Value')) for t in instance['Tags']
                if t['Key'] == 'Retention'][0]
        except IndexError:
            retention_days = 10
 
        for dev in instance['BlockDeviceMappings']:
            if dev.get('Ebs', None) is None:
                continue
            vol_id = dev['Ebs']['VolumeId']
            print("Found EBS volume {} on instance {}".format(vol_id, instance['InstanceId']))
 
            snap = ec.create_snapshot(
                VolumeId=vol_id,
            )
 
            to_tag[retention_days].append(snap['SnapshotId'])
 
            print("Retaining snapshot {} of volume {} from instance {} for {} days".format(
                snap['SnapshotId'],
                vol_id,
                instance['InstanceId'],
                retention_days,
            ))
 
 
    for retention_days in to_tag.keys():
        delete_date = datetime.date.today() + datetime.timedelta(days=retention_days)
        delete_fmt = delete_date.strftime('%Y-%m-%d')
        print("Will delete {} snapshots on {}".format(len(to_tag[retention_days]), delete_fmt))
        ec.create_tags(
            Resources=to_tag[retention_days],
            Tags=[
                {'Key': 'DeleteOn', 'Value': delete_fmt},
                {'Key': 'Name', 'Value': "LIVE-BACKUP"}
            ]
        )

It works for me.