Python function returns inconsistent results

Question

I wrote a script that lists EC2 instances in Amazon Web Services. It writes the results to confluence. But it's behaving oddly.

I'm on windows 10. The first time I open a powershell terminal and run it it reports the correct number of servers in the AWS account.

The next time I run it (without changing anything at all) it doubles the resutls. And each time you run it again with the same arguments it reports the same incorrect (doubled) amount.

This is the function that lists the instances and this is where I think the trouble is, but I'm having trouble finding it:

def list_instances(aws_account, aws_account_number, interactive, regions, fieldnames, show_details):
    today, aws_env_list, output_file, output_file_name, fieldnames = initialize(interactive, aws_account)     options = arguments()     instance_list = ''     session = ''     ec2 = ''     account_found = ''     PrivateDNS = None     block_device_list = None     instance_count = 0     account_type_message = ''     profile_missing_message = ''     region = ''
    # Set the ec2 dictionary
    ec2info = {}
    # Write the file headers
    if interactive == 1:
        with open(output_file, mode='w+') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames=fieldnames, delimiter=',', lineterminator='\n')
            writer.writeheader()
            if 'gov' in aws_account and not 'admin' in aws_account:
                try:
                    session = boto3.Session(profile_name=aws_account,region_name=region)
                    account_found = 'yes'
                except botocore.exceptions.ProfileNotFound as e:
                    message = f"An exception has occurred: {e}"
                    account_found = 'no'
                    banner(message)
            else:
                try:
                    session = boto3.Session(profile_name=aws_account,region_name=region)
                    account_found = 'yes'
                except botocore.exceptions.ProfileNotFound as e:
                    message = f"An exception has occurred: {e}"
                    account_found = 'no'
                    banner(message)
    print(Fore.CYAN)
    report_gov_or_comm(aws_account, account_found)
    print(Fore.RESET)
    for region in regions:
        if 'gov' in aws_account and not 'admin' in aws_account:
            try:
                session = boto3.Session(profile_name=aws_account,region_name=region)
            except botocore.exceptions.ProfileNotFound as e:
                profile_missing_message = f"An exception has occurred: {e}"                 account_found = 'no' pass else: try:                 session = boto3.Session(profile_name=aws_account,region_name=region)                 account_found = 'yes' except botocore.exceptions.ProfileNotFound as e:                 profile_missing_message = f"An exception has occurred: {e}" pass try:             ec2 = session.client("ec2") except Exception as e: pass
    # Loop through the instances
    try:
        instance_list = ec2.describe_instances()
    except Exception as e:
        pass
    try:
        for reservation in instance_list["Reservations"]:
            for instance in reservation.get("Instances", []):
                instance_count = instance_count + 1
                launch_time = instance["LaunchTime"]
                launch_time_friendly = launch_time.strftime("%B %d %Y")
                tree = objectpath.Tree(block_devices = set(tree.execute('$..BlockDeviceMappings['Ebs']['VolumeId']'))
                if block_devices:
                    block_devices = list(block_devices)
                    block_devices = str(block_devices).replace('[','').replace(']','').replace("'",'')
                else:
                    block_devices = None
                    private_ips =  set(tree.execute('$..PrivateIpAddress'))
                    if private_ips:
                        private_ips_list = list(private_ips)                         private_ips_list = str(private_ips_list).replace('[','').replace(']','').replace(''','')
                    else:
                        private_ips_list = None
                        type(private_ips_list)
                        public_ips =  set(tree.execute('$..PublicIp'))
                        if len(public_ips) == 0:
                            public_ips = None
                        if public_ips:
                            public_ips_list = list(public_ips)
                            public_ips_list = str(public_ips_list).replace('[','').replace(']','').replace("'",'')
                        else:
                            public_ips_list = None
                        if 'KeyName' in instance:
                            key_name = instance['KeyName']
                        else:
                            key_name = None
                            name = None
                        if 'Tags' in instance:
                            try:
                                tags = instance['Tags']
                                name = None
                                for tag in tags:
                                    if tag["Key"] == "Name":
                                        name = tag["Value"]
                                if tag["Key"] == "Engagement" or tag["Key"] == "Engagement Code":
                                    engagement = tag["Value"]
                            except ValueError:
                                print("Instance: %s has no tags" % instance_id
                                pass
                    if 'VpcId' in instance:
    vpc_id = instance['VpcId'] else:                         vpc_id = None if 'PrivateDnsName' in instance:                         private_dns = instance['PrivateDnsName'] else:                         private_dns = None if 'Platform' in instance:                         platform = instance['Platform'] else:                         platform = None print(f"Platform: {platform}")                     ec2info[instance['InstanceId']] = { 'AWS Account': aws_account, 'Account Number': aws_account_number, 'Name': name, 'Instance ID': instance['InstanceId'], 'Volumes': block_devices, 'Private IP': private_ips_list, 'Public IP': public_ips_list, 'Private DNS': private_dns, 'Availability Zone': instance['Placement']['AvailabilityZone'], 'VPC ID': vpc_id, 'Type': instance['InstanceType'], 'Platform': platform, 'Key Pair Name': key_name, 'State': instance['State']['Name'], 'Launch Date': launch_time_friendly                     } with open(output_file,'a') as csv_file:                         writer = csv.DictWriter(csv_file, fieldnames=fieldnames, delimiter=',', lineterminator='\n')                         writer.writerow({'AWS Account': aws_account, "Account Number": aws_account_number, 'Name': name, 'Instance ID': instance["InstanceId"], 'Volumes': block_devices,  'Private IP': private_ips_list, 'Public IP': public_ips_list, 'Private DNS': private_dns, 'Availability Zone': instance['Placement']['AvailabilityZone'], 'VPC ID': vpc_id, 'Type': instance["InstanceType"], 'Platform': platform, 'Key Pair Name': key_name, 'State': instance["State"]["Name"], 'Launch Date': launch_time_friendly})
    if show_details == 'y' or show_details == 'yes': for instance_id, instance in ec2info.items(): if account_found == 'yes': print(Fore.RESET + "-------------------------------------") for key in [ 'AWS Account', 'Account Number', 'Name', 'Instance ID', 'Volumes', 'Private IP', 'Public IP', 'Private DNS', 'Availability Zone', 'VPC ID', 'Type', 'Platform', 'Key Pair Name', 'State', 'Launch Date'                                 ]: print(Fore.GREEN + f"{key}: {instance.get(key)}") print(Fore.RESET + "-------------------------------------") else: pass                     ec2info = {} with open(output_file,'a') as csv_file:                         csv_file.close() except Exception as e: pass if profile_missing_message == '*':         banner(profile_missing_message) print(Fore.GREEN)     report_instance_stats(instance_count, aws_account, account_found) print(Fore.RESET + '\n') return output_file

This is a paste of the whole code for context: aws_ec2_list_instances.py

The first time you run it from the command line it reports the correct total of servers in EC2 (can be verified in the AWS console):

----------------------------------------------------------
There are: 51 EC2 instances in AWS Account: company-lab.
----------------------------------------------------------

The next time it's run with ABSOLUTELY NOTHING changed it reports this total:

----------------------------------------------------------
There are: 102 EC2 instances in AWS Account: company-lab.
----------------------------------------------------------

You literally just up arrow the command and it doubles the results. When it writes to confluence you can see duplicate servers listed. It does that each time you up arrow to run it again, with the same incorrect total (102 servers).

If you close the powershell command line and open again it's back to reporting the correct result (51 servers) which corresponds to what you see in the AWS console. What the heck is happening, here? Why is it doing that and how do I correct the problem?

Answer 1

This is pretty mysterious! I don't think I'm going to be able to debug it without access to your environment. My suggestion is that you use pdb to try to figure out how instance_count is being incremented past 51. I'd recommend adding import pdb; pdb.set_trace() import pdb; pdb.set_trace() at line 210, that is, right after for reservation in instance_list["Reservations"]: . You can then inspect the value of instance_count each time through the loop, see whether instance_list["Reservations"] actually has duplicate data somehow, etc.