将具有相同键值对的多个字典合并到一个字典python

Question

I have a list of dicts stored in the list. I want to merge the same dicts in to one. i have three fields. Task_id provides which field to be checked. value is the value of that field. First it checks the value in the dict and it creates a new dict for the dicts to be merged. if all the values are same and one of the value is only different, then it merges the dict to one. How to make it possible

Here's the sample code i tried:

field_to_be_check ="state"
merger = ["city", "ads"]
merge_name = ["cities", "my_ads"]

data = [
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'tirunelveli'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad4', 'city': 'nagerkoil'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'tuticorin'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'madurai'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'chennai'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'palakad'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'guruvayor'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'kolikodu'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'kottayam'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'idukki'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Akola'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Washim'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Jalna'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Nanded'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Latur'}
]

d = []
list1 = []
for item in data:
    value = item[field_to_be_check]
    inserted = False
    for l in list1: 
        if l[field_to_be_check] == value:
            inserted = True
            for m_name in merge_name:

    if inserted == False:
        list1.append(item)

print(list1)

Required Output:

   [
    {'state': 'tamil nadu','my_ads':[{'ads': 'ad1'},{'ads': 'ad4'}], 'cities':[{'city': 'tirunelveli'},{'city': 'nagerkoil'},{'city': 'tuticorin'},{'city': 'madurai'},{'city': 'chennai'}]}, 
    {'state': 'kerala',,'my_ads':[{'ads': 'ad2'}], 'cities': [{'city': 'palakad'},{'city': 'guruvayor'},{'city': 'kolikodu'},{'city': 'kottayam'},{'city': 'idukki'}]}, 
    {'state': 'mumbai', 'my_ads':[{'ads': 'ad3'}],'cities':[{'city': 'Akola'},{'city': 'Washim'},{'city': 'Jalna'},{'city': 'Nanded'},{'city': 'Latur'}]}
    ]

Answer 1

field_to_be_check ="state"
merger = ["city", "ads"]
merge_name = ["cities", "my_ads"]

data = [
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'tirunelveli'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad4', 'city': 'nagerkoil'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'tuticorin'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'madurai'}, 
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'chennai'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'palakad'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'guruvayor'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'kolikodu'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'kottayam'}, 
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'idukki'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Akola'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Washim'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Jalna'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Nanded'}, 
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Latur'}
]
# merger and merge_name must be one to one.
the_dict = {m:mn for m, mn in zip(merger, merge_name)}
# {"city":"cities", "ads":"my_ads"}  merge_name
newdata = data.copy()
# create new_ret as result
new_ret = [{field_to_be_check:i, **{i:[] for i in merge_name}} for i in set([i[field_to_be_check] for i in data])]
# print(new_ret, "this is new_ret")
for val in new_ret:
    for k in newdata:
        if val[field_to_be_check] != k[field_to_be_check]:
            continue
        tmp = {i:k[i] for i in merger}
        for single in tmp:
            if {single:tmp[single]} not in val[the_dict[single]]:
                val[the_dict[single]].append({single:tmp[single]})
print(new_ret)

Answer 2

This is a perfect scenario to see the power of itertools.groupby Note that I have assumed haps, state and ads will be present in all dictionaries, and will be similar in repetitions

from itertools import groupby

field_to_be_check =  "state"
merger = ["city", "ads"]
merge_name = ["cities", "my_ads"]

data = [
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'tirunelveli'},
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad4', 'city': 'nagerkoil'},
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'tuticorin'},
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'madurai'},
{'haps': 'hap0', 'state': 'tamil nadu', 'ads': 'ad1', 'city': 'chennai'},
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'palakad'},
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'guruvayor'},
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'kolikodu'},
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'kottayam'},
{'haps': 'hap1', 'state': 'kerala', 'ads': 'ad2', 'city': 'idukki'},
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Akola'},
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Washim'},
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Jalna'},
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Nanded'},
{'haps': 'hap2', 'state': 'mumbai', 'ads': 'ad3', 'city': 'Latur'}
]

#Function to make the merger lists
def process_group(group, merger_item):

    item_set = set()
    item_list = []
    for item in group:
        item_set.add(item[merger_item])

    for item in item_set:
        item_list.append({merger_item: item})

    return item_list

#Group on haps, state and ads
grp = groupby(data,key=lambda x:(x[field_to_be_check]))
result = []

#Iterate through the group and build your result list
for model, group in grp:
    cities_dict = {}

    cities_dict[field_to_be_check] = model

    group_list = list(group)

    #Make the list for merger fields
    for idx, name in enumerate(merger):
        cities_dict[merge_name[idx]] = process_group(group_list, name)

    result.append(cities_dict)

print(result)

The output will look like

[{'state': 'tamil nadu', 
'cities': [{'city': 'nagerkoil'}, {'city': 'tuticorin'}, {'city': 'chennai'}, {'city': 'madurai'}, {'city': 'tirunelveli'}], 
'my_ads': [{'ads': 'ad4'}, {'ads': 'ad1'}]}, 
{'state': 'kerala', 
'cities': [{'city': 'guruvayor'}, {'city': 'idukki'}, {'city': 'kottayam'}, {'city': 'palakad'}, {'city': 'kolikodu'}], 
'my_ads': [{'ads': 'ad2'}]}, 
{'state': 'mumbai', 
'cities': [{'city': 'Jalna'}, {'city': 'Nanded'}, {'city': 'Washim'}, {'city': 'Latur'}, {'city': 'Akola'}], 
'my_ads': [{'ads': 'ad3'}]}]