根据相似度合并两个字典（不包含键）

Question

I have the following three dictionaries in an array:

items = [ 
{
    'FirstName': 'David',
    'LastName': 'Smith',
    'Language': set(['en'])
},

{
    'FirstName': 'David',
    'LastName': 'Smith',
    'Language': set(['fr'])
},

{
    'FirstName': 'Bob',
    'LastName': 'Jones',
    'Language': set(['en'])
} ]

I want to merge together these dictionaries if the two dictionaries are the same minus the specified key: and add that key together. If using the "Language" key it would merge the array into the following:

[ {
    'FirstName': 'David',
    'LastName': 'Smith',
    'Language': set(['en','fr'])
},{
    'FirstName': 'Bob',
    'LastName': 'Jones',
    'Language': set(['en'])
} ]

Here is what I'm currently doing:

from copy import deepcopy

def _merge_items_on_field(items, field):
    '''Given an array of dicts, merge the 
       dicts together if they are the same except for the 'field'.

       If merging dicts, add the unique values of that field together.'''

    items = deepcopy(items)
    items_merged_on_field = []

    for num, item in enumerate(items):

        # Remove that key/value from the dict
        field_value = item.pop(field)

        # Get an array of items *without* that field to compare against
        items_without_field = deepcopy(items_merged_on_field)
        map(lambda d: d.pop(field), items_without_field)

        # If the dict item is found ("else"), add the fields together
        # If not ("except"), then add in the dict item to the array
        try:
            index = items_without_field.index(item) 
        except ValueError:
            item[field] = field_value
            items_merged_on_field.append(item)
        else:
            items_merged_on_field[index][field] = items_merged_on_field[index][field].union(field_value)

    return items_merged_on_field

>>> items = [{'LastName': 'Smith', 'Language': set(['en']), 'FirstName': 'David'}, {'LastName': 'Smith', 'Language': set(['fr']), 'FirstName': 'David'}, {'LastName': 'Jones', 'Language': set(['en']), 'FirstName': 'Bob'}]
>>> _merge_items_on_field(items, 'Language')
[{'LastName': 'Smith', 'Language': set(['fr', 'en']), 'FirstName': 'David'}, {'LastName': 'Jones', 'Language': set(['en']), 'FirstName': 'Bob'}]

This seems a bit complicated -- is there a better way to do this?

Answer 1

There are a couple of ways of doing this. The most painless method to my knowledge utilises the pandas library—in particular, a groupby + apply .

import pandas as pd

merged = (
    pd.DataFrame(items)
      .groupby(['FirstName', 'LastName'], sort=False)
      .Language
      .apply(lambda x: set.union(*x))
      .reset_index()
      .to_dict(orient='records')
)

print(merged)
[
    {'FirstName': 'David', 'LastName': 'Smith', 'Language': {'en', 'fr'}},
    {'FirstName': 'Bob', 'LastName': 'Jones', 'Language': {'en'}}
]

The other method (that I mentioned) uses itertools.groupby , but seeing as you have 30 columns to group on, I'd just recommend sticking to pandas.

---

If you want to turn this into a function,

def merge(items, field):
    df = pd.DataFrame(items)
    columns = df.columns.difference([field]).tolist()
    return (
        df.groupby(columns, sort=False)[field]
          .apply(lambda x: set.union(*x))
          .reset_index()
          .to_dict(orient='records')
    )

merged = merge(items, 'Language')
print(merged)
[
    {'FirstName': 'David', 'LastName': 'Smith', 'Language': {'en', 'fr'}},
    {'FirstName': 'Bob', 'LastName': 'Jones', 'Language': {'en'}}
]

Answer 2

You can use itertools.groupby :

import itertools
d = [{'FirstName': 'David', 'LastName': 'Smith', 'Language': {'en'}}, {'FirstName': 'David', 'LastName': 'Smith', 'Language': {'fr'}}, {'FirstName': 'Bob', 'LastName': 'Jones', 'Language': {'en'}}]
v = [[a, list(b)] for a, b in itertools.groupby(sorted(d, key=lambda x:x['FirstName']), key=lambda x:x['FirstName'])]
final_dict = [{**{'FirstName':a}, **{'LastName':(lambda x:[list(set(x)), x[0]][len(set(x)) == 1])([i['LastName'] for i in b])}, **{'Language':set([list(i['Language'])[0] for i in b])}} for a, b in v]

Output:

[{'FirstName': 'Bob', 'LastName': 'Jones', 'Language': {'en'}}, {'FirstName': 'David', 'LastName': 'Smith', 'Language': {'en', 'fr'}}]

Answer 3

If pandas is not an option:

from itertools import groupby
from functools import reduce

arr = [
    {'FirstName': 'David', 'LastName': 'Smith', 'Language': set(['en'])},
    {'FirstName': 'David', 'LastName': 'Smith', 'Language': set(['fr'])},
    {'FirstName': 'David', 'LastName': 'Jones', 'Language': set(['sp'])}
]

def reduce_field(items, field, op=set.union, sort=False):

    def _key(d):
        return tuple((k, v) for k, v in d.items() if k != field)

    if sort:
        items = sorted(items, key=_key)
    res = []
    for k, g in groupby(items, key=_key):
        d = dict(k)
        d[field] = reduce(op, (el[field] for el in g))
        res.append(d)

    return res

reduce_field(arr, 'Language')

Answer 4

You can try it manually :

new_dict={}
#
#
#
d = [{'FirstName': 'David', 'LastName': 'Smith', 'Language': {'en'}},
     {'FirstName': 'David', 'LastName': 'Smith', 'Language': {'fr'}},
     {'FirstName': 'Bob', 'LastName': 'Jones', 'Language': {'en'}}]




for i in d:
     if (i['FirstName'],i['LastName']) not in new_dict:
          new_dict[(i['FirstName'],i['LastName'])]=i
     else:
          new_dict[(i['FirstName'],i['LastName'])]['Language']=set(list(new_dict[(i['FirstName'],i['LastName'])]['Language'])+list(i['Language']))
print(new_dict.values())

output:

# dict_values([{'FirstName': 'Bob', 
#               'LastName': 'Jones', 
#               'Language': {'en'}}, 
#              {'FirstName': 'David', 
#               'LastName': 'Smith', 
#                'Language': {'fr', 'en'}}])