group list of dicts by multiple dynamic keys

Question

I have a dynamic list of dicts that is not the same each time and i want to group this list of dicts based on multiple keys one key at time with appending the last used keys with sum float& int values. for grouping each time i group with multi key but in the last iteration of grouping i get final data in form without sum.

To explain what i want to do here is a sample list of dicts (this example is not static the key/values pair of dicts may change):

[{'invoice_num': 'INV/2019/0012 ', 'tax_id': 'Tax 15.00%S', 'vat': False, 'total_amount': 805.0, 'fiscal_position_id': False, 'date': '2019-12-05', 'amount_exclude': 700.0, 'desc': '[AT] Air Flight', 'partner_id': 'Agrolait', 'amount_tax': 105.0, 'record_type': 'sale'}, {'invoice_num': 'INV/2019/0011 ', 'tax_id': 'Tax 15.00%S', 'vat': False, 'total_amount': 805.0, 'fiscal_position_id': False, 'date': '2019-12-05', 'amount_exclude': 700.0, 'desc': '[AT] Air Flight', 'partner_id': 'Agrolait', 'amount_tax': 105.0, 'record_type': 'sale'}, {'invoice_num': 'BILL/2019/0007 ', 'tax_id': 'Tax 15.00% P', 'vat': False, 'total_amount': 51750.0, 'fiscal_position_id': False, 'date': '2019-12-05', 'amount_exclude': 45000.0, 'desc': '[CONS_DEL02] Little server', 'partner_id': "Administrator, Pieter Parter's Farm", 'amount_tax': 6750.0, 'record_type': 'purchase'}, {'invoice_num': 'BILL/2019/0006 ', 'tax_id': 'Tax 15.00% P', 'vat': False, 'total_amount': 5749.99, 'fiscal_position_id': False, 'date': '2019-12-05', 'amount_exclude': 4999.99, 'desc': "Coffee Machine with huge 'employee's performances boosting perk'", 'partner_id': 'ASUSTeK', 'amount_tax': 750.0, 'record_type': 'purchase'}, {'invoice_num': 'BILL/2019/0002 ', 'tax_id': 'Tax 15.00% P', 'vat': False, 'total_amount': 5749.99, 'fiscal_position_id': False, 'date': '2019-11-15', 'amount_exclude': 4999.99, 'desc': "Coffee Machine with huge 'employee's performances boosting perk'", 'partner_id': 'ASUSTeK', 'amount_tax': 750.0, 'record_type': 'purchase'}]

And i want to group by:

['record_type', 'tax_id','partner_id']

So in the first group (by record_type) i want the data to be grouping only to sum the values then in the second group (by record_type & tax_id) i want to get the values with sum of the float& int values but in the third group (by 'record_type'&'tax_id'&'partner_id') i want to get the records without sum,then i will insert those to xls file.

The final representation will be :

I have tried to group the items one key at a time but i can't group the data with same elements and present the last item and in the table :

item_data = []
for item in group_by:
    final_data = [(a, list(b)) for a, b in
                  itertools.groupby([i.items() for i in records_read], key=lambda x: dict(x)[item])]
    new_final_data = [
        {i[0][0]: sum(c[-1] for c in i if isinstance(c[-1], float) or isinstance(c[-1], int)) if i[0][0] != item else
        i[0][-1] for i in
         zip(*b)} for a, b in final_data]
    item_data.append(new_final_data)

Answer 1

This script prints the tree based on fields list:

data = [{'invoice_num': 'INV/2019/0012 ', 'tax_id': 'Tax 15.00%S', 'vat': 'Undefined', 'total_amount': 805.0, 'fiscal_position_id': 'Undefined', 'date': '2019-12-05', 'amount_exclude': 700.0, 'desc': '[AT] Air Flight', 'partner_id': 'Agrolait', 'amount_tax': 105.0, 'record_type': 'sale'}, {'invoice_num': 'INV/2019/0011 ', 'tax_id': 'Tax 15.00%S', 'vat': 'Undefined', 'total_amount': 805.0, 'fiscal_position_id': 'Undefined', 'date': '2019-12-05', 'amount_exclude': 700.0, 'desc': '[AT] Air Flight', 'partner_id': 'Agrolait', 'amount_tax': 105.0, 'record_type': 'sale'}, {'invoice_num': 'BILL/2019/0007 ', 'tax_id': 'Tax 15.00% P', 'vat': 'Undefined', 'total_amount': 51750.0, 'fiscal_position_id': 'Undefined', 'date': '2019-12-05', 'amount_exclude': 45000.0, 'desc': '[CONS_DEL02] Little server', 'partner_id': "Administrator, Pieter Parter's Farm", 'amount_tax': 6750.0, 'record_type': 'purchase'}, {'invoice_num': 'BILL/2019/0006 ', 'tax_id': 'Tax 15.00% P', 'vat': 'Undefined', 'total_amount': 5749.99, 'fiscal_position_id': 'Undefined', 'date': '2019-12-05', 'amount_exclude': 4999.99, 'desc': "Coffee Machine with huge 'employee's performances boosting perk'", 'partner_id': 'ASUSTeK', 'amount_tax': 750.0, 'record_type': 'purchase'}, {'invoice_num': 'BILL/2019/0002 ', 'tax_id': 'Tax 15.00% P', 'vat': 'Undefined', 'total_amount': 5749.99, 'fiscal_position_id': 'Undefined', 'date': '2019-11-15', 'amount_exclude': 4999.99, 'desc': "Coffee Machine with huge 'employee's performances boosting perk'", 'partner_id': 'ASUSTeK', 'amount_tax': 750.0, 'record_type': 'purchase'}]
fields = ['record_type', 'tax_id', 'partner_id', 'invoice_num']

from itertools import groupby
from operator import itemgetter

def group_by_fields(data, *fields):
f = itemgetter(*fields)
for v, g in groupby(sorted(data, key=f), f):
    g = list(g)
    float_keys = []
    for item in g:
        float_keys.extend(
            [key for key in item if isinstance(item[key], float) or isinstance(item[key], int)])
    float_keys = list(set(float_keys))
    yield (v,) if isinstance(v, str) else v, [sum(i[k] for i in g) for k in float_keys]


def print_tree(data, *fields):
    rv = []
    for i in range(1, len(fields)+1):
        rv.extend(group_by_fields(data, *fields[:i]))

    for v, g in groupby(sorted(rv, key=lambda k: (k[0], len(k[0])) ), lambda k: (k[0], len(k[0])) ):
        print('{:<60}'.format( (' ' * (v[1] * 4)) + v[0][-1]), end=' ')
        for item in g:
            print('{:<10.2f} {:<10.2f} {:<10.2f} '.format(*item[1][0:]))

print_tree(data, *fields)

Prints:

purchase                                                 63249.98   54999.98   8250.00    
    Tax 15.00% P                                         63249.98   54999.98   8250.00    
        ASUSTeK                                          11499.98   9999.98    1500.00    
            BILL/2019/0002                               5749.99    4999.99    750.00     
            BILL/2019/0006                               5749.99    4999.99    750.00     
        Administrator, Pieter Parter's Farm              51750.00   45000.00   6750.00    
            BILL/2019/0007                               51750.00   45000.00   6750.00    
sale                                                     1610.00    1400.00    210.00     
    Tax 15.00%S                                          1610.00    1400.00    210.00     
        Agrolait                                         1610.00    1400.00    210.00     
            INV/2019/0011                                805.00     700.00     105.00     
            INV/2019/0012                                805.00     700.00     105.00