How to convert csv into nested json in python pandas?

Question

I have a csv like this:

    Art        Category  LEVEL 2    LEVEL 3 LEVEL 4 LEVEL 5 Location
0   PRINTMAKING VISUAL  CONTEMPORARY    2D  NaN NaN NaN
1   PAINTING    VISUAL  CONTEMPORARY    2D  NaN NaN NaN
2   AERIAL  VISUAL  CONTEMPORARY    2D  PHOTOGRAPHY AERIAL  NaN
3   WILDLIFE    VISUAL  CONTEMPORARY    2D  PHOTOGRAPHY WILDLIFE    NaN
4   NATURE  VISUAL  CONTEMPORARY    2D  PHOTOGRAPHY NATURE  NaN

The art and category will be there but the levels from l1 to l6 can be null. What I want to achive is like so:

art: PRINTMAKING
category: VISUAL
tags: [CONTEMPORARY, 2D]

The levels are basically tags for a particular art which are to stored in an array.

I am new to python and so far I have written the following code. How can I achive this.

import pandas as pd
import json
data = pd.read_excel("C:\\Users\\Desktop\\visual.xlsx")
rec = {}
rec['art'] = data['Art']
rec['category'] = data['Category']
rec['tags'] = data['LEVEL 2'] + ',' + data['LEVEL 3'] + ',' + data['LEVEL 4'] + ',' + data['LEVEL 5']

I guess this is not the correct way to do it.

Answer 1

for convert values of tags to lists without NaN s use:

df['tags'] = df.filter(like='LEVEL').apply(lambda x: x.dropna().tolist(), axis=1)
#alternative, should be faster
#df['tags'] = [[y for y in x if isinstance(y, str)] for x in
#                 df.filter(like='LEVEL').values]

d = df[['Art','Category','tags']].to_dict(orient='records')

[{
    'Art': 'PRINTMAKING',
    'Category': 'VISUAL',
    'tags': ['CONTEMPORARY', '2D']
}, {
    'Art': 'PAINTING',
    'Category': 'VISUAL',
    'tags': ['CONTEMPORARY', '2D']
}, {
    'Art': 'AERIAL',
    'Category': 'VISUAL',
    'tags': ['CONTEMPORARY', '2D', 'PHOTOGRAPHY', 'AERIAL']
}, {
    'Art': 'WILDLIFE',
    'Category': 'VISUAL',
    'tags': ['CONTEMPORARY', '2D', 'PHOTOGRAPHY', 'WILDLIFE']
}, {
    'Art': 'NATURE',
    'Category': 'VISUAL',
    'tags': ['CONTEMPORARY', '2D', 'PHOTOGRAPHY', 'NATURE']
}]

Answer 2

df

   Art     Category   LEVEL             2 LEVEL.1            3   LEVEL.2   4  \
0    0  PRINTMAKING  VISUAL  CONTEMPORARY      2D          NaN       NaN NaN   
1    1     PAINTING  VISUAL  CONTEMPORARY      2D          NaN       NaN NaN   
2    2       AERIAL  VISUAL  CONTEMPORARY      2D  PHOTOGRAPHY    AERIAL NaN   
3    3     WILDLIFE  VISUAL  CONTEMPORARY      2D  PHOTOGRAPHY  WILDLIFE NaN   
4    4       NATURE  VISUAL  CONTEMPORARY      2D  PHOTOGRAPHY    NATURE NaN   

   LEVEL.3   5  Location  
0      NaN NaN       NaN  
1      NaN NaN       NaN  
2      NaN NaN       NaN  
3      NaN NaN       NaN  
4      NaN NaN       NaN  

df = df.set_index(['Art','Category']).apply(lambda x: [','.join([str(a) for a in x.values if str(a) != 'nan'])], axis=1)

print(df.reset_index(name='tags'))

   Art     Category                                           tags
0    0  PRINTMAKING                       [VISUAL,CONTEMPORARY,2D]
1    1     PAINTING                       [VISUAL,CONTEMPORARY,2D]
2    2       AERIAL    [VISUAL,CONTEMPORARY,2D,PHOTOGRAPHY,AERIAL]
3    3     WILDLIFE  [VISUAL,CONTEMPORARY,2D,PHOTOGRAPHY,WILDLIFE]
4    4       NATURE    [VISUAL,CONTEMPORARY,2D,PHOTOGRAPHY,NATURE]

To dict

df.to_dict(orient='records')

Output

[{'Art': 0, 'Category': 'PRINTMAKING', 'tags': ['VISUAL,CONTEMPORARY,2D']},
 {'Art': 1, 'Category': 'PAINTING', 'tags': ['VISUAL,CONTEMPORARY,2D']},
 {'Art': 2,
  'Category': 'AERIAL',
  'tags': ['VISUAL,CONTEMPORARY,2D,PHOTOGRAPHY,AERIAL']},
 {'Art': 3,
  'Category': 'WILDLIFE',
  'tags': ['VISUAL,CONTEMPORARY,2D,PHOTOGRAPHY,WILDLIFE']},
 {'Art': 4,
  'Category': 'NATURE',
  'tags': ['VISUAL,CONTEMPORARY,2D,PHOTOGRAPHY,NATURE']}]

Answer 3

You should use pd.Series.str.cat combined with functools.reduce to concatenate all of the tags:

df = pd.DataFrame({
    'art': ['a1', 'a2', 'a3'],
    'category': ['c1', 'c2', 'c3'],
    'l1': ['abc', '', 'lmn'],
    'l2': ['def', 'xyz', 'qwe'],
})

from functools import reduce
tag_cols = [x for x in df.columns if x not in ['art', 'category']]
df['tags'] = reduce(lambda a, b: df[a].str.cat(df[b], sep=','), 
tag_cols).apply(lambda x: [t for t in x.split(",") if t != ''])
d = df.to_dict(orient='records')

Output

  [{'art': 'a1',
  'category': 'c1',
  'l1': 'abc',
  'l2': 'def',
  'tags': ['abc', 'def']},
 {'art': 'a2', 'category': 'c2', 'l1': '', 'l2': 'xyz', 'tags': ['xyz']},
 {'art': 'a3',
  'category': 'c3',
  'l1': 'lmn',
  'l2': 'qwe',
  'tags': ['lmn', 'qwe']}]

Answer 4

This might solve your problem:

from io import StringIO
import csv
# help(csv)
categories="""art,category, l1, l2, l3, l4, l5, l6
a1,c1,abc,def
a2,c2,,,,xyz,pqr,
a3,c3,lmn,,,qwe,rtg,
"""

f=StringIO(categories)
rows=csv.DictReader(f,delimiter=',')
data=[]
for row in rows:
#     print(row)
    d={
        "cateory":row.get("category",''),
        "art":row.get("art",'')
    }
    try:
        del row["category"]
        del row["art"]
    except KeyError as ke:
        print(ke)
#     print(row)
    d["levels"]=list(row.values())
    print(d)

Sample output:

{'cateory': 'c1', 'art': 'a1', 'levels': ['abc', 'def', None, None, None, None]}
{'cateory': 'c2', 'art': 'a2', 'levels': ['', '', '', 'xyz', 'pqr', '']}
{'cateory': 'c3', 'art': 'a3', 'levels': ['lmn', '', '', 'qwe', 'rtg', '']}