Use Panda to merge .csv files by matching dates

Question

I have multiple .csv files in a directory. I would like to merge/concatenate them into one file. I would like to merge those dataframes by matching dates (they all have their first column named 'date' ).

The code I have work but doesn't do the matching on dates. I tried many workaround but cannot find a way :(.

I guess I should not use pd.concat but pd.merge but no luck using something like that: dataV = pd.merge(list_data, on='Date', how='outer')

import pandas as pd
import glob
csv_files = glob.glob('./desktop/2019/*.csv')

list_data = []
for filename in csv_files:
    data = pd.read_csv(filename,usecols=['Date','Quantity'])
    list_data.append(data)
list_data

dataV = pd.concat(list_data,axis=1,sort=False)
dataV.to_csv("./desktop/test.csv")

Answer 1

With the .csv file contents:

DateCol;QuantityCol
2015-01-02;10
2015-01-03;20
2015-01-04;30
2015-01-05;40

You could use the reduce approach:

import os
import pandas as pd
from functools import reduce

pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

os.chdir(r'C:\Temp')

dfs = [
    pd.read_csv(csv_file, sep=';') for csv_file in
    [f for f in os.listdir() if os.path.splitext(f)[1] == '.csv']
]
merged = reduce(lambda left, right: pd.merge(left, right, on='DateCol'), dfs)
print(merged)
merged.to_csv('out.csv', sep=';', index=False)

The output .csv file contents:

DateCol;QuantityCol_x;QuantityCol_y;QuantityCol_x;QuantityCol_y;QuantityCol
2015-01-02;10;100;1000;10000;100000
2015-01-03;20;200;2000;20000;200000
2015-01-04;30;300;3000;30000;300000
2015-01-05;40;400;4000;40000;400000

You can rename the columns before exporting to the .csv file using merged.columns = ['DateCol', 'Quan1', 'Quan2', 'Quan3', 'Quan4', 'Quan5'] .

You can also count the number of .csv files read (or the number of data frames created) and then construct a list of columns to use such as

columns_to_use = ['DateCol'] + ['Quantity_{}'.format(idx) for idx in range(1, len(dfs) + 1)]
merged.columns = columns_to_use

Answer 2

import pandas as pd
import os

dir = '/home/brb/bugs/mecd/abs-data/'

first = True
for folder, subfolders, files in os.walk(dir):
    for f in files:
        file = str(folder)+str(f)
        if file.split('.')[-1] == 'csv':
            if first:
                data = pd.read_csv(file)
                first = False
            else:
                df = pd.read_csv(file)
                data = pd.merge(data, df, on=['Date', 'Date'])