根据csv名称在Pandas中读取和输出csv文件

Question

I am trying to write a python code that will automatically read csv files from certain folder and save the final csv with a name depending on original csv names. For instance: In the folder "Torque and Drag" I have 4 raw csv files:

PU 30 0.15 0.25 1.35 0.csv,
PU 0 0.15 0.25 1.35 0.csv,
DA 30 0.15 0.25 1.35 8.csv,
DA 0 0.15 0.25 1.35 8.csv.

I want my function in python to look into this folder and select the csv files starting with PU, put them in the same dataframe and output as a single final csv. The same with csv files starting with DA. The folder "Torque and Drag" will be constantly updated with similar csv files and I might have 20 csv files starting with PU and 20 files starting with DA, and I want my function to output only two final csv files (1 for PU and 1 for DA meaning; PU.csv will contain my 20 csv file data appended one after another and same for DA.csv). I want the headers only once and then skip it as other csvs are appended. I wrote the following code, but it only outputs 1 csv file which contains all of the csvs in that folder (PU and DA) and not separating them based on the initial two letters of the name of the csv. I couldn't handle the condition where I could indicate the initial two letter of the csv name.

import pandas as pd
import os
import glob

def csv_upload():
   folder_name = 'Torque and Drag' #write your folder name where all csvs will be kept
   file_type = 'csv'
   seperator =','

   path = os.getcwd()
   csv_files = glob.glob(os.path.join(path+ "/" + folder_name, "*.csv"))
   for file in csv_files:
      csv_name = file.split("\\")[-1]

   name = csv_name.split(" ")[0]
   initials = f"{name}.csv"
   

   df = pd.concat([pd.read_csv(f, sep=seperator) for f in glob.glob(folder_name + "/*."+file_type) if name=='PU'],
               ignore_index=True)

   df.to_csv(f'{name}.csv',mode='a',header=True,index=False)

Answer 1

I wrote this code, and it works ok, but it doesn't look optimal to me because I am repeating the operations for each csv output, how can I eliminate that and make it more optimal?

import pandas as pd
import os
import glob

def csv_upload():
   folder_name = 'Torque and Drag' #write your folder name where all csvs will be kept
   file_type = 'csv'
   seperator =','
   path = os.getcwd()
   csv_files = glob.glob(os.path.join(path + "/" + folder_name, "*.csv"))

   for file in csv_files:
      csv_name = file.split("\\")[-1]
      name = csv_name.split(" ")[0]
      initials = f"{name}.csv"
    
   PU_df = pd.concat([pd.read_csv(f, sep=seperator) for f in csv_files if f.split("\\")[-1][0:2]=='PU'], ignore_index=True)
   DA_df = pd.concat([pd.read_csv(f, sep=seperator) for f in csv_files if f.split("\\")[-1][0:2]=='DA'], ignore_index=True)
    
   PU_df.rename(columns={'MD\r\n[m]':"MD (m)",'Hole size\r\n[inch]':'Hole size (inch)','Tool OD\r\n[inch]':'Tool OD (inch)',
                       'Tool ID\r\n[inch]':'Tool ID (inch)',
                       'Weight\r\n[kg]':'Weight (kg)','Inc\r\n[deg]':'Inc (deg)',
                       'Dir\r\n[deg Az]':'Dir (deg Az)',
                      'Friction\r\nFactor':'Friction (nFactor)',
                      'Tension limit\r\n[kg]':'Tension limit (kg)',
                      'Torsion limit\r\n[kN.m]':'Torsion limit (kN.m)',
                      'F Sin.\r\n[kg]':'F Sin (kg)','F Hel.\r\n[kg]':'F Hel (kg)',
                      'F normal\r\n[kg]':'F normal (kg)','Tension\r\n[kg]':'Tension (kg)',
                      'Torque\r\n[kN.m]':'Torque, (kN.m)'}, inplace = True)

   DA_df.rename(columns={'MD\r\n[m]':"MD (m)",'Hole size\r\n[inch]':'Hole size (inch)','Tool OD\r\n[inch]':'Tool OD (inch)',
                       'Tool ID\r\n[inch]':'Tool ID (inch)',
                       'Weight\r\n[kg]':'Weight (kg)','Inc\r\n[deg]':'Inc (deg)',
                       'Dir\r\n[deg Az]':'Dir (deg Az)',
                      'Friction\r\nFactor':'Friction (nFactor)',
                      'Tension limit\r\n[kg]':'Tension limit (kg)',
                      'Torsion limit\r\n[kN.m]':'Torsion limit (kN.m)',
                      'F Sin.\r\n[kg]':'F Sin (kg)','F Hel.\r\n[kg]':'F Hel (kg)',
                      'F normal\r\n[kg]':'F normal (kg)','Tension\r\n[kg]':'Tension (kg)',
                      'Torque\r\n[kN.m]':'Torque, (kN.m)'}, inplace = True)

    maxMD = PU_df['MD (m)'].max()
    PU_df['Depth (m)'] = maxMD - PU_df['MD (m)']
    PU_df['State'] = csv_name.split(" ")[0]
    PU_df['RPM'] = csv_name.split(" ")[1]
    PU_df['Casing Friction Factor'] = csv_name.split(" ")[2]
    PU_df['Open Hole Friction Factor'] = csv_name.split(" ")[3]
    PU_df['MW'] = csv_name.split(" ")[4]
    PU_df['WOB'] = csv_name.split(" ")[5][0]

    maxMD = DA_df['MD (m)'].max()
    DA_df['Depth (m)'] = maxMD - DA_df['MD (m)']
    DA_df['State'] = csv_name.split(" ")[0]
    DA_df['RPM'] = csv_name.split(" ")[1]
    DA_df['Casing Friction Factor'] = csv_name.split(" ")[2]
    DA_df['Open Hole Friction Factor'] = csv_name.split(" ")[3]
    DA_df['MW'] = csv_name.split(" ")[4]
    DA_df['WOB'] = csv_name.split(" ")[5][0]

    PU_df.to_csv('PU.csv',mode='a',header=True,index=False)
    DA_df.to_csv('DA.csv',mode='a',header=True,index=False)

csv_upload()