![](/img/trans.png)
[英]Append existing excel sheet with new dataframe using python pandas without loading the old one
[英]Append existing excel sheet with new dataframe using python pandas
我目前有这个代码。 它完美地工作。
它遍历文件夹中的 excel 文件,删除前 2 行,然后将它们保存为单独的 excel 文件,并将循环中的文件保存为附加文件。
目前,每次运行代码时,附加文件都会覆盖<\/strong>现有文件。
我需要将新数据附加到已经存在的 excel 表<\/strong>('master_data.xlsx)的底部
dfList = []
path = 'C:\\Test\\TestRawFile'
newpath = 'C:\\Path\\To\\New\\Folder'
for fn in os.listdir(path):
# Absolute file path
file = os.path.join(path, fn)
if os.path.isfile(file):
# Import the excel file and call it xlsx_file
xlsx_file = pd.ExcelFile(file)
# View the excel files sheet names
xlsx_file.sheet_names
# Load the xlsx files Data sheet as a dataframe
df = xlsx_file.parse('Sheet1',header= None)
df_NoHeader = df[2:]
data = df_NoHeader
# Save individual dataframe
data.to_excel(os.path.join(newpath, fn))
dfList.append(data)
appended_data = pd.concat(dfList)
appended_data.to_excel(os.path.join(newpath, 'master_data.xlsx'))
熊猫团队干得好!
根据pandas-dev github中的DocString,ExcelWriter将支持参数if_sheet_exists='overlay'
if_sheet_exists : {'error', 'new', 'replace', 'overlay'}, default 'error' How to behave when trying to write to a sheet that already exists (append mode only). * error: raise a ValueError. * new: Create a new sheet, with a name determined by the engine. * replace: Delete the contents of the sheet before writing to it. * overlay: Write contents to the existing sheet without removing the old contents. .. versionadded:: 1.3.0 .. versionchanged:: 1.4.0 Added ``overlay`` option
对于 < 1.4.0 的 Pandas 版本,请在下方找到用于将 Pandas DataFrame 附加到现有 Excel 文件的辅助函数。
如果 Excel 文件不存在,则将创建该文件。
以下功能已经过测试:
from pathlib import Path
from copy import copy
from typing import Union, Optional
import numpy as np
import pandas as pd
import openpyxl
from openpyxl import load_workbook
from openpyxl.utils import get_column_letter
def copy_excel_cell_range(
src_ws: openpyxl.worksheet.worksheet.Worksheet,
min_row: int = None,
max_row: int = None,
min_col: int = None,
max_col: int = None,
tgt_ws: openpyxl.worksheet.worksheet.Worksheet = None,
tgt_min_row: int = 1,
tgt_min_col: int = 1,
with_style: bool = True
) -> openpyxl.worksheet.worksheet.Worksheet:
"""
copies all cells from the source worksheet [src_ws] starting from [min_row] row
and [min_col] column up to [max_row] row and [max_col] column
to target worksheet [tgt_ws] starting from [tgt_min_row] row
and [tgt_min_col] column.
@param src_ws: source worksheet
@param min_row: smallest row index in the source worksheet (1-based index)
@param max_row: largest row index in the source worksheet (1-based index)
@param min_col: smallest column index in the source worksheet (1-based index)
@param max_col: largest column index in the source worksheet (1-based index)
@param tgt_ws: target worksheet.
If None, then the copy will be done to the same (source) worksheet.
@param tgt_min_row: target row index (1-based index)
@param tgt_min_col: target column index (1-based index)
@param with_style: whether to copy cell style. Default: True
@return: target worksheet object
"""
if tgt_ws is None:
tgt_ws = src_ws
# https://stackoverflow.com/a/34838233/5741205
for row in src_ws.iter_rows(min_row=min_row, max_row=max_row,
min_col=min_col, max_col=max_col):
for cell in row:
tgt_cell = tgt_ws.cell(
row=cell.row + tgt_min_row - 1,
column=cell.col_idx + tgt_min_col - 1,
value=cell.value
)
if with_style and cell.has_style:
# tgt_cell._style = copy(cell._style)
tgt_cell.font = copy(cell.font)
tgt_cell.border = copy(cell.border)
tgt_cell.fill = copy(cell.fill)
tgt_cell.number_format = copy(cell.number_format)
tgt_cell.protection = copy(cell.protection)
tgt_cell.alignment = copy(cell.alignment)
return tgt_ws
def append_df_to_excel(
filename: Union[str, Path],
df: pd.DataFrame,
sheet_name: str = 'Sheet1',
startrow: Optional[int] = None,
max_col_width: int = 30,
autofilter: bool = False,
fmt_int: str = "#,##0",
fmt_float: str = "#,##0.00",
fmt_date: str = "yyyy-mm-dd",
fmt_datetime: str = "yyyy-mm-dd hh:mm",
truncate_sheet: bool = False,
storage_options: Optional[dict] = None,
**to_excel_kwargs
) -> None:
"""
Append a DataFrame [df] to existing Excel file [filename]
into [sheet_name] Sheet.
If [filename] doesn't exist, then this function will create it.
@param filename: File path or existing ExcelWriter
(Example: '/path/to/file.xlsx')
@param df: DataFrame to save to workbook
@param sheet_name: Name of sheet which will contain DataFrame.
(default: 'Sheet1')
@param startrow: upper left cell row to dump data frame.
Per default (startrow=None) calculate the last row
in the existing DF and write to the next row...
@param max_col_width: maximum column width in Excel. Default: 40
@param autofilter: boolean - whether add Excel autofilter or not. Default: False
@param fmt_int: Excel format for integer numbers
@param fmt_float: Excel format for float numbers
@param fmt_date: Excel format for dates
@param fmt_datetime: Excel format for datetime's
@param truncate_sheet: truncate (remove and recreate) [sheet_name]
before writing DataFrame to Excel file
@param storage_options: dict, optional
Extra options that make sense for a particular storage connection, e.g. host, port,
username, password, etc., if using a URL that will be parsed by fsspec, e.g.,
starting “s3://”, “gcs://”.
@param to_excel_kwargs: arguments which will be passed to `DataFrame.to_excel()`
[can be a dictionary]
@return: None
Usage examples:
>>> append_df_to_excel('/tmp/test.xlsx', df, autofilter=True,
freeze_panes=(1,0))
>>> append_df_to_excel('/tmp/test.xlsx', df, header=None, index=False)
>>> append_df_to_excel('/tmp/test.xlsx', df, sheet_name='Sheet2',
index=False)
>>> append_df_to_excel('/tmp/test.xlsx', df, sheet_name='Sheet2',
index=False, startrow=25)
>>> append_df_to_excel('/tmp/test.xlsx', df, index=False,
fmt_datetime="dd.mm.yyyy hh:mm")
(c) [MaxU](https://stackoverflow.com/users/5741205/maxu?tab=profile)
"""
def set_column_format(ws, column_letter, fmt):
for cell in ws[column_letter]:
cell.number_format = fmt
filename = Path(filename)
file_exists = filename.is_file()
# process parameters
# calculate first column number
# if the DF will be written using `index=True`, then `first_col = 2`, else `first_col = 1`
first_col = int(to_excel_kwargs.get("index", True)) + 1
# ignore [engine] parameter if it was passed
if 'engine' in to_excel_kwargs:
to_excel_kwargs.pop('engine')
# save content of existing sheets
if file_exists:
wb = load_workbook(filename)
sheet_names = wb.sheetnames
sheet_exists = sheet_name in sheet_names
sheets = {ws.title: ws for ws in wb.worksheets}
with pd.ExcelWriter(
filename.with_suffix(".xlsx"),
engine="openpyxl",
mode="a" if file_exists else "w",
if_sheet_exists="new" if file_exists else None,
date_format=fmt_date,
datetime_format=fmt_datetime,
storage_options=storage_options
) as writer:
if file_exists:
# try to open an existing workbook
writer.book = wb
# get the last row in the existing Excel sheet
# if it was not specified explicitly
if startrow is None and sheet_name in writer.book.sheetnames:
startrow = writer.book[sheet_name].max_row
# truncate sheet
if truncate_sheet and sheet_name in writer.book.sheetnames:
# index of [sheet_name] sheet
idx = writer.book.sheetnames.index(sheet_name)
# remove [sheet_name]
writer.book.remove(writer.book.worksheets[idx])
# create an empty sheet [sheet_name] using old index
writer.book.create_sheet(sheet_name, idx)
# copy existing sheets
writer.sheets = sheets
else:
# file doesn't exist, we are creating a new one
startrow = 0
# write out the DataFrame to an ExcelWriter
df.to_excel(writer, sheet_name=sheet_name, **to_excel_kwargs)
worksheet = writer.sheets[sheet_name]
if autofilter:
worksheet.auto_filter.ref = worksheet.dimensions
for xl_col_no, dtyp in enumerate(df.dtypes, first_col):
col_no = xl_col_no - first_col
width = max(df.iloc[:, col_no].astype(str).str.len().max(),
len(df.columns[col_no]) + 6)
width = min(max_col_width, width)
column_letter = get_column_letter(xl_col_no)
worksheet.column_dimensions[column_letter].width = width
if np.issubdtype(dtyp, np.integer):
set_column_format(worksheet, column_letter, fmt_int)
if np.issubdtype(dtyp, np.floating):
set_column_format(worksheet, column_letter, fmt_float)
if file_exists and sheet_exists:
# move (append) rows from new worksheet to the `sheet_name` worksheet
wb = load_workbook(filename)
# retrieve generated worksheet name
new_sheet_name = set(wb.sheetnames) - set(sheet_names)
if new_sheet_name:
new_sheet_name = list(new_sheet_name)[0]
# copy rows written by `df.to_excel(...)` to
copy_excel_cell_range(
src_ws=wb[new_sheet_name],
tgt_ws=wb[sheet_name],
tgt_min_row=startrow + 1,
with_style=True
)
# remove new (generated by Pandas) worksheet
del wb[new_sheet_name]
wb.save(filename)
wb.close()
import os from openpyxl import load_workbook def append_df_to_excel(filename, df, sheet_name='Sheet1', startrow=None, truncate_sheet=False, **to_excel_kwargs): """ Append a DataFrame [df] to existing Excel file [filename] into [sheet_name] Sheet. If [filename] doesn't exist, then this function will create it. @param filename: File path or existing ExcelWriter (Example: '/path/to/file.xlsx') @param df: DataFrame to save to workbook @param sheet_name: Name of sheet which will contain DataFrame. (default: 'Sheet1') @param startrow: upper left cell row to dump data frame. Per default (startrow=None) calculate the last row in the existing DF and write to the next row... @param truncate_sheet: truncate (remove and recreate) [sheet_name] before writing DataFrame to Excel file @param to_excel_kwargs: arguments which will be passed to `DataFrame.to_excel()` [can be a dictionary] @return: None Usage examples: >>> append_df_to_excel('d:/temp/test.xlsx', df) >>> append_df_to_excel('d:/temp/test.xlsx', df, header=None, index=False) >>> append_df_to_excel('d:/temp/test.xlsx', df, sheet_name='Sheet2', index=False) >>> append_df_to_excel('d:/temp/test.xlsx', df, sheet_name='Sheet2', index=False, startrow=25) (c) [MaxU](https://stackoverflow.com/users/5741205/maxu?tab=profile) """ # Excel file doesn't exist - saving and exiting if not os.path.isfile(filename): df.to_excel( filename, sheet_name=sheet_name, startrow=startrow if startrow is not None else 0, **to_excel_kwargs) return # ignore [engine] parameter if it was passed if 'engine' in to_excel_kwargs: to_excel_kwargs.pop('engine') writer = pd.ExcelWriter(filename, engine='openpyxl', mode='a') # try to open an existing workbook writer.book = load_workbook(filename) # get the last row in the existing Excel sheet # if it was not specified explicitly if startrow is None and sheet_name in writer.book.sheetnames: startrow = writer.book[sheet_name].max_row # truncate sheet if truncate_sheet and sheet_name in writer.book.sheetnames: # index of [sheet_name] sheet idx = writer.book.sheetnames.index(sheet_name) # remove [sheet_name] writer.book.remove(writer.book.worksheets[idx]) # create an empty sheet [sheet_name] using old index writer.book.create_sheet(sheet_name, idx) # copy existing sheets writer.sheets = {ws.title:ws for ws in writer.book.worksheets} if startrow is None: startrow = 0 # write out the new sheet df.to_excel(writer, sheet_name, startrow=startrow, **to_excel_kwargs) # save the workbook writer.save()
使用示例:
filename = r'C:\OCC.xlsx'
append_df_to_excel(filename, df)
append_df_to_excel(filename, df, header=None, index=False)
append_df_to_excel(filename, df, sheet_name='Sheet2', index=False)
append_df_to_excel(filename, df, sheet_name='Sheet2', index=False, startrow=25)
c:/temp/test.xlsx:
PS如果您不想重复列名,您可能还想指定header=None
...
更新:您可能还想检查这个旧解决方案
如果您不是严格寻找 excel 文件,则将输出作为 csv 文件获取,然后将 csv 复制到新的 excel 文件中
df.to_csv('filepath', mode='a', index = False, header=None)
模式 = 'a'
a 表示追加
这是一种迂回的方式,但效果很好!
import pandas as pd
import openpyxl
workbook = openpyxl.load_workbook("test.xlsx")
writer = pd.ExcelWriter('test.xlsx', engine='openpyxl')
writer.book = workbook
writer.sheets = dict((ws.title, ws) for ws in workbook.worksheets)
data_df.to_excel(writer, 'Existing_sheetname')
writer.save()
writer.close()
基于 MaxU 和其他人的代码和注释,但简化为仅修复 pandas ExcelWriter 导致 to_excel 创建新工作表而不是在附加模式下附加到现有工作表的错误。
正如其他人所指出的,to_excel 使用 ExcelWriter.sheets 属性,并且在 ExcelWriter 时不会填充该属性。
Fix 是单行的,否则代码是标准的 pandas 方法,如 to_excel 中所述。
# xl_path is destination xlsx spreadsheet
with pd.ExcelWriter(xl_path, 'openpyxl', mode='a') as writer:
# fix line
writer.sheets = dict((ws.title, ws) for ws in writer.book.worksheets)
df.to_excel(writer, sheet_name)
如果您每次都在工作表上使用 ExcelWriter,它将覆盖前一个工作表,那么可见的只是您附加到工作簿的最后一个数据表。 相反,您可以维护一个最初为 1 的计数器,您需要为其初始化 excel 工作表并使用现有的方法添加初始数据
writer = pd.ExcelWriter(output_file, engine='openpyxl')
df = pd.read_excel(output_file, sheet_name='TestSheet1')
这个问题在这里已经有一段时间了。 答案还可以,但我相信这将解决大多数人的问题。
只需使用 glob 访问特定目录中的文件,遍历它们,为每个文件创建一个数据框,将其附加到最后一个文件,然后导出到文件夹。 我还包括了注释掉的代码,以通过 csvs 运行。
import os
import pandas as pd
import glob
# put in path to folder with files you want to append
# *.xlsx or *.csv will get all files of that type
path = "C:/Users/Name/Folder/*.xlsx"
#path = "C:/Users/Name/Folder/*.csv"
# initialize a empty df
appended_data = pd.DataFrame()
#loop through each file in the path
for file in glob.glob(path):
print(file)
# create a df of that file path
df = pd.read_excel(file, sheet_name = 0)
#df = pd.read_csv(file, sep=',')
# appened it
appended_data = appended_data.append(df)
appended_data
# export the appeneded data to a folder of your choice
exportPath = 'C:/My/EXPORT/PATH/appended_dataExport.csv'
appended_data.to_csv(os.path.join(exportPath),index=False)
作为@david的补充,如果您不关心索引<\/strong>并且可以使用.csv<\/strong> ,此功能有助于将任何 df 附加到现有 csv
def append_df(self, path_file, df):
with open(path_file, 'a+') as f:
df.to_csv(f, header=f.tell() == 0, encoding='utf-8', index=False)
这对我有用
import os
import openpyxl
import pandas as pd
from openpyxl.utils.dataframe import dataframe_to_rows
file = r"myfile.xlsx"
df = pd.DataFrame({'A': 1, 'B': 2})
# create excel file
if os.path.isfile(file): # if file already exists append to existing file
workbook = openpyxl.load_workbook(file) # load workbook if already exists
sheet = workbook['my_sheet_name'] # declare the active sheet
# append the dataframe results to the current excel file
for row in dataframe_to_rows(df, header = False, index = False):
sheet.append(row)
workbook.save(file) # save workbook
workbook.close() # close workbook
else: # create the excel file if doesn't already exist
with pd.ExcelWriter(path = file, engine = 'openpyxl') as writer:
df.to_excel(writer, index = False, sheet_name = 'my_sheet_name')
from openpyxl import load_workbook
wb = load_workbook(filepath)
ws = wb["Sheet1"]
df = dataframe.values.tolist()
for i in range(len(df)):
ws.append(df[i])
wb.save(filepath)
将 DataFrame 附加到现有的 excel 文件<\/strong>
使用 ExcelWrite 将 DataFrame 附加到现有的 Excel 文件。 这是一种简单的方法,并使用现有的库功能。
with pd.ExcelWriter('existing_excel_file.xlsx',mode='a') as writer:
df.to_excel(writer, sheet_name='existing_sheet_name')
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.