在R中的一个大矩阵中加速加载和合并多个.csv文件

Question

我在这里关注一些帖子如何在R中合并多个.csv文件？ 在这里同时读取R中的许多CSV文件并将全部合并到一个数据帧中

我的目的基本上是相同的：在R中将一个很大的csv文件组合成一个大的矩阵倍数。我有这个解决方案，我想尽可能地加快速度：

这里是一个完全可复制的例子； 我有更多更大的文件

 setwd("C:/") #### set an easy directory to create acceptably large files
 #### this takes about 60 seconds
 for(i in 1:80){
   print(80-i)
   write.table(matrix(rnorm(20*3891,0,1),ncol=20),col.names=F,row.names=F,sep=",",file=paste(i,"file.csv",sep=""))
 }
 listfiles<-list.files(path="C:/",pattern="*.csv")
 #### now the problem: this takes about 30-40 seconds; as I have bigger (and much more) files I want to speed up this step
 library(plyr)
 mybigmatrix<-ldply(listfiles,read.csv,header=F)

预先感谢您的任何帮助

Answer 1

也许使用特殊的包和功能，例如readr和功能read_csv（）

mybigmatrix<-ldply(listfiles,readr::read_csv,header=F)

Answer 2

这是一个完全可复制的示例，该示例显示了fread（）的问题，该问题不允许我强制将data.table对象矩阵化。

 setwd("C:/") #### set an easy directory to create acceptably large files
 #### this takes few seconds
 for(i in 1:5){
   print(5-i)
   write.table(matrix(rnorm(5*3891,0,1),nrow=5),col.names=F,row.names=F,sep=",",file=paste(i,"file.csv",sep=""))
 }
 listfiles<-list.files(path="C:/",pattern="*.csv")


 myfread<-function(file){
 data_frame <- fread(file,sep=",",header=FALSE,stringsAsFactors=FALSE,select=c(1:3891),colClasses=c(rep("as.numeric",3891)))
 data_frame
 }

    ######  this is a matrix 25*3891 I want an array of 1297x3x25
    alld<-rbindlist(lapply(listfiles,myfread)) 
    ### why this is in characters??
     as.matrix(alld)
    k<-1297
     m<-3
    vectorr<-as.vector(t(as.matrix(alld)))
    tem <- vectorr
    n <- length(tem)/(k * m)
   tem <- array(tem, c(m, k, n))
   tem <- aperm(tem, c(2, 1, 3))
   xup <- tem #######  here I have characters

Answer 3

我认为这些选项都应该对您有效。

---

setwd("C:/Users/your_path_here/test")
fnames <- list.files()
csv <- lapply(fnames, read.csv)
result <- do.call(rbind, csv)

---

filedir <- setwd("C:/Users/your_path_here/csv_files")
file_names <- dir(filedir)
your_data_frame <- do.call(rbind,lapply(file_names,read.csv))

---

filedir <- setwd("C:/Users/your_path_here/csv_files")
file_names <- dir(filedir)
your_data_frame <- do.call(rbind, lapply(file_names, read.csv, skip = 1, header = FALSE))

---

filedir <- setwd("C:/Users/your_path_here/csv_files")
file_names <- dir(filedir)
your_data_frame <- do.call(rbind, lapply(file_names, read.csv, header = FALSE))

---

temp <- setwd("C:/Users/Excel/Desktop/test")
temp = list.files(pattern="*.csv")
myfiles = lapply(temp, read.delim)

最后，尝试以下操作：

setwd("C:/Users/your_path_here/")

file_list <- list.files()

file_list <- list.files("C:/Users/your_path_here/")

for (file in file_list){

  # if the merged dataset doesn't exist, create it
  if (!exists("dataset")){
    dataset <- read.table(file, header=TRUE, sep="\t")
  }

  # if the merged dataset does exist, append to it
  if (exists("dataset")){
    temp_dataset <-read.table(file, header=TRUE, sep="\t")
    dataset<-rbind(dataset, temp_dataset)
    rm(temp_dataset)
  }

}