使用 R 批量插入/更新到 mongoDB

Question

I am trying to insert a lot of data (millions of documents) into mongodb using R from a variety of data frames which I will obtain at different times.

Each data frame will have the same primary id, but can have the same or different attributes.

If the record exists, I would like to add any new attributes and append any existing ones. If the record doesn't exist, I would like to create it.

Is this possible in R efficiently? I have tried to use the wonderful mongolite package, but the insert option fails because of duplicate records existing.

Any pointers greatly appreciated.

Thanks

Iain

id<-LETTERS[1:5]
value1<-paste0("value1_",letters[1:5])
value2<-paste0("value2_",letters[1:5])
value3<-paste0("additional_value_1",letters[1:5])
df1<-as.data.frame(cbind(id,value1))
df2<-as.data.frame(cbind(id,value2))
df3<-as.data.frame(cbind(id,value3))

colnames(df1)<-c('_id','value1')
colnames(df2)<-c('_id','value2')
colnames(df3)<-c('_id','value1')

desired_value1<-paste0( "[",paste(paste0("'",value1,"'"),paste0("'",value3,"'"),sep=","),"]")
df4<-cbind(id,desired_value1,value2)
df4<-as.data.frame(cbind(id,desired_value1,value2))
colnames(df4)<-c("_id","value1","value2")

Answer 1

Note

This answer uses library(rmongodb) , which is no longer supported nor on Cran.

---

This answer will partly depend on how you're getting your 'new' data.frames. I also can't answer the efficiently part either without knowing your setup and size of data, but hopefully this will get you started. Plus, I've found inserting/retreiving millions of records into mongo from R to be quite slow.

One way of doing this would be for every new data.frame you get, bring back the matching records into R and 'join/update' them, then update the database for just those documents, while appending the new data using an update / upsert query.

I also use library(rmongodb) for most of my r-mongodb work

Slightly modifying your data to use id instead of _id :

id<-LETTERS[1:5]
value1<-paste0("value1_",letters[1:5])
value2<-paste0("value2_",letters[1:5])
value3<-paste0("additional_value_1",letters[1:5])
df1<-as.data.frame(cbind(id,value1), stringsAsFactors = F)  ## removed factor levels
df2<-as.data.frame(cbind(id,value2), stringsAsFactors = F)
df3<-as.data.frame(cbind(id,value3), stringsAsFactors = F)

colnames(df1)<-c('id','value1')
colnames(df2)<-c('id','value2')
colnames(df3)<-c('id','value1')

desired_value1<-paste0( "[",paste(paste0("'",value1,"'"),paste0("'",value3,"'"),sep=","),"]")
df4<-cbind(id,desired_value1,value2)
df4<-as.data.frame(cbind(id,desired_value1,value2))
colnames(df4)<-c("_id","value1","value2")

The first step is to insert it into the database

library(rmongodb)  ## my preferred r mongodb package 
library(jsonlite)  ## for viewing/checking results 
library(data.table) ## for fast rbind & data frame manipulation

mongo <- mongo.create()
mongo.is.connected(mongo)
# [1] TRUE

db <- "test"
coll <- "test"

bs <- mongo.bson.from.df(df1)
ns <- paste0(db, ".", coll)

## insert.batch - insert each 'row' of the df as a document
mongo.insert.batch(mongo = mongo, 
                   ns = ns, 
                   lst = bs)  
# [1] TRUE

Retrieve all documents to check the upload

f <- mongo.bson.from.list(list("_id" = 0))  ## to ignore the _id field
res <- mongo.find.all(mongo = mongo, ns = ns, fields = f)
toJSON(res, pretty=T)
# [
#   {
#     "id": ["A"],
#     "value1": ["value1_a"]
#   },
#   {
#     "id": ["B"],
#     "value1": ["value1_b"]
#   },
#   {
#     "id": ["C"],
#     "value1": ["value1_c"]
#   },
#   {
#     "id": ["D"],
#     "value1": ["value1_d"]
#   },
#   {
#     "id": ["E"],
#     "value1": ["value1_e"]
#   }
# ] 

Now, if we want to add our df2$value2 into those documents, we can bring them back into R to manipulate them, then update the database

qry <- list("id" = list("$in" = df2$id))
## mongo shell query: db.test.find({"id" : { "$in" : ["A", "B", ..., ]}})
qry <- mongo.bson.from.list(qry)
f <- list("_id" = 0)
res <- mongo.find.all(mongo = mongo, 
                      ns = ns,
                      query = qry,
                      fields = f)

dt_res <- rbindlist(res)

## set our df2 to data.table, and join onto dt_res
setDT(df2)

## add a new row to df2, with a new id, to check the update.upsert works
df2 <- rbindlist(list(df2, data.table(id = "Z", value2 = "value2_z")))

dt_res <- dt_res[ df2, on="id"]  ## left join to keep our 'z' row
dt_res
#    id   value1   value2
# 1:  A value1_a value2_a
# 2:  B value1_b value2_b
# 3:  C value1_c value2_c
# 4:  D value1_d value2_d
# 5:  E value1_e value2_e
# 6:  Z       NA value2_z

We can now update the database with these new values using udpate and upsert

for(i in 1:nrow(dt_res)){

  crit <- mongo.bson.from.list(list("id" = dt_res[i, id]))
  d <- c(dt_res[i, ])
  mongo.update(mongo = mongo, 
               ns = ns, 
               criteria = crit, 
               objNew = d, 
               flags = c(mongo.update.upsert))    
}

Check the documnets have been udpated by returning everything

f <- mongo.bson.from.list(list("_id" = 0))  ## to ignore the _id field
res <- mongo.find.all(mongo = mongo, ns = ns, fields = f)
toJSON(res, pretty=T)

# [
#   {
#     "id": ["A"],
#     "value1": ["value1_a"],
#     "value2": ["value2_a"]
#   },
#   {
#     "id": ["B"],
#     "value1": ["value1_b"],
#     "value2": ["value2_b"]
#   },
#   {
#     "id": ["C"],
#     "value1": ["value1_c"],
#     "value2": ["value2_c"]
#   },
#   {
#     "id": ["D"],
#     "value1": ["value1_d"],
#     "value2": ["value2_d"]
#   },
#   {
#     "id": ["E"],
#     "value1": ["value1_e"],
#     "value2": ["value2_e"]
#   },
#   {
#     "id": ["Z"],
#     "value1": {},
#     "value2": ["value2_z"]
#   }
# ] 

Note this includes our new 'z' id