用组内的值替换 NA 子集

Question

I need to replace missing values in all columns of a data frame within ID and time point for a subgroup that have data from several sources. If it is not too complicated, it would be best to prioritize data from source B (eg, in case of id 2 for variable Y in the data below).

Using the code below, it currently works (without prioritizing) for one column at the time, but since its a large data frame with millions of rows, it needs to be automated further. Also, I would like to keep it within the data.table framework if possible. Any advice?

# Data
id  time  X  Y   Source
1   2005  67 NA  A
1   2005  NA 1.1 B
1   2005  NA 1.1 B
2   2003  85 NA  B
2   2003  NA 0.4 A
2   2003  85 0.5 B

# Desired output
id  time  X  Y   Source
1   2005  67 1.1 A
1   2005  67 1.1 B
1   2005  67 1.1 B
2   2003  85 0.5 B
2   2003  85 0.4 A
2   2003  85 0.5 B

# Find duplicates
dup <- (duplicated(dat[,c('id','time')])|duplicated(dat[,c('id','time')], fromLast=TRUE))

# Replace NA in column X
library(data.table)
dat[dup & is.na(X), X := dat[!is.na(X)][.SD, on=.(id,time), mult="last", X]]

### Solution based on locf and an internal data.table loop (still slower than tidyverse)

    library(data.table)
    library(zoo)

    cols <- colnames(dat)[c(-1,-2)]
    dat <- dat[order(id,time,Source)] # this combined with na.locf0(fromLast=T) takes care of the priority.
    dup <- (duplicated(dat[,c('id','time')])|duplicated(dat[,c('id','time')], fromLast=TRUE))

    t1 <- Sys.time() 
      dat=rbind(
        dat[!dup],
        dat[dup, lapply(.SD, na.locf0,fromLast = TRUE), by=c('id','time'), .SDcols = cols][
            ,lapply(.SD, na.locf0), by=c('id','time'), .SDcols = cols]
      )
    t2 <- Sys.time()
    t2-t1

Answer 1

library(tidyverse)
library(data.table)

Data <- data.table(id = c(1,1,1,2,2,2), time = c(2005, 2005, 2005, 2003, 2003, 2003), X = c(67, NA, NA, 85, NA, 85),
                       Y = c(NA, 1.1, 1.1, NA, 0.4, 0.5), Source = c("A", "B", "B", "B", "A", "B"))

Data <- Data %>% 
  group_by(id, Source) %>% 
  fill(time, X, Y) %>%
  fill(time, X, Y, .direction = "up")

Data <- Data %>% 
  group_by(id) %>% 
  fill(time, X, Y) %>%
  fill(time, X, Y, .direction = "up")

I am not sure if you mean that source "B" is always preferred or if it is only preferred when the source of the sample is also "B" (and thus the preferred source would be "A" if that sample's source was "A"). This code solves the issue for the latter scenario. It requires tidyverse.

Answer 2

Here are 3 options:

1) Using a for loop with get :

for (x in updcols) {
    DT0[dup & is.na(get(x)), (x) := DT0[!is.na(get(x))][
        .SD, on=.(id,time), mult="last", get(x)]]   
}
DT0

2) Using for loop with non-standard evaluation:

nsef <- function(dat, coln) {
    eval(substitute(
        dat[dup & is.na(V), V := dat[!is.na(V)][.SD, on=.(id,time), mult="last", V]],
        list(V=as.name(coln))
    ))
}
for (x in updcols) {
    nsef(DT1, x)
}
DT1

3) Extract the last non-NA values and perform a join and then update by reference:

lu <- DT2[, lapply(.SD, function(x) last(x[!is.na(x)])), bycols, .SDcols=updcols]
DT2[(dup), (updcols) := 
    lu[.SD, on=bycols, Map(function(x, y) fcoalesce(x, y), 
        mget(paste0("i.", updcols)), mget(updcols))]
]
DT2

You can also use fifelse (version >= 1.12.4) instead of fcoalesce (ie fcoalesce(X, Y) == fifelse(is.na(X), Y, X) ).

I think timing will depend on the characteristics of your actual dataset.

output:

   id time  X   Y Source
1:  1 2005 67 1.1      A
2:  1 2005 67 1.1      B
3:  1 2005 67 1.1      B
4:  2 2003 85 0.5      B
5:  2 2003 85 0.4      A
6:  2 2003 85 0.5      B

data:

library(data.table) #data.table_1.12.6
DT <- fread("id  time  X  Y   Source
1   2005  67 NA  A
1   2005  NA 1.1 B
1   2005  NA 1.1 B
2   2003  85 NA  B
2   2003  NA 0.4 A
2   2003  85 0.5 B")
DT0 <- copy(DT)
DT1 <- copy(DT)
DT2 <- copy(DT)
bycols <- c('id','time')
updcols <- c("X", "Y")
dup <- duplicated(DT, by=bycols) | duplicated(DT, by=bycols, fromLast=TRUE)