data.table equivalent of dplyr::filter_at

Question

Consider the data:

library(data.table)
library(magrittr)

vec1 <- c("Iron", "Copper")

vec2 <- c("Defective", "Passed", "Error")

set.seed(123)
a1 <- sample(x = vec1, size = 20, replace = T)
b1 <- sample(x = vec2, size = 20, replace = T)

set.seed(1234)
a2 <- sample(x = vec1, size = 20, replace = T)
b2 <- sample(x = vec2, size = 20, replace = T)

DT <- data.table(
  c(1:20), a1, b1, a2, b2
) %>% .[order(V1)]

names(DT) <- c("id", "prod_name_1", "test_1", "prod_name_2", "test_2")

I need to filter the rows whose value for test_1 OR test_2 is "Passed" . Thus if neither of these columns have the specified value, then delete the row. With dplyr , we can use the filter_at() verb:

> # dplyr solution...
> 
> cols <- grep(x = names(DT), pattern = "test", value = T, ignore.case = T)
> 
> 
> DT %>% 
+   dplyr::filter_at(.vars = grep(x = names(DT), pattern = "test", value = T, ignore.case = T), 
+                    dplyr::any_vars(. == "Passed")) -> DT.2
> 
> DT.2
  id prod_name_1 test_1 prod_name_2    test_2
1  3        Iron Passed      Copper Defective
2  5      Copper Passed      Copper Defective
3  7      Copper Passed        Iron    Passed
4  8      Copper Passed        Iron     Error
5 11      Copper  Error      Copper    Passed
6 14      Copper  Error      Copper    Passed
7 16      Copper Passed      Copper     Error

Cool. Is there any similar way to perform this operation in data.table ?

This is the closest I've got:

> lapply(seq_along(cols), function(x){
+   
+   setkeyv(DT, cols[[x]])
+   
+   DT["Passed"]
+   
+ }) %>% 
+   do.call(rbind,.) %>% 
+   unique -> DT.3
> 
> DT.3
   id prod_name_1 test_1 prod_name_2    test_2
1:  3        Iron Passed      Copper Defective
2:  5      Copper Passed      Copper Defective
3:  8      Copper Passed        Iron     Error
4: 16      Copper Passed      Copper     Error
5:  7      Copper Passed        Iron    Passed
6: 11      Copper  Error      Copper    Passed
7: 14      Copper  Error      Copper    Passed
> 
> identical(data.table(DT.2)[order(id)], DT.3[order(id)])
[1] TRUE

Does any of you have a more elegant solution? Preferably something contained in a verb like dplyr::filter_at() .

Answer 1

We can specify the 'cols' in .SDcols , loop through the Subset of Data.table ( .SD ) to compare whether the value is "Passed", Reduce it to a single vector with | and subset the rows

res2 <- DT[DT[,  Reduce(`|`, lapply(.SD, `==`, "Passed")), .SDcols = cols]]

Comparing with the dplyr output in the OP's post

identical(as.data.table(res1), res2)
#[1] TRUE

Answer 2

I'd transform the data...

# store the data in long form...

m = melt(DT, id = "id", 
  meas = patterns("prod_name", "test"), 
  value.name = c("prod_name", "test"), variable.name = "prod_num")

setorder(m, id, prod_num)      

# store binary test variable as logical...

testmap = data.table(
  old = c("Defective", "Passed", "Error"), 
  new = c(FALSE, TRUE, NA))
m[testmap, on=.(test = old), passed := i.new]

m[, test := NULL]

So the data now looks like

    id prod_num prod_name passed
 1:  1        1      Iron     NA
 2:  1        2      Iron  FALSE
 3:  2        1    Copper     NA
 4:  2        2    Copper  FALSE
 5:  3        1      Iron   TRUE
 6:  3        2    Copper  FALSE
 7:  4        1    Copper     NA
 8:  4        2    Copper  FALSE
 9:  5        1    Copper   TRUE
10:  5        2    Copper  FALSE
11:  6        1      Iron     NA
12:  6        2    Copper     NA
13:  7        1    Copper   TRUE
14:  7        2      Iron   TRUE
15:  8        1    Copper   TRUE
16:  8        2      Iron     NA
17:  9        1    Copper  FALSE
18:  9        2    Copper     NA
19: 10        1      Iron  FALSE
20: 10        2    Copper  FALSE
21: 11        1    Copper     NA
22: 11        2    Copper   TRUE
23: 12        1      Iron     NA
24: 12        2    Copper  FALSE
25: 13        1    Copper     NA
26: 13        2      Iron  FALSE
27: 14        1    Copper     NA
28: 14        2    Copper   TRUE
29: 15        1      Iron  FALSE
30: 15        2      Iron  FALSE
31: 16        1    Copper   TRUE
32: 16        2    Copper     NA
33: 17        1      Iron     NA
34: 17        2      Iron  FALSE
35: 18        1      Iron  FALSE
36: 18        2      Iron  FALSE
37: 19        1      Iron  FALSE
38: 19        2      Iron     NA
39: 20        1    Copper  FALSE
40: 20        2      Iron     NA
    id prod_num prod_name passed

You can then filter to ids with passed products like...

res = m[, if(isTRUE(any(passed))) .SD, by=id]

    id prod_num prod_name passed
 1:  3        1      Iron   TRUE
 2:  3        2    Copper  FALSE
 3:  5        1    Copper   TRUE
 4:  5        2    Copper  FALSE
 5:  7        1    Copper   TRUE
 6:  7        2      Iron   TRUE
 7:  8        1    Copper   TRUE
 8:  8        2      Iron     NA
 9: 11        1    Copper     NA
10: 11        2    Copper   TRUE
11: 14        1    Copper     NA
12: 14        2    Copper   TRUE
13: 16        1    Copper   TRUE
14: 16        2    Copper     NA

For browsability...

dcast(res, id ~ prod_num, value.var = c("prod_name", "passed"))

   id prod_name_1 prod_name_2 passed_1 passed_2
1:  3        Iron      Copper     TRUE    FALSE
2:  5      Copper      Copper     TRUE    FALSE
3:  7      Copper        Iron     TRUE     TRUE
4:  8      Copper        Iron     TRUE       NA
5: 11      Copper      Copper       NA     TRUE
6: 14      Copper      Copper       NA     TRUE
7: 16      Copper      Copper     TRUE       NA