如何使用 tidyr::unite 函数删除 NA？

Question

After combining several columns with tidyr::unite() , NAs from missing data remain in my character vector, which I do not want.

I have a series of medical diagnoses per row (1 per column) and would like to benchmark searching for a series of codes via. %in% and grepl() .

There is an open issue on Github on this problem, is there any movement - or work arounds? I would like to keep the vector comma-separated.

Here is a representative example:

library(dplyr)
library(tidyr)

df <- data_frame(a = paste0("A.", rep(1, 3)), b = " ", c = c("C.1", "C.3", " "), d = "D.4", e = "E.5")

cols <- letters[2:4]
df[, cols] <- gsub(" ", NA_character_, as.matrix(df[, cols]))
tidyr::unite(df, new, cols, sep = ",")

Current output:

# # A tibble: 3 x 3
#   a     new        e    
#   <chr> <chr>      <chr>
# 1 A.1   NA,C.1,D.4 E.5  
# 2 A.1   NA,C.3,D.4 E.5  
# 3 A.1   NA,NA,D.4  E.5 

Desired output:

# # A tibble: 3 x 3
#   a     new        e    
#   <chr> <chr>      <chr>
# 1 A.1   C.1,D.4    E.5  
# 2 A.1   C.3,D.4    E.5  
# 3 A.1   D.4        E.5 

Answer 1

In the new tidyr , you can now use na.rm parameter to remove NA values.

library(tidyr)
library(dplyr)

df %>% unite(new, cols, sep = ",", na.rm = TRUE)

#   a     new     e    
#  <chr> <chr>   <chr>
#1 A.1   C.1,D.4 E.5  
#2 A.1   C.3,D.4 E.5  
#3 A.1   D.4     E.5  

However, NA s would not be removed if have columns are factors. We need to change them to character before using unite .

df %>% 
  mutate_all(as.character) %>%
  unite(new, cols, sep = ",", na.rm = TRUE)

---

You could also use base R apply method for the same.

apply(df[cols], 1, function(x) toString(na.omit(x)))
#[1] "C.1, D.4" "C.3, D.4" "D.4" 

data

df <- data_frame(
a = c("A.1", "A.1", "A.1"),
b = c(NA_character_, NA_character_, NA_character_),
c = c("C.1", "C.3", NA),
d = c("D.4", "D.4", "D.4"),
e = c("E.5", "E.5", "E.5")
)

cols <- letters[2:4]

Answer 2

You could use regex to remove the NAs after they are created:

library(dplyr)
library(tidyr)

df <- data_frame(a = paste0("A.", rep(1, 3)), 
                 b = " ", 
                 c = c("C.1", "C.3", " "), 
                 d = "D.4", e = "E.5")

cols <- letters[2:4]
df[, cols] <- gsub(" ", NA_character_, as.matrix(df[, cols]))
tidyr::unite(df, new, cols, sep = ",") %>% 
     dplyr::mutate(new = stringr::str_replace_all(new, 'NA,?', ''))  # New line

Output:

# A tibble: 3 x 3
  a     new     e    
  <chr> <chr>   <chr>
1 A.1   C.1,D.4 E.5  
2 A.1   C.3,D.4 E.5  
3 A.1   D.4     E.5  

Answer 3

You can avoid inserting them by iterating over the rows:

library(tidyverse)

df <- data_frame(
    a = c("A.1", "A.1", "A.1"),
    b = c(NA_character_, NA_character_, NA_character_),
    c = c("C.1", "C.3", NA),
    d = c("D.4", "D.4", "D.4"),
    e = c("E.5", "E.5", "E.5")
)

cols <- letters[2:4]

df %>% mutate(x = pmap_chr(.[cols], ~paste(na.omit(c(...)), collapse = ',')))
#> # A tibble: 3 x 6
#>   a     b     c     d     e     x      
#>   <chr> <chr> <chr> <chr> <chr> <chr>  
#> 1 A.1   <NA>  C.1   D.4   E.5   C.1,D.4
#> 2 A.1   <NA>  C.3   D.4   E.5   C.3,D.4
#> 3 A.1   <NA>  <NA>  D.4   E.5   D.4

or using tidyr 's underlying stringi package,

df %>% mutate(x = pmap_chr(.[cols], ~stringi::stri_flatten(
    c(...), collapse = ",", 
    na_empty = TRUE, omit_empty = TRUE
)))
#> # A tibble: 3 x 6
#>   a     b     c     d     e     x      
#>   <chr> <chr> <chr> <chr> <chr> <chr>  
#> 1 A.1   <NA>  C.1   D.4   E.5   C.1,D.4
#> 2 A.1   <NA>  C.3   D.4   E.5   C.3,D.4
#> 3 A.1   <NA>  <NA>  D.4   E.5   D.4

The problem is that iterating over rows usually entails making a lot of calls, and can therefore be quite slow at scale. Unfortunately, there doesn't appear to be a great vectorized alternative for removing NA s before joining the strings.

Answer 4

Thanks all, I've put together a summary of the solutions and bench-marked on my data:

library(microbenchmark)
library(dplyr)
library(stringr)
library(tidyr)
library(biometrics) # has my helper function for column selection

cols <- biometrics::variables(c("diagnosis", "dagger", "ediag"), 20) 
system.time({
  df <- dat[, cols]
  df <- gsub(" ", NA_character_, as.matrix(df)) %>% tbl_df()
})

microbenchmark(
  ## search by base R `match()` function
  match_spaces = apply(dat, 1, function(x) any(c("A37.0","A37.1","A37.8","A37.9") %in% x[cols])), # original search (match)

  match_NAs = apply(df, 1, function(x) any(c("A37.0","A37.1","A37.8","A37.9") %in% x[cols])), # matching with " " replaced by NAs with gsub 

  ## search by base R 'grep()' function - the same regex is used in each case
  regex_str_replace_all = tidyr::unite(df, new, cols, sep = ",") %>% # grepl search with NAs removed with `stringr::str_replace_all()`
    mutate(new = str_replace_all(new, "NA,?", "")) %>%
    apply(1, function(x) grepl("A37.*", x, ignore.case = T)),

  regex_toString = tidyr::unite(df, new, cols, sep = ",") %>%  # grepl search with NAs removed with `apply()` & `toString()`
    mutate(new = apply(df[cols], 1, function(x) toString(na.omit(x)))) %>%
    apply(1, function(x) grepl("A37.*", x, ignore.case = T)),

  regex_row_iteration = df %>% # grepl search after iterating over rows (using syntax I'm not familiar with and need to learn!)
    mutate(new = pmap_chr(.[cols], ~paste(na.omit(c(...)), collapse = ','))) %>%
    select(new) %>%
    apply(1, function(x) grepl("A37.*", x, ignore.case = T)),

  regex_stringi = df %>% mutate(new = pmap_chr(.[cols], ~stringi::stri_flatten( # grepl after stringi
    c(...), collapse = ",", 
    na_empty = TRUE, omit_empty = TRUE
  ))) %>%
    select(new) %>%
    apply(1, function(x) grepl("A37.*", x, ignore.case = T)),

  times = 10L
)

# Unit: milliseconds
#                   expr        min        lq      mean    median        uq       max neval
#           match_spaces 14820.2076 15060.045 15558.092 15573.885 15901.015 16521.855    10
#              match_NAs   998.3184  1061.973  1191.691  1203.849  1301.511  1378.314    10
#  regex_str_replace_all  1464.4502  1487.473  1637.832  1596.522  1701.718  2114.055    10
#         regex_toString  4324.0914  4341.725  4631.998  4487.373  4977.603  5439.026    10
#    regex_row_iteration  5794.5994  6107.475  6458.339  6436.273  6720.185  7256.980    10
#          regex_stringi  4772.3859  5267.456  5466.510  5436.804  5806.272  6011.713    10

It looks like %in% is the winner - after replacing empty values (" ") with NAs. If If I go with regular expressions, then removing NAs with stringr::string_replace_all() is the quickest.

Answer 5

You might get some errors if you remove them while you use the unite function. I would just remove them from the column after the fact.

df <- data_frame(a = paste0("A.", rep(1, 3)), b = " ", c = c("C.1", "C.3", " "), d = "D.4", e = "E.5")

cols <- letters[2:4]
df[, cols] <- gsub(" ", NA_character_, as.matrix(df[, cols]))
df <- tidyr::unite(df, new, cols, sep = ",")

df$new <- gsub("NA,","",df$new)