错误：`data` 和 `reference` 应该是相同级别的因子。 Logistic 回归的混淆矩阵

Question

I have seen lots of answers with regards to this particular error. I haven't found any answer to it with specifics to my particular issue. Therefore, my problem

This is what I do:

    shortness_breath_data <- data_categ_nosev %>%
dplyr::select(shortness_breath, obesity, asthma, diabetes_type_one, diabetes_type_two, obesity, hypertension, heart_disease, lung_condition, liver_disease, kidney_disease, Covid_tested, Gender) 

And this is put(head(shortness_breath_data)) :

structure(list(shortness_breath = structure(c(1L, 2L, 1L, 1L, 
1L, 2L), .Label = c("No", "Yes"), class = "factor"), obesity = structure(c(1L, 
1L, 2L, 2L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    asthma = structure(c(2L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), diabetes_type_one = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    diabetes_type_two = structure(c(2L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), hypertension = structure(c(1L, 
    1L, 1L, 2L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    heart_disease = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), lung_condition = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    liver_disease = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), kidney_disease = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    Covid_tested = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("negative", 
    "positive"), class = "factor"), Gender = structure(c(2L, 
    1L, 2L, 1L, 1L, 2L), .Label = c("Female", "Male", "Other"
    ), class = "factor")), row.names = c(NA, -6L), class = c("tbl_df", 
"tbl", "data.frame"), problems = structure(list(row = c(2910L, 
35958L), col = c("how_unwell", "how_unwell"), expected = c("a double", 
"a double"), actual = c("How Unwell", "How Unwell"), file = c("'/Users/gabrielburcea/Rprojects/data/data_lev_categorical_no_sev.csv'", 
"'/Users/gabrielburcea/Rprojects/data/data_lev_categorical_no_sev.csv'"
)), row.names = c(NA, -2L), class = c("tbl_df", "tbl", "data.frame"
)))

And I divide this into training and testing dataset.

shortness_breath_data$shortness_breath <- as.factor(shortness_breath_data$shortness_breath)

n <- nrow(shortness_breath_data)
set.seed(22)
trainingdx <- sample(1:n, 0.7 * n)

train <- shortness_breath_data[trainingdx,]
validate <- shortness_breath_data[-trainingdx,]

train %>% distinct(shortness_breath)
validate %>% distinct(shortness_breath)

And just to do the same in case it will ease you job in finding the issue, I provided dput(head(train)) and dput(head(validate))

train dataset:

structure(list(shortness_breath = structure(c(1L, 1L, 1L, 1L, 
1L, 1L), .Label = c("No", "Yes"), class = "factor"), obesity = structure(c(2L, 
1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    asthma = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), diabetes_type_one = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    diabetes_type_two = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), hypertension = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    heart_disease = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), lung_condition = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    liver_disease = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), kidney_disease = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    Covid_tested = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("negative", 
    "positive"), class = "factor"), Gender = structure(c(1L, 
    1L, 1L, 2L, 1L, 2L), .Label = c("Female", "Male", "Other"
    ), class = "factor")), row.names = c(NA, -6L), class = c("tbl_df", 
"tbl", "data.frame"), problems = structure(list(row = c(2910L, 
35958L), col = c("how_unwell", "how_unwell"), expected = c("a double", 
"a double"), actual = c("How Unwell", "How Unwell"), file = c("'/Users/gabrielburcea/Rprojects/data/data_lev_categorical_no_sev.csv'", 
"'/Users/gabrielburcea/Rprojects/data/data_lev_categorical_no_sev.csv'"
)), row.names = c(NA, -2L), class = c("tbl_df", "tbl", "data.frame"
)))

validate dataset:

structure(list(shortness_breath = structure(c(1L, 2L, 2L, 1L, 
1L, 1L), .Label = c("No", "Yes"), class = "factor"), obesity = structure(c(1L, 
1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    asthma = structure(c(2L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), diabetes_type_one = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    diabetes_type_two = structure(c(2L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), hypertension = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    heart_disease = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), lung_condition = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    liver_disease = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("No", 
    "Yes"), class = "factor"), kidney_disease = structure(c(1L, 
    1L, 1L, 1L, 1L, 1L), .Label = c("No", "Yes"), class = "factor"), 
    Covid_tested = structure(c(1L, 1L, 1L, 1L, 1L, 1L), .Label = c("negative", 
    "positive"), class = "factor"), Gender = structure(c(2L, 
    1L, 2L, 2L, 1L, 1L), .Label = c("Female", "Male", "Other"
    ), class = "factor")), row.names = c(NA, -6L), class = c("tbl_df", 
"tbl", "data.frame"), problems = structure(list(row = c(2910L, 
35958L), col = c("how_unwell", "how_unwell"), expected = c("a double", 
"a double"), actual = c("How Unwell", "How Unwell"), file = c("'/Users/gabrielburcea/Rprojects/data/data_lev_categorical_no_sev.csv'", 
"'/Users/gabrielburcea/Rprojects/data/data_lev_categorical_no_sev.csv'"
)), row.names = c(NA, -2L), class = c("tbl_df", "tbl", "data.frame"
)))

And then, I build my logistic regression model with stepwise, forward method.

null_model <- glm(shortness_breath ~ 1, data = train, family = "binomial")

fm_shortness_breath <- glm(shortness_breath ~., data = train, family = "binomial")

stepmodel <- step(null_model, scope = list(lower = null_model, upper = fm_shortness_breath), direction = "forward")

Then I get my summary model and store the predictions in the source data frame.

summary(stepmodel)

validate$pred <- predict(stepmodel, validate, type = "response")

validate$real <- validate$shortness_breath

train$pred <- predict(stepmodel, train, type = "response")
train$real <- train$shortness_breath

Then I plot my ROC curve with no problem:

plot.roc(validate$real, validate$pred, col = "red", main = "ROC Validation Set", percent = TRUE, print.auc = TRUE)

Yet, when I am trying to get my confusion matrix, this is where I get my error. But this is my code:

cm_stepmodel <- confusionMatrix(stepmodel, validate)

And then, the error comes in:

Error: `data` and `reference` should be factors with the same levels.

With Show Traceback:

3.
stop("`data` and `reference` should be factors with the same levels.", call. = FALSE)
2.
confusionMatrix.default(stepmodel, validate)
1.
confusionMatrix(stepmodel, validate)

I simply do not see the problem. And tried several other options but did not work. I have reproduced, step by step the exact approach I am undertaking. And I do not get my answer. Also, I have tag this issue with RMarkdown as well, alongside caret and R, just in case.

Also, libraries used are:

library(tidyverse)
library(conflicted)
library(tidymodels)
library(ggrepel)
library(corrplot)
library(dplyr)
library(corrr) 
library(themis)
library(rsample)
library(caret)
library(forcats)
library(rcompanion)
library(MASS)
library(pROC)
library(ROCR)
library(data.table)

Answer 1

Try to convert your predicted probabilities to labels, and then run your confusionMatrix on this:

validate$pred <- predict(stepmodel, validate, type = "response")
validate$pred_label <- as.factor(ifelse(validate$pred >= 0.5, "Yes", "No"))
confusionMatrix(validate$real, validate$pred) # Error
confusionMatrix(validate$real, validate$pred_label) # This will work

Check that you are correctly assigning labels as in your original dataset in the validate$pred_label statement.

I'm not particularly familiar with confusionMatrix , but the general idea is that you make predictions of labels and compare to the actual labels of your data. It threw an error because you were comparing labels with probabilities -- you needed to assign the labels. Please correct me if I made a conceptual error or coding mistake above.