KFold cross validation in Tensorflow

Question

I am trying to implement KFold validation using sklearn and Tensorflow package in Neural Network.

my code looks like this.

def training(self):
    n_split = 3
    instances = self.instance
    labels = self.labels
    for train_index, test_index in KFold(n_split).split(instances):
        x_train, x_test = instances[train_index], instances[test_index]
        y_train, y_test = labels[train_index], labels[test_index]
        model = self.mlp_model()
        model.fit(x_train, y_train, epochs=20)

        print('Model Evaluation', model.evaluate(x_test, y_test))

Unfortunately, I am getting an error

raise KeyError(f"None of [{key}] are in the [{axis_name}]")
KeyError: "None of [Int64Index([160, 161, 162, 163, 164, 165, 166, 167, 168, 169,\n            ...\n            468, 469, 470, 471, 472, 473, 474, 475, 476, 477],\n           dtype='int64', length=318)] are in the [columns]"

The shape of my instances is (478,15) and labels is (478,).

The instance dataset looks like

|     | SibSp | Parch | Female | Male | C | Q | S | 1 | 2 | 3 | Master |
|-----|-------|-------|--------|------|---|---|---|---|---|---|--------|
| 374 | 0     | 0     | 0      | 1    | 0 | 0 | 1 | 0 | 0 | 1 | 1      |
| 579 | 0     | 0     | 0      | 1    | 0 | 0 | 1 | 0 | 0 | 1 | 0      |
| 784 | 0     | 0     | 0      | 1    | 1 | 0 | 1 | 0 | 0 | 1 | 0      |

labels look like

|     | Age |
|-----|-----|
| 127 | 24  |
| 262 | 52  |
| 615 | 24  |

What can be the reason for the problem and what can be the possible solution? Looking forward to your reply. Thanks in advance.

Answer 1

Here is a code example of using KFold cross-validation over the CIFAR10 dataset from TensorFlow .

Get the data

import tensorflow as tf
import numpy as np 

(input, target), (_, _) = tf.keras.datasets.cifar10.load_data()

# Parse numbers as floats
input = input.astype('float32') / 255
target = tf.keras.utils.to_categorical(target , num_classes=10)

print(input.shape, target.shape)
# (50000, 32, 32, 3) (50000, 10)

Model

def my_model():
    return tf.keras.Sequential(
        [
            tf.keras.Input(shape=(32, 32, 3)),
            tf.keras.layers.Conv2D(16, 3, activation="relu"),
            tf.keras.layers.Conv2D(32, 3, activation="relu"),
            tf.keras.layers.Conv2D(64, 3, activation="relu"),
            tf.keras.layers.Conv2D(128, 3, activation="relu"),
            tf.keras.layers.Conv2D(256, 3, activation="relu"),
            tf.keras.layers.GlobalAveragePooling2D(), 
            tf.keras.layers.Dense(10, activation='softmax')
        
        ]
    )

We will call this bad boy in the KFold loop.

K-Fold training

from sklearn.model_selection import KFold
import numpy as np

for kfold, (train, test) in enumerate(KFold(n_splits=3, 
                                shuffle=True).split(input, target)):
    # clear the session 
    tf.keras.backend.clear_session()

    # calling the model and compile it 
    seq_model = my_model()
    seq_model.compile(
        loss  = tf.keras.losses.CategoricalCrossentropy(),
        metrics  = tf.keras.metrics.CategoricalAccuracy(),
        optimizer = tf.keras.optimizers.Adam())

    print('Train Set')
    print(input[train].shape)
    print(target[train].shape)

    print('Test Set')
    print(input[test].shape)
    print(target[test].shape)

    # run the model 
    seq_model.fit(input[train], target[train],
              batch_size=128, epochs=2, validation_data=(input[test], target[test]))
    seq_model.save_weights(f'wg_{kfold}.h5')

Logs

Train Set
(33333, 32, 32, 3)
(33333, 10)
Test Set
(16667, 32, 32, 3)
(16667, 10)
Epoch 1/2
11s 41ms/step - loss: 1.9961 - categorical_accuracy: 0.2363 - 
val_loss: 1.6851 - val_categorical_accuracy: 0.3435
Epoch 2/2
10s 37ms/step - loss: 1.6322 - categorical_accuracy: 0.3836 - 
val_loss: 1.5780 - val_categorical_accuracy: 0.4193

Train Set
(33333, 32, 32, 3)
(33333, 10)
Test Set
(16667, 32, 32, 3)
(16667, 10)
Epoch 1/2
11s 39ms/step - loss: 2.0254 - categorical_accuracy: 0.2197 - 
val_loss: 1.6799 - val_categorical_accuracy: 0.3601
Epoch 2/2
10s 37ms/step - loss: 1.6687 - categorical_accuracy: 0.3739 - 
val_loss: 1.5222 - val_categorical_accuracy: 0.4362

Train Set
(33334, 32, 32, 3)
(33334, 10)
Test Set
(16666, 32, 32, 3)
(16666, 10)
Epoch 1/2
11s 41ms/step - loss: 2.0170 - categorical_accuracy: 0.2212 - 
val_loss: 1.7452 - val_categorical_accuracy: 0.3134
Epoch 2/2
10s 37ms/step - loss: 1.7110 - categorical_accuracy: 0.3363 - 
val_loss: 1.5928 - val_categorical_accuracy: 0.4164

Updated

Another approach with a pandas data frame.

# load
df = pd.read_csv('train.csv')

# [optional]: shuffle the data
df = df.sample(frac=1).reset_index(drop=True)

# fold split 
kfold = KFold(n_splits=3, shuffle=True)

for each_fold, (trn_idx, val_idx) in enumerate(.split(np.arange(df.shape[0]), 
                                                       df.target.values)):
       # get the folded data 
       train_labels = df.iloc[trn_idx].reset_index(drop=True) 
       val_labels   = df.iloc[val_idx].reset_index(drop=True) 

       # train_labels: training pairs (data + label)
       # do something

Let's give another example with multilabel data. This time we will create a folding column first. See the example below.

# !pip install iterative-stratification
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold

# load the data 
df = pd.read_csv('train.csv')

# add a new column, for now setting all value to -1
# we will update it later 
df.loc[:, 'kfold'] = -1 

# [optional]: shuffle 
df = df.sample(frac=1).reset_index(drop=True)

# as we are now dealing with multi label cases, 
# there would be more than one column for target value 
# so let's grab the target values only 
# we will drop id / image_id / image_name / blabla.. column
target = df.drop('id', axis-1).values 

mskf = MultilabelStratifiedKFold(n_splits=5)
for each_kfold, (trn_idx, val_idx) in enumerate(mskf.split(df, target):
       df.loc[val_idx, 'kfold] = each_fold # updating `kfold` column 

After that, n_split = 5 will appear in column 'kfold', also check with df.head() . Next, we can do something like as follows

def program(fold):
    # get the folded data
    train_labels = df[df.kfold != fold].reset_index(drop=True)
    val_labels   = df[df.kfold == fold].reset_index(drop=True)

    # train_labels: training pairs (data + label)
    # do something


for i in range(n_split):
      program(fold=i) 

Answer 2

So, I found the problem. It was related to how we are accessing the values from dataframe.

for train_index, test_index in KFold(n_split).split(instances):
    x_train, x_test = instances.iloc[train_index], instances.iloc[test_index]
    y_train, y_test = labels.iloc[train_index], labels.iloc[test_index]

Using iloc solves the problem.