How to input numerical data into Tensorflow ResNet50 model for regression?

Question

I currently have a TensorFlow model (ResNet50), which takes a single image input and outputs a continuous value via regression (ranges from 0.8 - 2.0). The dataset has 3000 different patients, each of which has an image and several numerical data points (specifically age, gender, weight). I'm able to get decent accuracy by training on the images, but I was wondering how I could also add the numerical data points as separate inputs. The numerical data is in a csv file where each row is a seperate patients and different columns contain different values. Here is what I have for 1 image:

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.resnet import ResNet50
from tensorflow.keras.applications.resnet_rs import ResNetRS50
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, GlobalMaxPooling2D
from tensorflow.keras.optimizers import Adam
import warnings

labels = pd.read_csv('[path goes here]')
labels.head()

def load_train(path):
    
    """
    It loads the train part of dataset from path
    """
    #1 / 255
    labels = pd.read_csv('[Target Dataframe path]')
    train_datagen = ImageDataGenerator(validation_split=0.2, rescale=None)
    train_gen_flow = train_datagen.flow_from_dataframe(
        dataframe=labels,
        directory='[Images folder path]',
        x_col='ID',
        y_col='Value',
        target_size=(224, 224),
        batch_size=32,
        class_mode='raw',
        subset = 'training',
        seed=1234)

    return train_gen_flow


def load_test(path):
    
    """
    It loads the validation/test part of dataset from path
    """
    labels = pd.read_csv('[Target Dataframe path]')
    validation_datagen = ImageDataGenerator(validation_split=0.2, rescale=None)
    test_gen_flow = validation_datagen.flow_from_dataframe(
    dataframe = labels,
    directory='[Images folder path]',
    x_col="ID",
    y_col="Value", 
    class_mode="raw", 
    target_size=(224,224), 
    batch_size=32,
    subset = "validation",
    seed=1234,
    )


    return test_gen_flow


def create_model(input_shape):
    
    """
    It defines the model
    """

    backbone = ResNetRS50(input_shape=input_shape, weights='imagenet', include_top=False)
    model = Sequential()
    model.add(backbone)
    
    model.add(Dropout(0.3))
    model.add(GlobalMaxPooling2D())
    
    model.add(Dense(1, activation='linear'))
    optimizer = Adam(learning_rate=0.0003)
    model.compile(optimizer=optimizer, loss='mae', metrics=['mae'])
    print(model.summary())

    return model


def train_model(model, train_data, test_data, batch_size=32, epochs=100,
                steps_per_epoch=None, validation_steps=None):

    """
    Trains the model given the parameters
    """
    history = model.fit(train_data, validation_data=test_data, batch_size=batch_size, 
              epochs=epochs, steps_per_epoch=steps_per_epoch, 
              validation_steps=validation_steps, verbose=2)

    # Get training and test loss histories
    training_loss = history.history['loss']
    test_loss = history.history['val_loss']

# Create count of the number of epochs
    epoch_count = range(1, len(training_loss) + 1)

# Visualize loss history
    plt.plot(epoch_count, training_loss, 'r--')
    plt.plot(epoch_count, test_loss, 'b-')
    plt.legend(['Training Loss', 'Test Loss'])
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.show();

    return model

path = '[Full set path (contains image folder, target csv, and numerical feature csv]'

train_data = load_train(path)
test_data = load_test(path)

#build a model
model = create_model(input_shape = (224, 224, 3))

model = train_model(model, train_data, test_data)

Answer 1

You can use Functional API that gives you more flexifility (multiple inputs or outputs, skip connections between layers etc.). In your case you probably want to have two seperate inputs. Encode the numerical features, pass them throught a dense layer(s) and add them to your main network before the last dense layer.

https://www.tensorflow.org/guide/keras/functional

https://machinelearningmastery.com/keras-functional-api-deep-learning/