Here is my keras model"
input1 = keras.layers.Input(shape=(100,))
...
net1 = keras.layers.Dense(n1, activation='relu')()
input2 = keras.layers.Input(shape=(50,))
...
net2 = keras.layers.Dense(n2, activation='relu')()
merge = keras.layers.concatenate([net1,net2])
output = keras.layers.Dense(num_classes, activation='softmax')(merge)
model = keras.Model(inputs=[input1, input1], outputs=[output])
and tfrecord data (big data):
feature_description = {
'f1': tf.io.FixedLenFeature([100], tf.int64),
'f2': tf.io.FixedLenFeature([50], tf.int64),
'label': tf.io.FixedLenFeature([], tf.int64),
}
def parser(example_proto):
return tf.io.parse_single_example(example_proto, feature_description)
ilename_queue = tf.data.Dataset.list_files(filename_queue, shuffle=True)
dataset = tf.data.TFRecordDataset(filename_queue).map(parser)
how do i fit the data to the model?
I know you are meant to do this:
model.fit([f1,f2], y, epochs=epochs, batch_size=batch_size,validation_split=0.3)
and f1,f2 is df.dataframe.
Here the is snippet i wrote to create dummy dataset and write it to a tfrecord file and also build the model.
import tensorflow as tf
num_classes = 10
n_samples = 10000
f1 = tf.random.uniform(shape=[n_samples, 100], maxval=500, dtype=tf.int32).numpy()
f2 = tf.random.uniform(shape=[n_samples, 50], maxval=500, dtype=tf.int32).numpy()
labels = tf.random.uniform(shape=[n_samples], maxval=num_classes, dtype=tf.int32).numpy()
def make_example(f1, f2, label):
feature = {
'f1': tf.train.Feature(int64_list=tf.train.Int64List(value=f1)),
'f2': tf.train.Feature(int64_list=tf.train.Int64List(value=f2)),
'label': tf.train.Feature(int64_list=tf.train.Int64List(value=[label])),
}
return tf.train.Example(features=tf.train.Features(feature=feature))
def write_tfrecord(f1, f2, labels, tfrecord_path):
with tf.io.TFRecordWriter(tfrecord_path) as writer:
for i in range(len(f1)):
example = make_example(f1[i], f2[i], labels[i])
writer.write(example.SerializeToString())
write_tfrecord(f1, f2, labels, 'test.tfrecord')
n1 = 16
n2 = 16
input1 = tf.keras.layers.Input(shape=(100,))
input2 = tf.keras.layers.Input(shape=(50,))
net1 = tf.keras.layers.Dense(n1, activation='relu')(input1)
net2 = tf.keras.layers.Dense(n2, activation='relu')(input2)
merge = tf.keras.layers.concatenate([net1, net2])
output = tf.keras.layers.Dense(num_classes, activation='softmax')(merge)
model = tf.keras.Model(inputs=[input1, input2], outputs=[output])
Now parsing your tfrecord file and creating a tf.data.Dataset
object should be strightforward. Since your model has two inputs and a single output, your tf.data.Dataset
should have a matching structure. So this is how i did it
feature_description = {
'f1': tf.io.FixedLenFeature([100], tf.int64),
'f2': tf.io.FixedLenFeature([50], tf.int64),
'label': tf.io.FixedLenFeature([], tf.int64),
}
def parser(example_proto):
parsed_example = tf.io.parse_single_example(example_proto, feature_description)
f1 = parsed_example['f1']
f2 = parsed_example['f2']
label = parsed_example['label']
return (f1, f2), label
dataset = tf.data.TFRecordDataset('test.tfrecord')
dataset = dataset.map(parser, num_parallel_calls=tf.data.experimental.AUTOTUNE)
dataset = dataset.batch(4)
dataset = dataset.shuffle(16)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
On printing the structure of dataset
you should see the following output
<PrefetchDataset shapes: (((None, 100), (None, 50)), (None,)), types: ((tf.int64, tf.int64), tf.int64)>
Making sure all of this works and the training loop runs without any errors
model.summary()
model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd')
model.fit(dataset, steps_per_epoch=10, epochs=2)
Here is the final output
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_7 (InputLayer) [(None, 100)] 0
__________________________________________________________________________________________________
input_8 (InputLayer) [(None, 50)] 0
__________________________________________________________________________________________________
dense_6 (Dense) (None, 16) 1616 input_7[0][0]
__________________________________________________________________________________________________
dense_7 (Dense) (None, 16) 816 input_8[0][0]
__________________________________________________________________________________________________
concatenate_2 (Concatenate) (None, 32) 0 dense_6[0][0]
dense_7[0][0]
__________________________________________________________________________________________________
dense_8 (Dense) (None, 10) 330 concatenate_2[0][0]
==================================================================================================
Total params: 2,762
Trainable params: 2,762
Non-trainable params: 0
__________________________________________________________________________________________________
Train for 10 steps
Epoch 1/2
10/10 [==============================] - 0s 12ms/step - loss: 4735.8942
Epoch 2/2
10/10 [==============================] - 0s 1ms/step - loss: 2.7339
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.