CNN ValueError：形状 (10, 4) 和 (10, 128, 128, 4) 不兼容

Question

I am building a CNN for a multi class audio classification problem. I have extracted the spectrogram features from my audio files and I am trying to pass the 4D array to my CNN. I have also performed One-Hot Encoding for my labels. But I don't understand why exactly I get this ValueError. Kindly, Help.

This is my code:

D = [] # Dataset

for row in df.itertuples():
    file_path = os.path.join('/Users/akellaniranjan/MyWorkspace/Projects/Hobby_Projects/Whistle_Based_Automation/Folder_Approach/',row.Fold,row.File)
    y, sr = librosa.load(file_path,sr = 44100)  
    ps = librosa.feature.melspectrogram(y=y, sr=sr)
    ps = ps[:,:128]
    if ps.shape != (128, 128): continue
    D.append( (ps, row.Class) )

X_train, y_train = zip(*D)

X_train = np.array([x.reshape( (128, 128, 1) ) for x in X_train])

le = LabelEncoder()
y_train = np_utils.to_categorical(le.fit_transform(y_train))

X_train.shape  #Output - (50,128,128,1)
y_train.shape  #Output - (50,4)


#Model Creation
num_classes = 4 #Number of Classes

model = Sequential()

model.add(Conv2D(256,(5,5),input_shape=(128,128,1)))
model.add(Activation('relu'))
model.add(Dropout(0.3))

model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.3))

model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.3))

model.add(Dense(num_classes))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')

Model Summary:

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 124, 124, 256)     6656      
_________________________________________________________________
activation (Activation)      (None, 124, 124, 256)     0         
_________________________________________________________________
dropout (Dropout)            (None, 124, 124, 256)     0         
_________________________________________________________________
dense (Dense)                (None, 124, 124, 512)     131584    
_________________________________________________________________
activation_1 (Activation)    (None, 124, 124, 512)     0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 124, 124, 512)     0         
_________________________________________________________________
dense_1 (Dense)              (None, 124, 124, 256)     131328    
_________________________________________________________________
activation_2 (Activation)    (None, 124, 124, 256)     0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 124, 124, 256)     0         
_________________________________________________________________
dense_2 (Dense)              (None, 124, 124, 4)       1028      
_________________________________________________________________
activation_3 (Activation)    (None, 124, 124, 4)       0         
=================================================================
Total params: 270,596
Trainable params: 270,596
Non-trainable params: 0

#Fit Model
model.fit(X_train,y_train,batch_size=10,epochs=200)

Error:

Epoch 1/200
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-63-215c349ba276> in <module>
----> 1 model.fit(X_train,y_train,batch_size=10,epochs=200)

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
   1098                 _r=1):
   1099               callbacks.on_train_batch_begin(step)
-> 1100               tmp_logs = self.train_function(iterator)
   1101               if data_handler.should_sync:
   1102                 context.async_wait()

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
    826     tracing_count = self.experimental_get_tracing_count()
    827     with trace.Trace(self._name) as tm:
--> 828       result = self._call(*args, **kwds)
    829       compiler = "xla" if self._experimental_compile else "nonXla"
    830       new_tracing_count = self.experimental_get_tracing_count()

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
    869       # This is the first call of __call__, so we have to initialize.
    870       initializers = []
--> 871       self._initialize(args, kwds, add_initializers_to=initializers)
    872     finally:
    873       # At this point we know that the initialization is complete (or less

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
    723     self._graph_deleter = FunctionDeleter(self._lifted_initializer_graph)
    724     self._concrete_stateful_fn = (
--> 725         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access
    726             *args, **kwds))
    727 

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
   2967       args, kwargs = None, None
   2968     with self._lock:
-> 2969       graph_function, _ = self._maybe_define_function(args, kwargs)
   2970     return graph_function
   2971 

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
   3359 
   3360           self._function_cache.missed.add(call_context_key)
-> 3361           graph_function = self._create_graph_function(args, kwargs)
   3362           self._function_cache.primary[cache_key] = graph_function
   3363 

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
   3194     arg_names = base_arg_names + missing_arg_names
   3195     graph_function = ConcreteFunction(
-> 3196         func_graph_module.func_graph_from_py_func(
   3197             self._name,
   3198             self._python_function,

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
    988         _, original_func = tf_decorator.unwrap(python_func)
    989 
--> 990       func_outputs = python_func(*func_args, **func_kwargs)
    991 
    992       # invariant: `func_outputs` contains only Tensors, CompositeTensors,

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
    632             xla_context.Exit()
    633         else:
--> 634           out = weak_wrapped_fn().__wrapped__(*args, **kwds)
    635         return out
    636 

~/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
    975           except Exception as e:  # pylint:disable=broad-except
    976             if hasattr(e, "ag_error_metadata"):
--> 977               raise e.ag_error_metadata.to_exception(e)
    978             else:
    979               raise

ValueError: in user code:

    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:805 train_function  *
        return step_function(self, iterator)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:795 step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:1259 run
        return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:2730 call_for_each_replica
        return self._call_for_each_replica(fn, args, kwargs)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:3417 _call_for_each_replica
        return fn(*args, **kwargs)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:788 run_step  **
        outputs = model.train_step(data)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:755 train_step
        loss = self.compiled_loss(
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/engine/compile_utils.py:203 __call__
        loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/losses.py:157 __call__
        losses = call_fn(y_true, y_pred)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/losses.py:261 call  **
        return ag_fn(y_true, y_pred, **self._fn_kwargs)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:201 wrapper
        return target(*args, **kwargs)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/losses.py:1562 categorical_crossentropy
        return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py:201 wrapper
        return target(*args, **kwargs)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/keras/backend.py:4869 categorical_crossentropy
        target.shape.assert_is_compatible_with(output.shape)
    /Users/akellaniranjan/miniforge3/envs/DL/lib/python3.8/site-packages/tensorflow/python/framework/tensor_shape.py:1134 assert_is_compatible_with
        raise ValueError("Shapes %s and %s are incompatible" % (self, other))

    ValueError: Shapes (10, 4) and (10, 128, 128, 4) are incompatible

Answer 1

Add an flatten layer before your last block:

model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Flatten()) # <-- this is new


model.add(Dense(num_classes))
model.add(Activation('softmax'))

More info here: https://www.tensorflow.org/api_docs/python/tf/keras/layers/Flatten or here: https://keras.io/api/layers/reshaping_layers/flatten/

Answer 2

I think for your case you should add a Flatten layer before the last layer in your model.

>>> model.add(Flatten())

In your case, the output is multidimensional vector, however you need only one dimensional output which corresponds to the classes.

More info: https://keras.io/api/layers/reshaping_layers/flatten/

PS: You should also think about using Maxpooling layers, they are going to help your model train faster since they reduce dimensions of the input.