我的 model.fit 和 model.evaluate 工作不正常，我收到一個錯誤

Question

train_x = train['text']
valid_x = valid["text"]
train_y = train["label"]
valid_y = valid["label"]
train_y = train_y.values.reshape(-1,1)

vectorizer = TfidfVectorizer()
vectorizer.fit_transform(train_x)

x_train_count = vectorizer.fit_transform(train_x)

x_valid_count = vectorizer.fit_transform(valid_x)

x_test_count  = vectorizer.fit_transform(test["text"])

model = tf.keras.Sequential() 

model.add(Dense(50,input_dim=x_train_count.shape[1], kernel_initializer="uniform", activation="relu")) 
model.add(Dense(1, kernel_initializer="uniform", activation="sigmoid")) 
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
# # Fit the model
history = model.fit(x_train_count, train_y, validation_data=(x_valid_count,valid_y), epochs=3, batch_size=128)

loss, acc = model.evaluate(x_test_count, test["label"], verbose=0)

錯誤

--------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
<ipython-input-1-be49478b4c50> in <module>
         71 model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
         72 # # Fit the model
    ---> 73 history = model.fit(x_train_count, train_y, validation_data=(x_valid_count,valid_y), epochs=3, batch_size=128)
         74 
         75 loss, acc = model.evaluate(x_test_count, test["label"], verbose=0)

~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
   1048          training_utils.RespectCompiledTrainableState(self):
   1049       # Creates a `tf.data.Dataset` and handles batch and epoch iteration.
-> 1050       data_handler = data_adapter.DataHandler(
   1051           x=x,
   1052           y=y,

~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in __init__(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model, steps_per_execution)
   1098 
   1099     adapter_cls = select_data_adapter(x, y)
-> 1100     self._adapter = adapter_cls(
   1101         x,
   1102         y,

~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in __init__(self, x, y, sample_weights, sample_weight_modes, batch_size, steps, shuffle, **kwargs)
    564     inputs = pack_x_y_sample_weight(x, y, sample_weights)
    565 
--> 566     dataset = dataset_ops.DatasetV2.from_tensor_slices(inputs)
    567     num_samples = int(nest.flatten(x)[0].shape[0])
    568     if shuffle:

~\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py in from_tensor_slices(tensors)
    689       Dataset: A `Dataset`.
    690     """
--> 691     return TensorSliceDataset(tensors)
    692 
    693   class _GeneratorState(object):

~\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py in __init__(self, element)
   3155     element = structure.normalize_element(element)
   3156     batched_spec = structure.type_spec_from_value(element)
-> 3157     self._tensors = structure.to_batched_tensor_list(batched_spec, element)
   3158     self._structure = nest.map_structure(
   3159         lambda component_spec: component_spec._unbatch(), batched_spec)  # pylint: disable=protected-access

~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in to_batched_tensor_list(element_spec, element)
    362   # pylint: disable=protected-access
    363   # pylint: disable=g-long-lambda
--> 364   return _to_tensor_list_helper(
    365       lambda state, spec, component: state + spec._to_batched_tensor_list(
    366           component), element_spec, element)

~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in _to_tensor_list_helper(encode_fn, element_spec, element)
    337     return encode_fn(state, spec, component)
    338 
--> 339   return functools.reduce(
    340       reduce_fn, zip(nest.flatten(element_spec), nest.flatten(element)), [])
    341 

~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in reduce_fn(state, value)
    335   def reduce_fn(state, value):
    336     spec, component = value
--> 337     return encode_fn(state, spec, component)
    338 
    339   return functools.reduce(

~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in <lambda>(state, spec, component)
    363   # pylint: disable=g-long-lambda
    364   return _to_tensor_list_helper(
--> 365       lambda state, spec, component: state + spec._to_batched_tensor_list(
    366           component), element_spec, element)
    367 

~\anaconda3\lib\site-packages\tensorflow\python\framework\sparse_tensor.py in _to_batched_tensor_list(self, value)
    367       raise ValueError(
    368           "Unbatching a sparse tensor is only supported for rank >= 1")
--> 369     return [gen_sparse_ops.serialize_many_sparse(
    370         value.indices, value.values, value.dense_shape,
    371         out_type=dtypes.variant)]

~\anaconda3\lib\site-packages\tensorflow\python\ops\gen_sparse_ops.py in serialize_many_sparse(sparse_indices, sparse_values, sparse_shape, out_type, name)
    493       return _result
    494     except _core._NotOkStatusException as e:
--> 495       _ops.raise_from_not_ok_status(e, name)
    496     except _core._FallbackException:
    497       pass

~\anaconda3\lib\site-packages\tensorflow\python\framework\ops.py in raise_from_not_ok_status(e, name)
   6860   message = e.message + (" name: " + name if name is not None else "")
   6861   # pylint: disable=protected-access
-> 6862   six.raise_from(core._status_to_exception(e.code, message), None)
   6863   # pylint: enable=protected-access
   6864 

~\anaconda3\lib\site-packages\six.py in raise_from(value, from_value)

InvalidArgumentError: indices[1] = [0,40295] 出現故障。 許多稀疏操作需要排序索引。 使用tf.sparse.reorder創建一個正確排序的副本。

Answer 1

TfIdfVectorizer 的fit_transform()和fit()方法返回的object是壓縮的稀疏行格式矩陣。 因此，為了將它們轉換為密集矩陣，請使用.toarray() function。

此外，您應該僅在 train_set 上安裝 TfIdfVectorizer，然后使用它來轉換您的驗證和測試集，而無需每次都重新安裝它，以避免使用來自測試集的數據，因為它可能會引入一些數據泄漏並導致過於樂觀的性能. 此外，由於測試和驗證集通常很小，僅在它們上擬合 TfIdf 會導致矢量化效果不佳。 最后，對於訓練、驗證和測試集中的相同單詞，Idf 因子會有所不同，這通常是不希望的。

出於上述原因，我建議僅在訓練集上擬合 TfIdfVectorizer ，然后使用擬合的矢量化器來轉換驗證集和測試集。

這里有一個例子：

 vectorizer = TfidfVectorizer()
 x_train_count = vectorizer.fit_transform(train_x).toarray()
 x_valid_count = vectorizer.transform(valid_x).toarray()
 x_test_count  = vectorizer.transform(test["text"]).toarray()

如果您的數據不適合 memory，您可以考慮一次迭代地轉換為密集矩陣一次。 這里是 function 從稀疏 x_train 矩陣創建批處理生成器：

def sparse_matrix_batch_generator(X, y, batch_size=32):
    num_samples = np.shape(y)[0]
    shuffled_index = np.arange(num_samples)
    np.random.shuffle(shuffled_index)
    X =  X[shuffled_index, :]
    y =  y[shuffled_index]
    for n in range(num_samples//batch_size):
        batch_index = shuffled_index[n*batch_size:(n+1)*batch_size]
        X_batch = X[batch_index, :].todense()
        y_batch = y[batch_index]
        yield (X_batch, y_batch)