使用CNN模型進行文本分類的預測腳本中的錯誤

Question

我嘗試為該教程編寫腳本的預測部分： https : //mxnet.incubator.apache.org/tutorials/nlp/cnn.html

import mxnet as mx

from collections import Counter
import os
import re
import threading
import sys
import itertools
import numpy as np

from collections import namedtuple

SENTENCES_DIR = 'C:/code/mxnet/sentences'
CURRENT_DIR = 'C:/code/mxnet'

def clean_str(string):
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def load_data_sentences(filename):
    sentences_file = open( filename, "r")
    # Tokenize
    x_text = [line.decode('Latin1').strip() for line in sentences_file.readlines()] 
    x_text = [clean_str(sent).split(" ") for sent in x_text]
    return x_text


def pad_sentences(sentences, padding_word=""):"
    sequence_length = max(len(x) for x in sentences)
    padded_sentences = []
    for i in range(len(sentences)):
        sentence = sentences[i]
        num_padding = sequence_length - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences


def build_vocab(sentences):
    word_counts = Counter(itertools.chain(*sentences))
    vocabulary_inv = [x[0] for x in word_counts.most_common()]
    vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}
    return vocabulary, vocabulary_inv

def build_input_data(sentences, vocabulary):
    x = np.array([
            [vocabulary[word] for word in sentence]
            for sentence in sentences])
    return x

def predict(mod, sen):
    mod.forward(Batch(data=[mx.nd.array(sen)]))
    prob = mod.get_outputs()[0].asnumpy()
    prob = np.squeeze(prob)
    a = np.argsort(prob)[::-1]    
    for i in a[0:5]:
        print('probability=%f' %(prob[i]))   


sentences = load_data_sentences( os.path.join( SENTENCES_DIR, 'test-pos-1.txt') )
sentences_padded = pad_sentences(sentences)
vocabulary, vocabulary_inv = build_vocab(sentences_padded)
x = build_input_data(sentences_padded, vocabulary)


Batch = namedtuple('Batch', ['data'])

sym, arg_params, aux_params = mx.model.load_checkpoint( os.path.join( CURRENT_DIR, 'cnn'), 19)
mod = mx.mod.Module(symbol=sym, context=mx.cpu(), label_names = None)
mod.bind(for_training=False, data_shapes=[('data', (50,56))], label_shapes=mod._label_shapes)
mod.set_params(arg_params, aux_params, allow_missing=True)

predict(mod, x)

但是我得到了錯誤：

infer_shape錯誤。 參數：數據：（50、26L）回溯（最近一次調用為最新）：文件“ C：\\ code \\ mxnet \\ test2.py”，第152行，位於predict（mod，x）文件“ C：\\ code \\ mxnet \\ test2.py“，預測mod.forward（Batch（data = [mx.nd.array（sen）]））中的第123行...

MXNetError：運算符重整0中的錯誤：[16:20:21] c：\\ projects \\ mxnet-distro-win \\ mxnet-build \\ src \\ operator \\ tensor./matrix_op-inl.h：187：檢查失敗：oshape.Size （）== dshape.Size（）（840000與390000）目標形狀大小與源形狀不同。 目標：[50,1,56,300]來源：[50,26,300]

源是具有50個字符串字符串的文本文件

不幸的是，我沒有在Internet上找到任何幫助。 請看一下。 操作系統：Windows10。Python 2.7謝謝。

Answer 1

我相信您遇到的錯誤是因為輸入句子的填充與模型預期的不同。 pad_sentences的工作方式是將句子填充到傳入的最長句子的長度，因此，如果您使用其他數據集，則幾乎可以肯定會獲得與模型填充（56）不同的填充。 在這種情況下，您看起來填充為26（來自錯誤消息“源：[50、26、300]”）。

通過如下修改pad_sentence並以sequence_length = 56使其運行以匹配模型，可以使您的代碼成功運行。

def pad_sentences(sentences, sequence_length, padding_word=""):
    padded_sentences = []
    for i in range(len(sentences)):
        sentence = sentences[i]
        num_padding = sequence_length - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences

注意，當您成功運行時，會遇到錯誤，因為prob [i]不是浮點數。

def predict(mod, sen):
    mod.forward(Batch(data=[mx.nd.array(sen)]))
    prob = mod.get_outputs()[0].asnumpy()
    prob = np.squeeze(prob)
    a = np.argsort(prob)[::-1]    
    for i in a[0:5]:
        print('probability=%f' %(prob[i]))   << prob is a numpy.ndarray, not a float.

Vishaal