测试和训练 CSV 文件 python

Question

how to simply test and train data from CSV file in python pycharm

you check data from file " https://drive.google.com/file/d/1pvcuGk2nRTsYcd-l-_yNBzvvRj2qW5rF/view " file name is " Papers data.csv "

This is simple a simple copy past code how below file replace by "Papers data.csv"

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

import os

import gensim

test_data_dir = os.path.join(gensim.__path__[0], 'test', 'test_data')
lee_train_file = os.path.join(test_data_dir, 'lee_background.cor')

lee_test_file = os.path.join(test_data_dir, 'lee.cor')


import smart_open
def read_corpus(fname, tokens_only=False):
    with smart_open.open(fname, encoding="iso-8859-1") as f:
        for i, line in enumerate(f):
            tokens = gensim.utils.simple_preprocess(line)
            if tokens_only:
                yield tokens
            else:
                yield gensim.models.doc2vec.TaggedDocument(tokens, [i])

train_corpus = list(read_corpus(lee_train_file))
test_corpus = list(read_corpus(lee_test_file, tokens_only=True))

# Let's take a look at the training corpus
print(train_corpus[:2])

# And the testing corpus looks like this:
print(test_corpus[:2])

model = gensim.models.doc2vec.Doc2Vec(vector_size=50, min_count=2, epochs=40)
model.build_vocab(train_corpus)

model.train(train_corpus, total_examples=model.corpus_count, epochs=model.epochs)
vector = model.infer_vector(['only', 'you', 'can', 'prevent', 'forest', 'fires'])
print(vector)

Answer 1

Tensorflow provides library to read CSV dataset to train and test on model. Such as tf.io.decode_csv and tf.data.experimental.CsvDataset .

Complete guideline on Loading CSV data is explained on Tensorflow documentation .