Sentiment analysis on reviews using NLTK in Python

Question

I have a csv data file containing column 'notes' with satisfaction answers in Hebrew.

I would like to use Sentiment analysis in order to assign a score for each word or bigrm in the data and receive positive/negative probability using logistic regression.

My code so far:

PYTHONIOENCODING="UTF-8"  
df= pd.read_csv('keep.csv', encoding='utf-8' , usecols=['notes'])

txt = df.notes.str.lower().str.replace(r'\|', ' ').str.cat(sep=' ')
words = nltk.tokenize.word_tokenize(txt)
tokens=[word.lower() for word in words if word.isalpha()]
bigrm = list(nltk.bigrams(tokens))

word_index = {}
current_index = 0
    for token in tokens:
    if token not in word_index:
        word_index[token] = current_index
        current_index += 1

def tokens_to_vector(tokens, label):
    x = np.zeros(len(word_index) + 1) 
    for t in tokens:
        i = word_index[t]
        x[i] += 1
    x = x / x.sum() 
    x[-1] = label
    return x

N= len(word_index)
data = np.zeros((N, len(word_index) + 1))
i = 0
for token in tokens:
xy = tokens_to_vector(tokens, 1)
data[i,:] = xy
i += 1

This loop isn't working. How can I generate the data and then receive positive/negative probabilities for each bigrm?

Answer 1

Is your code snippet correct? You need indent in all for loops.

df= pd.read_csv('keep.csv', encoding='utf-8' , usecols=['notes'])

txt = df.notes.str.lower().str.replace(r'\|', ' ').str.cat(sep=' ')
words = nltk.tokenize.word_tokenize(txt)
tokens=[word.lower() for word in words if word.isalpha()]
bigrm = list(nltk.bigrams(tokens))

word_index = {}
current_index = 0
    for token in tokens:
        if token not in word_index:
            word_index[token] = current_index
            current_index += 1

def tokens_to_vector(tokens, label):
    x = np.zeros(len(word_index) + 1) 
    for t in tokens:
        i = word_index[t]
        x[i] += 1
    x = x / x.sum() 
    x[-1] = label
    return x

N= len(word_index)
data = np.zeros((N, len(word_index) + 1))
i = 0
for token in tokens:
    xy = tokens_to_vector(tokens, 1)
    data[i,:] = xy
    i += 1```