python中计算词频和排序列表的高效快速方法

Question

I have analyzed text data and now I want to count keywords that meet specific conditions (dates, category, etc), from the result of the analysis. The result of the analysis is over 50 thousand each, and I have 1500 conditions. Is there an efficient/fast way to extract keywords meeting the condition?

Below is the code I wrote and it is very time-consuming so I need some efficient way.

from collections import defaultdict
from typing import DefaultDict

# function for counting keywords
def count_words(top_rel: DefaultDict, top_pos: DefaultDict, top_neg: DefaultDict, data: pd.DataFrame):
    if isinstance(data.loc[:, "3"].values[0], str):
        for i, item in data.loc[:, "0":"3"].iterrows():
            for pos_word in ast.literal_eval(item["1"]):
                top_pos[pos_word] += 1
            for neg_word in ast.literal_eval(item["2"]):
                top_neg[neg_word] += 1
            for rel_word in ast.literal_eval(item["3"]):
                top_rel[rel_word] += 1
    else:
        for i, item in data.loc[:, "0":"3"].iterrows():

            for pos_word in item["1"]:
                top_pos[pos_word] += 1
            for neg_word in item["2"]:
                top_neg[neg_word] += 1
            for rel_word in item["3"]:
                top_rel[rel_word] += 1
    return top_rel, top_pos, top_neg

# Create conditions 
cat_ids = [subcats['id'] for subcats in cp.cat_config['cat'].values()] # cat ids in the category table
index = pd.MultiIndex.from_product([cat_ids, data.code.unique(), [start_date.strftime("%Y%m%d")], 
    data.target.unique(), [datetime.datetime.strptime(str(data._dates.unique()[0]), 
    "%Y%m%d").date().isocalendar()[1]]], names=["category_code", "region_code", "start_date",  
    "target", "year_week"]) # Cartesian product
top_word_id = pd.DataFrame(index=index).reset_index()

# Create defaultdict for each condition
top_word_id.loc[:, 'weekly_associated_top_word'] = [defaultdict(int) for _ in range(top_word_id.shape[0])]
top_word_id.loc[:, 'weekly_positive_top_word'] = [defaultdict(int) for _ in range(top_word_id.shape[0])]
top_word_id.loc[:, 'weekly_negative_top_word'] = [defaultdict(int) for _ in range(top_word_id.shape[0])]

# for specific periods,
while dates_queue:
    date = dates_queue.popleft()
    date_str = date.strftime("%Y%m%d.tsv")
    data = pd.read_csv(PurePath("../results", date_str), sep='\t', engine='python', encoding='utf-8')

    for i, item in top_word_id.iterrows(): # for each condition
        # find data matched to the condition
        id = item.loc["category_code"]
        target = item.loc['target']
        code = item.loc['region_code']
        
        category_data = data[data.loc[:, id] == 1]

        if category_data.shape[0] == 0:
            continue

        temp = category_data[(category_data.loc[:, 'target'] == target) & (category_data.loc[:, 'code'] == code)]
        
        if temp.shape[0] == 0:
            continue
            
        top_pos, top_neg, top_rel = count_words(top_word_id.iloc[i, 6], top_word_id.iloc[i, 7], top_word_id.iloc[i, 8], data)
        top_word_id.at[i, "weekly_associated_top_word"] = rel
        top_word_id.at[i, "weekly_positive_top_word"] = pos
        top_word_id.at[i, "weekly_negative_top_word"] = neg

EDIT

I really want to show you a sample, but it is too large and Korean language, I don't think you can get it. Instead, I illustrated the pseudo-code of the logic.

• Input
  • data ( pd.DataFrame ): The input is collection of documents in a day. It has columns named target , category and code . Also, the data contains columns named 0, 1, 2, 3 . Each element is a list of words. (eg data.loc[0, "0"] = ['a', 'b', 'c'] , data.loc[0, "1"] = ['hello', 'world', '.'] )
  • top_word_id ( pd.DataFrame ): Each row of the DataFrame represents each condition.
• Algorithm: What I want is to find the row of the data where the row meets some specific conditions (ie target , category and code ). A condition is each row of tow_word_id as I mentioned before.
• Output: Let's say I want to find the data which meets the condition and the condition is j-th row of top_word_id . The number of the data where meets the condition are 2 which are i1 and i2 of the data. So I want to aggregate the word frequency of i1 and i2 of the data. The result of the aggregation must be kept because I want to aggregate the word frequencies of the documents of today and tomorrow.

Answer 1

In the simple use case, given an iterable, you can use collections.Counter object, https://docs.python.org/3/library/collections.html#collections.Counter eg

>>> from collections import Counter
>>> mylist = [1,2,3,3,2,1]
>>> Counter(mylist)
Counter({1: 2, 2: 2, 3: 2})

Given a string:

>>> text = "This is a sentence with repeated words words words in the sentence"
>>> tokenized_text = text.split()
>>> Counter(tokenized_text)
Counter({'This': 1,
         'is': 1,
         'a': 1,
         'sentence': 2,
         'with': 1,
         'repeated': 1,
         'words': 3,
         'in': 1,
         'the': 1})

To update a counter:

>> counter = Counter()
>>> counter.update(tokenized_text_1) # assuming tokenized text is an iterable of strings.
>>> counter.update(tokenized_text_2)

Answer 2

I achieved efficient and fast logic by utilizing collections.Counter , Cython and multiprocessing.Pool . I replace the counting part with Counter and utilizing Cython and multiprocessing.Pool for efficiency.

The below is the entire code:

from collections import defaultdict, Counter
from typing import DefaultDict

def count_words(top_pos: DefaultDict, top_neg: DefaultDict, top_rel: DefaultDict, data: pd.DataFrame):
    print(data)
    if isinstance(data.loc[:, "3"].values[0], str):
        data_pos = data.loc[:, "1"].apply(lambda x: Counter(ast.literal_eval(x)))
        data_neg = data.loc[:, "2"].apply(lambda x: Counter(ast.literal_eval(x)))
        data_rel = data.loc[:, "3"].apply(lambda x: Counter(ast.literal_eval(x)))
        
        print(data_pos)
        print(data_neg)
        print(data_rel)
        for item in data_pos:
            for k, v in item.items():
                top_pos[k] += v
        for item in data_neg:
            for k, v in item.items():
                top_neg[k] += v
        for item in data_rel:
            for k, v in item.items():
                top_rel[k] += v
                
    elif isinstance(data.loc[:, "3"].values[0], list):
        
        print(data_pos)
        print(data_neg)
        print(data_rel)
        data_pos = data.loc[:, "1"].apply(lambda x: Counter(x))
        data_neg = data.loc[:, "2"].apply(lambda x: Counter(x))
        data_rel = data.loc[:, "3"].apply(lambda x: Counter(x))
        for item in data_pos:
            for k, v in item.items():
                top_pos[k] += v
        for item in data_neg.items():
            for k, v in item.items():
                top_neg[k] += v
        for item in data_rel.items():
            for k, v in item.items():
                top_rel[k] += v
    else:
        raise ValueError("The type must be either list or str")
            
    return top_pos, top_neg, top_rel 

def test(data, top_word_id):
    for i, item in top_word_id.iterrows():
        id = item.loc["category_code"]
        target = item.loc['target']
        code = item.loc['region_code']

        category_data = data[data.loc[:, id] == 1]

        if category_data.shape[0] == 0:
            continue

        temp = category_data[(category_data.loc[:, 'target'] == target) & (category_data.loc[:, 'code'] == code)]

        if temp.shape[0] == 0:
            continue

        top_pos, top_neg, top_rel = count_words(top_word_id.loc[i, "weekly_positive_top_word"], top_word_id.loc[i, "weekly_negative_top_word"], top_word_id.loc[i, "weekly_associated_top_word"], data)
        top_word_id.at[i, "weekly_associated_top_word"] = top_rel
        top_word_id.at[i, "weekly_positive_top_word"] = top_pos
        top_word_id.at[i, "weekly_negative_top_word"] = top_neg

    return top_word_id

from multiprocessing import Pool, cpu_count
from contextlib import contextmanager
import numpy as np

@contextmanager
def poolcontext(*args, **kwargs):
    try:
        pool = Pool(*args, **kwargs)
        yield pool
    finally:
        pool.terminate()

def parallelize_aggregation(data, top_word_id, func):
    num_cores = cpu_count()
    df_split = np.array_split(top_word_id, num_cores, axis=0)
    
    with poolcontext(processes=num_cores) as pool:
        results = pool.starmap(test, zip([data for _ in range(num_cores)], df_split))
    return results

parallelize_aggregation(data, top_word_id, aggregate.test)

The below table illustrates times of the codes:

Code	Times
Cython (the code in the question)	4749s
Cython + Counter	3066s
Cython + Counter + multiprocessing.Pool	10s