帶計數器的多處理嵌套 for 循環

Question

我正在尋找一種簡單的解決方案，它可以幫助我利用 PC 的全部功能來處理我的數據。 我認為，將任務划分到不同的核心將有助於減少處理時間，但我不知道該怎么做，我已經在 stackoverflow 上搜索了類似的問題，但沒有任何解決方案可以解決我的問題。 我正在處理大約長度的數據：3000 並且由於我使用嵌套 for 循環來查找列表中相似（在 +- 0.5 范圍內）元素的數量，它將運行 3000x3000 次，大約需要 2 分鍾，我想減少花費的時間。

repeat= []
values = []
for i in completeList:
    count = 0
    for j in completeList:
        if isfloat(i) and isfloat(j):
            if float(i)-0.5 <= float(j) <= float(i)+0.5:
                count = count + 1
    repeat.append(count)
    values.append(i)

任何幫助，將不勝感激。

關於，馬尼什

Answer 1

由於您仍然沒有發布isfloat的實際代碼或顯示completeList的元素是什么樣的，所以我能做的最好的就是推測它們可能是什么。 它有所不同，因為正如我所提到的，執行isfloat和float以轉換completeList的元素所需的 CPU 越多，使用多處理獲得的收益就越大。

對於CASE 1 ，我假設completeList由字符串組成，並且isfloat需要使用正則表達式來確定字符串是否與我們預期的浮點格式匹配，因此float需要從字符串轉換。 這將是我想象中 CPU 最密集的情況。 對於CASE 2 ， completeList由浮點數組成， isfloat只返回True ，而float不必進行任何實際轉換。

我的桌面有 8 個核心處理器：

情況1

import multiprocessing as mp
import time
import random
import re
from functools import partial

def isfloat(s):
    return not re.fullmatch(r'\d*\.\d+', s) is None

def single_process(complete_list):
    #repeat = []
    values = []
    for idx_i, v_i in enumerate(complete_list):
        count = 0
        for idx_j, v_j in enumerate(complete_list):
            if idx_i == idx_j:
                continue # don't compare an element with itself
            if isfloat(v_i) and isfloat(v_j):
                f_i = float(v_i)
                if f_i-0.5 <= float(v_j) <= f_i+0.5:
                    count = count + 1
        # repeat will end up being a copy of complete_list
        # why are we doing this?
        #repeat.append(v_i)
        values.append(count) # these are actually counts
    return values


def multi_worker(complete_list, index_range):
    values = []
    for idx_i in index_range:
        v_i = complete_list[idx_i]
        count = 0
        for idx_j, v_j in enumerate(complete_list):
            if idx_i == idx_j:
                continue # don't compare an element with itself
            if isfloat(v_i) and isfloat(v_j):
                f_i = float(v_i)
                if f_i-0.5 <= float(v_j) <= f_i+0.5:
                    count = count + 1
        values.append(count) # these are actually counts
    return values


def multi_process(complete_list):

    def split(a, n):
        k, m = divmod(len(a), n)
        return (a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))

    n = len(complete_list)
    POOL_SIZE = mp.cpu_count()
    range_splits = split(range(0, n), POOL_SIZE)
    pool = mp.Pool(POOL_SIZE)
    value_lists = pool.map(partial(multi_worker, complete_list), range_splits)
    values = []
    # join results together:
    for value_list in value_lists:
        values.extend(value_list)
    return values

def main():
    # generate 3000 random numbers:
    random.seed(0)
    complete_list = [str(random.uniform(1.0, 3.0)) for _ in range(3000)]
    t = time.time()
    values = single_process(complete_list)
    print(time.time() - t, values[0:10], values[-10:-1])

    t = time.time()
    values = multi_process(complete_list)
    print(time.time() - t, values[0:10], values[-10:-1])


# required for Windows:
if __name__ == '__main__':
    main()

印刷：

27.7540442943573 [1236, 1491, 1464, 1477, 1494, 1472, 1410, 1450, 1502, 1537] [1485, 1513, 1513, 1501, 1283, 1538, 804, 1459, 1457]
7.187546253204346 [1236, 1491, 1464, 1477, 1494, 1472, 1410, 1450, 1502, 1537] [1485, 1513, 1513, 1501, 1283, 1538, 804, 1459, 1457]

案例二

import multiprocessing as mp
import time
import random
from functools import partial

def isfloat(s):
    return True

def single_process(complete_list):
    values = []
    for idx_i, v_i in enumerate(complete_list):
        count = 0
        for idx_j, v_j in enumerate(complete_list):
            if idx_i == idx_j:
                continue # don't compare an element with itself
            if isfloat(v_i) and isfloat(v_j):
                f_i = float(v_i)
                if f_i-0.5 <= float(v_j) <= f_i+0.5:
                    count = count + 1
        values.append(count) # these are actually counts
    return values


def multi_worker(complete_list, index_range):
    values = []
    for idx_i in index_range:
        v_i = complete_list[idx_i]
        count = 0
        for idx_j, v_j in enumerate(complete_list):
            if idx_i == idx_j:
                continue # don't compare an element with itself
            if isfloat(v_i) and isfloat(v_j):
                f_i = float(v_i)
                if f_i-0.5 <= float(v_j) <= f_i+0.5:
                    count = count + 1
        values.append(count) # these are actually counts
    return values


def multi_process(complete_list):

    def split(a, n):
        k, m = divmod(len(a), n)
        return (a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))

    n = len(complete_list)
    POOL_SIZE = mp.cpu_count()
    range_splits = split(range(0, n), POOL_SIZE)
    pool = mp.Pool(POOL_SIZE)
    value_lists = pool.map(partial(multi_worker, complete_list), range_splits)
    values = []
    # join results together:
    for value_list in value_lists:
        values.extend(value_list)
    return values

def main():
    # generate 3000 random numbers:
    random.seed(0)
    complete_list = [random.uniform(1.0, 3.0) for _ in range(3000)]
    t = time.time()
    values = single_process(complete_list)
    print(time.time() - t, values[0:10], values[-10:-1])

    t = time.time()
    values = multi_process(complete_list)
    print(time.time() - t, values[0:10], values[-10:-1])


# required for Windows:
if __name__ == '__main__':
    main()

印刷：

4.181002378463745 [1236, 1491, 1464, 1477, 1494, 1472, 1410, 1450, 1502, 1537] [1485, 1513, 1513, 1501, 1283, 1538, 804, 1459, 1457]
1.325998067855835 [1236, 1491, 1464, 1477, 1494, 1472, 1410, 1450, 1502, 1537] [1485, 1513, 1513, 1501, 1283, 1538, 804, 1459, 1457]

結果

案例 1 的加速比為 3.86，案例 2 的加速比僅為 3.14。