并发进程有时会挂起

Question

我正在尝试使用Retinaface提取人脸。 为了加快这个过程，我尝试同时运行 function。 并发任务 function 基本上需要所有目录的列表，将它们分解为较小的目录，然后运行，然后在每个列表上运行相应的 function。

问题是有时我可以使用更多数量的进程，有时我只能使用 2 个。更麻烦的是，有时这些功能只是挂起，而其他时候它们可以工作。 就像说我从 3 个进程开始，一段时间后，终端什么都没有打印，只是一个空白屏幕，当我 go 到活动监视器时，我的 ram 使用量在 4gb 左右恢复正常（当我开始时，使用量飙升至 15ish gb，我的总内存是 16gb）。 我认为我的流程卡住了，我只是不知道为什么。

def concurent_tasks(targets, num_processes, dir_list, *args):
    processes = {}
    list2D = split_dir_to_2Dlists(dir_list, num_processes)

    for i in range(num_processes):
        for j in targets:
            new_args = [list2D[i]] + list(args)
            processes["process_" + str(j) +str(i)] = multiprocessing.Process(target=j, args = new_args)
    for key in processes:
        processes[key].start()
    
    for key in processes:
        processes[key].join()

    for key in processes:
        processes[key].terminate()

def extract_faces(org_dir, new_dir_path):
    #org_dir is list of all folders in a directory
    for dir in org_dir: # gets a subfolder in the current directory
        counter = 0
        folder_name = dir.split('/')[-1] # name of the sub folder

        start =time.time()
        for entry in os.scandir(dir): # gets an image from the subfolder

            #print("directory: ", dir)
            print("entry: ", entry.path)
            #print("folder name: ", folder_name)

            try:
                #Extracting the face
                faces = RetinaFace.extract_faces(entry.path, align=True)
                if faces is not None: #returns 1 if a face is detected:

                    new_folder_path = os.path.join(new_dir_path,folder_name)
                    if not os.path.exists(new_folder_path): #creating a sub folder in the new directory if it does not exist
                        os.makedirs(new_folder_path)

                    #_, file_extention = os.path.splitext(entry.path)
                    for i in range(len(faces)):
                        new_file_name = folder_name + str(counter) +'_'+ str(i) + '.jpg'
                        new_file_path = os.path.join(new_folder_path,new_file_name)
                        cv2.imwrite(new_file_path, cv2.cvtColor(faces[i], cv2.COLOR_RGB2BGR))
                        
                        gc.collect()
                        gc.collect()
                        gc.collect()
                        gc.collect()
                        
                    counter+=1
            except:
                pass
        
        stop = time.time()
        gc.collect()
        gc.collect()
        gc.collect()
        gc.collect()
        print("Time for subfolder ", entry, " : ", stop-start)