Python 在多个嵌套循环中集成多处理

Question

I am making a scrabble word generator that is incredibly inefficient due to my lack of coding skill. In this program the user enters a series of letters and the program uses brute force to find every valid scrabble word. In order to speed this process up I want to implement multiprocessing but am unable to get it to work successfully. The working non multiprocessing code is bellow

from multiprocessing import Process
usrList = input("type the letters you have     ")
usrList = list(usrList.upper())
usrList.sort()
print(usrList)    


storedList = []

def word2 (usrList):
    print('trying to find two letter words')
    for i in range(0,len(usrList)):
        for j in range(0,len(usrList)):
            if i != j:
                if str(usrList[i])+str(usrList[j]) not in storedList and str(usrList[i])+str(usrList[j])+'\n' in dicList:
                    print(str(usrList[i])+str(usrList[j]))
                    storedList.append(str(usrList[i])+str(usrList[j]))

def word3(usrList):
    print('trying to find three leter words')
    if len(usrList) > 2:
        for i in range(0,len(usrList)):
            for j in range(0,len(usrList)):
                for k in range(0,len(usrList)):
                    if i != j and i != k and j != k:
                        if  str(usrList[i])+str(usrList[j])+str(usrList[k]) not in storedList and str(usrList[i])+str(usrList[j])+str(usrList[k])+'\n' in dicList :
                            print(str(usrList[i])+str(usrList[j])+str(usrList[k]))
                            storedList.append(str(usrList[i])+str(usrList[j])+str(usrList[k]))

def word4(usrList):
    print('trying to find four letter words')
    if len(usrList) > 3:
        for i in range(0,len(usrList)):
            for j in range(0,len(usrList)):
                for k in range(0,len(usrList)):
                    for l in range(0,len(usrList)):
                        if i !=j and i != k and i!= l and j!= k and j!= l and k != l:
                            if str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l]) not in storedList and str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+'\n' in dicList: 
                                print(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l]))
                                storedList.append(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l]))


def word5(usrList):
    print('trying to find five letter words')
    if len(usrList) > 4:
        for i in range(0,len(usrList)):
            for j in range(0,len(usrList)):
                for k in range(0,len(usrList)):
                    for l in range(0,len(usrList)):
                        for m in range(0,len(usrList)):
                            if i !=j and i != k and i!= l and i != m and j!= k and j!= l and j!= m and k != l and k != m and l !=m:
                                if str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m]) not in storedList and str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+'\n' in dicList:
                                    print(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m]))
                                    storedList.append(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m]))


def word6(usrList):
    print('trying to find six letter words')
    if len(usrList) > 5:
        for i in range(0,len(usrList)):
            for j in range(0,len(usrList)):
                for k in range(0,len(usrList)):
                    for l in range(0,len(usrList)):
                        for m in range(0,len(usrList)):
                            for n in range(0,len(usrList)):
                                if i !=j and i != k and i!= l and i != m and i != n and j!= k and j!= l and j!= m and j !=n and k != l and k != m and k != n and l !=m and l != n and m!= n:
                                    if str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n]) not in storedList and str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n])+'\n' in dicList:
                                        print(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n]))
                                        storedList.append(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n]))

def word7(usrList):
    print('trying to find seven letter words')
    if len(usrList) > 6:
        for i in range(0,len(usrList)):
            for j in range(0,len(usrList)):
                for k in range(0,len(usrList)):
                    for l in range(0,len(usrList)):
                        for m in range(0,len(usrList)):
                            for n in range(0,len(usrList)):
                                for o in range(0,len(usrList)):
                                    if i !=j and i != k and i!= l and i != m and i != n and i != 0 and j!= k and j!= l and j!= m and j !=n and j != o and k != l and k != m and k != n and k!= o and l !=m and l != n and l != 0 and m!= n and m != o and n != o:
                                        if str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n])+str(usrList[o]) not in storedList and str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n])+str(usrList[o])+'\n' in dicList :
                                            print(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n])+str(usrList[o]))
                                            storedList.append(str(usrList[i])+str(usrList[j])+str(usrList[k])+str(usrList[l])+str(usrList[m])+str(usrList[n])+str(usrList[o]))        



f = 'ScrabbleDic.txt'
with open(f,'r') as file:
    dicList=[]
    for line in file:
        dicList.append(line)
    file.close()

if __name__ == '__main__':
    word7(usrList)
    word6(usrList)
    word5(usrList)
    word4(usrList)
    word3(usrList)
    word2(usrList)

Answer 1

In general, you'll often get more value out of redesigning your algorithm than you will from using multiprocessing.

Here's a shorter implementation of your code. I've hardcoded the usrList, and since I don't have access to the dictionary file you're using, I'm using the default dictionary file that comes with MacOS. Instead of writing nested loops and checking for duplicate indices, I'm using the itertools module to generate all permutations of the usrList for a given length. This won't meaningfully speed up the code, but it makes it easier to demonstrate possible changes:

import itertools

usrList = ['P', 'Y', 'T', 'H', 'O', 'N', 'S']
storedList = []
with open('/usr/share/dict/words', 'r') as dict_file:
    dicList = [word.strip().upper() for word in dict_file]


def possible_words(length):
    for letter_permutation in itertools.permutations(usrList, length):
        word = ''.join(letter_permutation)  # itertools returns a tuple, not a string
        if word in dicList:  # This requires a linear search through the list
            storedList.append(word)


for word_length in range(2, 8):  # Note that the upper bound is 7 letters, not 8
    possible_words(word_length)

This takes about 47.4 seconds to run on my Macbook. To speed it up, let's add multiprocessing like you suggest. There are a few ways to use multiprocessing, but the easiest to implement is probably creating a Pool and calling its map() function.

This syntax can look a bit weird if you aren't used to functions that take other functions as arguments. Effectively, we're creating a pool of workers, then giving that pool a function and a range of arguments to use on that function. The individual function calls are then split across the pool instead of being called sequentially:

import itertools
import multiprocessing

usrList = ['P', 'Y', 'T', 'H', 'O', 'N', 'S']
storedList = []
with open('/usr/share/dict/words', 'r') as dict_file:
    dicList = [word.strip().upper() for word in dict_file]


def possible_words(length):
    for letter_permutation in itertools.permutations(usrList, length):
        word = ''.join(letter_permutation)
        if word in dicList:
            storedList.append(word)


if __name__ == '__main__':  # multiprocessing complains if this isn't isolated
    with multiprocessing.Pool(6) as p:  # Creates 6 worker processes
        p.map(possible_words, range(2, 8))  # Each process calls possible_words() with a different input

This runs in 32.3 seconds on my Macbook. We shaved off a quarter of the time, There are probably ways to squeeze a bit more performance out of this approach. but it's also worth looking at the algorithm to see whether there are other ways to speed this up.

Right now, you're creating a list of dictionary words. When you check whether a potential word is in that list, Python has to scan through the whole list until it finds a match or reaches the end. My built-in dictionary has 235K words, so this means it has to do 235K string comparisons for every nonsense combination of letters it generates!

If you switch from using a list to a set, Python can instead look up a value in nearly-constant time by using a hash function, rather than scanning each entry one at a time. Let's try that instead of multiprocessing:

import itertools

usrList = ['P', 'Y', 'T', 'H', 'O', 'N', 'S']
storedList = []
with open('/usr/share/dict/words', 'r') as dict_file:
    dicSet = {word.strip().upper() for word in dict_file}   # By changing [] to {}, this is now a set


def possible_words(length):
    for letter_permutation in itertools.permutations(usrList, length):
        word = ''.join(letter_permutation)
        if word in dicSet:  # This now only does 1 check, not 235,000
            storedList.append(word)


for word_length in range(2, 8):
    possible_words(word_length)

This version runs in 0.005 seconds , after changing just two characters!

In summary, multiprocessing is a useful tool, but it probably shouldn't be the first thing you try. You'll usually get much better results by thinking through the data structures and algorithms you're using and where the bottleneck is likely to be.

Answer 2

The classical solution to solving puzzles like this is to not check every permutation possible, but instead to convert the sample letters and the words in the dictionary to a consistent searchable permutation - by sorting their characters!

Now instead of searching a dictionary for every permutation of 'PYTHONS', you just sort the letters to create the key 'HNOPSTY' and all valid words with the same key will be found in the map.

Using a defaultdict, it is easy to create a lookup map of all words in your dictionary. We use a defaultdict(list) instead of a dict because multiple words may sort to the same key.

from collections import defaultdict
dictionary_mapping = defaultdict(list)

# assuming dictionary is a list of all valid words, regardless of length
for word in dictionary:
    key = ''.join(sorted(word.upper()))
    dictionary_mapping[key].append(word)

search_word = "PYTHONS"
search_key = ''.join(sorted(search_word.upper()))

# get all words that are anagrams of the search word, or the empty list if none
print(dictionary_mapping.get(search_key, []))