python：在不同文件中搜索单词

Question

I'm trying to write a script that collects the names of all files in a directory and then search each of them for given words. Each time the word is found, the name of the file and the full line that contains that word should be printed. Additionally, in a new file I want to print the number of times the word has been found.

This is what I have so far:

import os

print(os.listdir('./texts'), '\n\n\n')

suchwort ={"computational":0,"linguistics":0,"processing":0,"chunking":0,"coreference":0,"html":0,"machine":0}
hitlist = './hits.txt'


with open(hitlist, 'a+') as hits:
   for elem in os.listdir('./texts'):
      with open(os.path.join("./texts",elem)) as fh:
         for line in fh:
            words = line.split(' ')
            print(elem, " : ",line)
            for n in words:
               if n in suchwort:
                  if n in suchwort.keys():
                     suchwort[n]+=1
                  else:
                     suchwort[n]=1
   for k in suchwort:
      print(k,":",suchwort[k],file=hits)

The result in the new file (hits.txt) is:

chunking : 0
machine : 9
html : 0
processing : 4
linguistics : 12
coreference : 1
computational : 12

However the values seem to be wrong, because the word "html" is in one of the files.

Answer 1

import itertools
import multiprocessing as mp
import glob

def filesearcher(qIn, qOut):
    for fpath in iter(qIn.get, None):
        keywords = {"computational":{'count':0, 'lines':[]},
                    "linguistics":{'count':0, 'lines':[]},
                    "processing":{'count':0, 'lines':[]},
                    "chunking":{'count':0, 'lines':[]},
                    "coreference":{'count':0, 'lines':[]},
                    "html":{'count':0, 'lines':[]},
                    "machine":{'count':0, 'lines':[]}}

        with open(fpath) as infile:
           for line in file:
               for word in line.split():
                   word = word.lower()
                   if word not in keywords: continue
                   keywords[word]['count'] += 1
                   keywords[word]['lines'].append(line)
        qOut.put(fpath, keywords)
    qOut.put(None)


def main():
    numProcs = 4  # fiddle to taste
    qIn, qOut = [mp.Queue() for _ in range(2)]
    procs = [mp.Process(target=filesearcher, args=(qIn, qOut)) for _ in range(numProcs)]
    for p in procs: p.start()
    for fpath in glob.glob('./texts/*'): qIn.put(fpath)
    for _ in procs: qIn.put(None)

    done = 0
    while done < numProcs:
        d = qOut.get()
        if d is None:
            done += 1
            continue
        fpath, stats = d
        print("showing results for", fpath)
        for word, d in stats.items():
            print(word, ":", d['count'])
            for line in d['lines']:
                print('\t', line)

    for p in procs: p.terminate()

Answer 2

The problem is caused by the way the file is iterated line-by-line. In the snippet below, each "line" will have the trailing newline. So doing a split leave the last line with the trailing newline character.

  with open(os.path.join("./texts",elem)) as fh:
     for line in fh:
        words = line.split(' ')

If you print the "repr" of the words,

print repr(words)

You'd see that the last word contains the trailing newline,

['other', 'word\n']

instead of the expected:

['other', 'word']

To solve this problem you could use "strip", before processing each line:

line = line.strip() 

to remove trailing and leading whitespaces of the string.