This is a sort of "best practices" question. I'm doing my first multi-threaded code and I'm wondering if I am measuring progress properly. Here's my code which does 16 file-copy threads at once, and I'm trying to create a progress bar. This code works but I want to know if this is the "right" way of doing it (for example what if multiple threads are writing to "copyCount" at once?):
import Queue, threading, os
import shutil
fileQueue = Queue.Queue()
destPath = 'destination/folder/here'
class ThreadedCopy:
totalFiles = 0
copyCount = 0
def __init__(self):
with open("filelist.txt", "r") as txt:
fileList = txt.read().splitlines()
if not os.path.exists(destPath):
os.mkdir(destPath)
self.totalFiles = len(fileList)
print str(self.totalFiles) + " files to copy."
self.threadWorkerCopy(fileList)
def CopyWorker(self):
while True:
fileName = fileQueue.get()
shutil.copy(fileName, destPath)
fileQueue.task_done()
self.copyCount += 1
percent = (self.copyCount*100)/self.totalFiles
print str(percent) + " percent copied."
def threadWorkerCopy(self, fileNameList):
for i in range(16):
t = threading.Thread(target=self.CopyWorker)
t.daemon = True
t.start()
for fileName in fileNameList:
fileQueue.put(fileName)
fileQueue.join()
ThreadedCopy()
For anyone else interested yes I was definitely doing this wrong. I found this fantastic article which explains in pretty simple terms how to go about wrangling multiple threads using locks: http://effbot.org/zone/thread-synchronization.htm
And for an updated code sample:
import Queue, threading, os
import shutil
fileQueue = Queue.Queue()
destPath = 'destination/folder/here'
class ThreadedCopy:
totalFiles = 0
copyCount = 0
lock = threading.Lock()
def __init__(self):
with open("filelist.txt", "r") as txt:
fileList = txt.read().splitlines()
if not os.path.exists(destPath):
os.mkdir(destPath)
self.totalFiles = len(fileList)
print str(self.totalFiles) + " files to copy."
self.threadWorkerCopy(fileList)
def CopyWorker(self):
while True:
fileName = fileQueue.get()
shutil.copy(fileName, destPath)
fileQueue.task_done()
with self.lock:
self.copyCount += 1
percent = (self.copyCount*100)/self.totalFiles
print str(percent) + " percent copied."
def threadWorkerCopy(self, fileNameList):
for i in range(16):
t = threading.Thread(target=self.CopyWorker)
t.daemon = True
t.start()
for fileName in fileNameList:
fileQueue.put(fileName)
fileQueue.join()
ThreadedCopy()
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.