Any suggestions on why the Lock mechanism isn't getting implemented in the below python script?
Question
I'm using multithreading for anonymously login into publicly available FTP sites and listing out the hostnames along with the respective directories. I'm using the threading.lock() method to do the same however it isn't seems to be working and prints hosts and respective directories randomly. Where am I going wrong ? Below is the code I'm running.
#!/usr/bin/python
import threading
import Queue
from ftplib import FTP
ftpSiteList = [ 'ftp.x.org', 'ftp4.FreeBSD.org', 'ftp.ncsa.uiuc.edu', 'ftp.crans.org' ]
threadsList = []
class workerClass(threading.Thread):
def __init__(self,queue,tid):
threading.Thread.__init__(self)
self.queue = queue
self.lock = threading.Lock()
self.tid = tid
print "Worker Thread %d reporting for Service !!!"%self.tid
def run(self):
self.lock.acquire()
try:
while True:
host = ''
try:
host = self.queue.get(timeout=1)
except Queue.Empty:
print "Worker thread %d exiting. "%(self.tid)
return
try:
conn = FTP(host)
conn.login()
print 'Host: ' +host
print conn.retrlines('LIST')
except:
print "Error in listing HOST "+host
raise
self.queue.task_done()
finally:
self.lock.release()
queue = Queue.Queue()
for threads in range(5):
worker = workerClass(queue,threads)
worker.setDaemon(True)
worker.start()
threadsList.append(worker)
print "Thread %d started .."%threads
for host in ftpSiteList:
queue.put(host)
queue.join()
for threads in threadsList:
threads.join()
print "Scanning completed !!!"
===========================================================================
Output is below
Worker Thread 0 reporting for Service !!!
Thread 0 started ..
Worker Thread 1 reporting for Service !!!
Thread 1 started ..
Worker Thread 2 reporting for Service !!!
Thread 2 started ..
Worker Thread 3 reporting for Service !!!
Thread 3 started ..
Worker Thread 4 reporting for Service !!!
Thread 4 started ..
Worker thread 4 exiting.
Host: ftp4.FreeBSD.org
Host: ftp.crans.org
Host: ftp.ncsa.uiuc.edu
-rw-r--r-- 1 ftp ftp 5430 Jul 19 2014 favicon.ico
-rw-r--r-- 1 ftp ftp 655 Nov 24 2014 index.html
drwxr-xr-x 3 ftp ftp 3 Jul 19 2014 pub
226 Directory send OK.
lrwxrwxrwx 1 0 0 10 Jun 14 2012 custom -> pub/custom
lrwxrwxrwx 1 9865 9865 10 May 24 2012 debian -> pub/debian
lrwxrwxrwx 1 9865 9865 20 May 27 2012 debian-backports -> pub/debian-backports
lrwxrwxrwx 1 9865 9865 22 May 24 2012 debian-multimedia -> pub/debian-multimedia/
lrwxrwxrwx 1 9865 9865 20 May 24 2012 debian-security -> pub/debian-security/
lrwxrwxrwx 1 0 0 10 Apr 14 2014 events -> pub/events
drwxrwsr-x 27 0 4 4096 Jan 27 2014 git
drwx------ 2 9865 9865 16384 Jun 08 2012 lost+found
drwxr-xr-x 14 9865 9865 4096 Nov 26 2013 pub
lrwxrwxrwx 1 9865 9865 11 May 24 2012 ubuntu -> pub/ubuntu/
lrwxrwxrwx 1 0 0 12 Jun 14 2012 videolan -> pub/videolan
226 Directory send OK.
Worker thread 1 exiting.
-rw-r--r-- 1 12873 floppy 784 Aug 22 1994 .index
-rw-r--r-- 1 80 root 663 Jan 7 1995 .message
-rw-r--r-- 1 80 root 25819 Dec 29 1994 Brochure
drwx------ 2 80 ncsa 2048 Jun 16 1998 Cyberia
drwxr-xr-x 2 12873 root 2048 Aug 22 1994 DTM
drwxrwxrwx 4 root root 2048 Dec 4 2003 Director
drwxr-xr-x 13 14453 root 2048 Aug 22 1994 Documentation
drwxr-xr-x 9 root root 2048 Feb 5 1997 Education
drwxrwxrwx 2 root root 2048 Apr 9 1996 FigLeaf
drwxr-xr-x 13 19099 root 2048 Aug 22 1994 GlobalModels
drwxr-xr-x 8 12873 root 2048 May 17 1996 Mac
drwxr-xr-x 10 12873 root 2048 Feb 15 2002 Mosaic
drwxr-xr-x 5 12873 root 2048 Jan 5 1995 PC
-rw-r--r-- 1 80 root 16557 Jan 3 1995 README
-rw-r--r-- 1 12873 root 1933 Jan 12 1995 README.FIRST
drwxr-xr-x 5 12873 root 2048 Feb 15 2002 SGI
drwxr-xr-x 4 12873 root 2048 Dec 20 1994 Telnet
drwxr-xr-x 7 12873 root 2048 Aug 24 1994 Unix
drwxrwxrwx 12 root root 2048 Jan 30 2004 VR
drwxr-xr-x 24 12873 root 2048 May 31 1997 Visualization
drwxrwxrwx 2 root root 2048 May 20 1996 Vosaic
drwxr-xr-x 6 12873 root 2048 May 16 1997 Web
drwxrwxrwx 24 root root 2048 Sep 22 2006 aces
drwxr-xr-x 10 12873 root 2048 Aug 22 1994 aff
drwxrwxrwx 2 root root 2048 Jul 24 2002 aips
drwxrwxrwx 12 root root 43008 Aug 27 00:30 alg
drwxrwxrwx 4 root root 2048 Mar 17 2003 benchmarks
dr-xrwxrwx 2 root root 2048 Apr 21 2006 bin
drw-r--r-- 2 15332 wheel 2048 Jan 27 1999 chemistry
drwxrwxrwx 68 root root 4096 Apr 12 2007 cosmic
drwx------ 3 80 ncsa 2048 May 24 2002 dbalsara
drwx------ 2 80 games 2048 Dec 23 1997 emerge
drwxr-xr-x 3 80 root 2048 Jun 27 2006 etc
drwxr-xr-x 4 80 sys 2048 May 12 2009 incoming
drwx------ 3 80 games 2048 Aug 25 1997 java
drwx------ 2 80 games 2048 Jul 22 1997 joule
drwx------ 7 80 ncsa 2048 Jun 5 2000 lca
drwxrwxrwx 2 root root 2048 Nov 9 2001 lib
drwxrwxrwx 16 root root 2048 Sep 25 2003 media
drwxr-xr-x 14 12873 root 2048 Nov 21 2001 misc
drwxrwxrwx 21 root root 2048 Mar 16 2001 ncsapubs
drwxrwxrwx 14 root root 2048 Oct 3 2007 netdev
drwxrwxr-x 31 12984 wheel 2048 Mar 19 2015 outgoing
drwxr-xr-x 4 12873 root 2048 Dec 21 1999 sc22wg5
drwx------ 2 nobody games 2048 Mar 15 2007 security
drwxr-xr-x 2 root root 2048 Dec 22 1999 x3j3
Worker thread 3 exiting.
226 Transfer complete.
Worker thread 2 exiting.
Host: ftp.x.org
-rw-r--r-- 2 ftp ftp 479 Nov 6 2001 banner2
lrwxrwxrwx 1 ftp ftp 7 Oct 9 2005 bin -> usr/bin
drwxr-xr-x 34 ftp ftp 4096 Nov 6 2001 contrib
d--x--x--x 2 ftp ftp 4096 Aug 8 2001 dev
drwxr-xr-x 2 ftp ftp 4096 Oct 12 1998 digest
d--x--x--x 3 ftp ftp 4096 Aug 8 2001 etc
-rw-r--r-- 1 ftp ftp 3075 Oct 7 1998 GettingBroadway
-rw-r--r-- 1 ftp ftp 3075 Oct 7 1998 GettingR6.3
-rw-r--r-- 1 ftp ftp 1847 Feb 1 1999 GettingR6.4
-rw-r--r-- 1 ftp ftp 1378 Aug 24 2000 GettingR6.5.1
-rw-r--r-- 1 ftp ftp 1229 Apr 24 2001 GettingR6.6
-rw-r--r-- 1 ftp ftp 301 Dec 22 2005 GettingR6.7
-rw-r--r-- 1 ftp ftp 396 Dec 22 2005 GettingR6.8
-rw-r--r-- 1 ftp ftp 396 Dec 22 2005 GettingR6.9
-rw-r--r-- 1 ftp ftp 396 Dec 22 2005 GettingR7.0
-rw-r--r-- 1 ftp ftp 195 Dec 30 2004 MIRROR.README
drwxrwxr-x 36 ftp ftp 4096 Jun 30 2013 pub
drwxr-xr-x 56 ftp ftp 24576 May 4 2001 R5contrib
drwxr-xr-x 2 ftp ftp 4096 May 4 2001 rcsfaq
-r--r--r-- 1 ftp ftp 1571 Dec 22 2005 README
lrwxrwxrwx 1 ftp ftp 6 Nov 27 2005 README.txt -> README
d--x--x--x 5 ftp ftp 4096 Aug 8 2001 usr
-rw-r--r-- 2 ftp ftp 479 Nov 6 2001 welcome.msg
226 Transfer complete
Worker thread 0 exiting.
Scanning complete !!!
2 answers
solution1
1 ACCPTED 2015-10-20 04:14:57
That's because your lock variable is an instance variable, and not a variable shared between your threads as you want it to be.
You defined the variable as an instance variable when you wrote
def __init__(self, queue, tid):
...
self.lock = thread.Lock()
...
You can check this by
def __init__(self,queue,tid):
threading.Thread.__init__(self)
self.queue = queue
self.tid = tid
print "%d !!!"% id(self.lock) # all threads print different ids
For each of the threads, this gives a different id , meaning that they are independent objects.
What you want is to create a class variable. Class variables are shared across all instances of the class.
class workerClass(threading.Thread):
lock = threading.Lock()
def __init__(self,queue,tid):
threading.Thread.__init__(self)
self.queue = queue
self.tid = tid
def run(self):
lock.acquire()
# do stuff
lock.release()
solution2
0 2015-10-20 06:05:36
#!/usr/bin/python
import threading
import Queue
from ftplib import FTP
ftpSiteList = [ 'ftp.x.org', 'ftp4.FreeBSD.org', 'ftp.ncsa.uiuc.edu','ftp.crans.org' ]
threadsList = []
class workerClass(threading.Thread):
lock = threading.Lock()
def __init__(self,queue,tid):
threading.Thread.__init__(self)
self.queue = queue
self.tid = tid
print "Worker Thread %d reporting for Service with the lock id %d!!!"%(self.tid,id(workerClass.lock))
def run(self):
try:
workerClass.lock.acquire()
while True:
host = ''
try:
host = self.queue.get(timeout=1)
except Queue.Empty:
print "Worker thread %d exiting. "%(self.tid)
return
try:
conn = FTP(host)
conn.login()
print 'Host: ' +host
print conn.retrlines('LIST')
except:
print "Error in listing HOST "+host
raise
self.queue.task_done()
finally:
workerClass.lock.release()
queue = Queue.Queue()
for threads in range(5):
worker = workerClass(queue,threads)
worker.setDaemon(True)
worker.start()
threadsList.append(worker)
print "Thread %d started .."%threads
for host in ftpSiteList:
queue.put(host)
queue.join()
for threads in threadsList:
threads.join()
print "Scanning completed !!!"
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.
Related Question
Why isn't __add__ implemented in Python's deque?
Why this python script isn't executing in java
Why isn't the 'And' in my python script work
Why isn't reversed() implemented for all collections?
Why isn't my implemented normalization working?
Why isn't there any special method for __max__ in python?
why isn't python accessible from within a bash script?
New to python. Why Isn't my script uploading files?
Why isn't this simple python script using openCV working?
Why isn't my Python script returning the page source properly?
Related Tags