![](/img/trans.png)
[英]Why do I have a “segmentation fault” in C but no memory leak in Python?
[英]I think I have a memory leak in my python script
这是我的代码:
from xgoogle.search import GoogleSearch, SearchError
import urllib, urllib2, sys, argparse
global stringArr
stringArr = ["string 1",
"string 2",
"string 3",
"string etc"]
def searchIt(url):
try:
if(args.verbose>='1'): print "[INFO] Opening URL: "+url
response = urllib.urlopen(url)
except urllib2.URLError, e:
print "[ERROR] "+e.reason
return False
except KeyboardInterrupt:
print "Suspended by user..."
sys.exit()
if(checkForStr(response.read())):
if(args.verbose=='0'): print "[INFO] String found in URL: "+url
else:
if(args.verbose>='1'): print "[INFO] No string found in URL: "+url
def checkForStr(html):
global stringArr
try:
if any(checkStr in html for checkStr in stringArr):
return True
else:
return False
except KeyboardInterrupt:
print "Suspended by user..."
sys.exit()
def main():
try:
i=0
gs = GoogleSearch(args.keyword)
gs.results_per_page = 100
results = []
while True:
tmp = gs.get_results()
i = i+1 # page number
if not tmp: # no more results (pages) were found
break
results.extend(tmp)
for r in results: # process results for page
searchIt(r.url) # check for string
del results[:] # clean results
# finished
except SearchError, e:
print "[ERROR] Search failed: %s" % e
except KeyboardInterrupt:
print "Suspended by user..."
sys.exit()
if __name__ == '__main__':
try:
parser = argparse.ArgumentParser()
parser.add_argument('-v', dest='verbose', default='0', help='Verbosity level', choices='012')
group = parser.add_argument_group('required arguments')
group.add_argument('-k', dest='keyword', help='Keyword to use on google query', required=True)
args = parser.parse_args()
main()
except KeyboardInterrupt:
print "Suspended by user..."
sys.exit()
为了使它更易于阅读,我将其缩短了一点,但是它应该仍然可以正常工作。 此代码将是更大脚本的一部分。
我正在使用以下lib: XGOOGLE从Google抓取结果,然后访问每个结果以搜索网站是否包含来自stringArr的任何字符串。
我进行了第一个测试,没有任何问题(在少于10个结果的情况下,我ctrl + C进行了测试),但是我第一次让它运行,在测试了约100个网址之后,我遇到了这个错误:
File "./StringScan.py", line 99, in <module>
main()
File "./StringScan.py", line 83, in main
checkForStr(r.url)
File "./StringScan.py", line 39, in checkForStr
response = urllib.urlopen(url)
File "/usr/lib/python2.6/urllib.py", line 86, in urlopen
return opener.open(url)
File "/usr/lib/python2.6/urllib.py", line 205, in open
return getattr(self, name)(url)
File "/usr/lib/python2.6/urllib.py", line 344, in open_http
h.endheaders()
File "/usr/lib/python2.6/httplib.py", line 904, in endheaders
self._send_output()
File "/usr/lib/python2.6/httplib.py", line 776, in _send_output
self.send(msg)
File "/usr/lib/python2.6/httplib.py", line 735, in send
self.connect()
File "/usr/lib/python2.6/httplib.py", line 716, in connect
self.timeout)
File "/usr/lib/python2.6/socket.py", line 500, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
IOError: [Errno socket error] [Errno -2] Name or service not known
(行号不同,因为我修改了代码以将其发布在此处)
之后,我回到了Linux终端,就像脚本已完成一样。 但是我注意到我的电脑运行不正常,检查了系统监视器,然后看到Python进程使用了1.3gb的内存,我不得不终止该进程以使我的电脑恢复正常。
是我的代码中引起这种情况的原因,还是为什么会发生这种情况?
我知道我的代码可能有一些错误,但是现在我主要对可能导致内存问题的任何错误感兴趣。 任何帮助将不胜感激。
我对您的代码进行了一些重构,以使其更易于阅读。 我在这里看不到任何会泄漏内存的东西
from itertools import count
import urllib, urllib2, sys, argparse
from xgoogle.search import GoogleSearch, SearchError
stringArr = ["string 1",
"string 2",
"string 3",
"string etc"]
def searchIt(url):
try:
if(args.verbose>='1'):
print "[INFO] Opening URL: "+url
response = urllib.urlopen(url)
except urllib2.URLError, e:
print "[ERROR] "+e.reason
return False
if checkForStr(response.read()):
if(args.verbose=='0'):
print "[INFO] String found in URL: "+url
else:
if(args.verbose>='1'):
print "[INFO] No string found in URL: "+url
def checkForStr(html):
return any(checkStr in html for checkStr in stringArr)
def main():
try:
gs = GoogleSearch(args.keyword)
gs.results_per_page = 100
for i in count():
results = gs.get_results()
if not results: # no more results (pages) were found
break
for r in results: # process results for page
searchIt(r.url) # check for string
# finished
except SearchError, e:
print "[ERROR] Search failed: %s" % e
if __name__ == '__main__':
try:
parser = argparse.ArgumentParser()
parser.add_argument('-v', dest='verbose', default='0', help='Verbosity level', choices='012')
group = parser.add_argument_group('required arguments')
group.add_argument('-k', dest='keyword', help='Keyword to use on google query', required=True)
args = parser.parse_args()
main()
except KeyboardInterrupt:
print "Suspended by user..."
sys.exit()
可能是urllib.urlopen()。 参见http://bugs.python.org/issue1208304
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.