[英]Getting a memory error when parsing a large XML file in Python
我的XML文件如下所示:
<root>
<group from="1", to="100">
<link target="1"/>
...
<link target="100"/>
</group>
...
</root>
我有6000个<group>
元素和5M个<link>
元素。 我想要一本以元组( from
, to
)作为键的字典,并列出<link>
的target
属性,但是使用以下代码遇到内存错误:
from lxml import etree
from gzip import open as gopen
def extractTargets(fin):
targets = dict()
with gopen(fin) as xml:
context = etree.iterparse(xml, tag="group")
for event, elem in context:
targets[(elem.get("from"), elem.get("to"))] = elem.xpath("link/@target")
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
del context
尝试以下代码:
import lxml.etree
from gzip import open as gopen
class GroupDictTarget(object):
def __init__(self, d):
self.d = d
def start(self, tag, attrib):
if tag == 'group':
self.group = self.d[attrib['from'], attrib['to']] = []
elif tag == 'link':
self.group.append(attrib['target'])
def close(self):
pass
def extractTargets(fin):
with gopen(fin) as xml:
targets = {}
parser = lxml.etree.XMLParser(target=GroupDictTarget(targets))
lxml.etree.parse(xml, parser)
return targets
import xml.parsers.expat
from gzip import open as gopen
class GroupDictTarget(object):
# Same as above
def extractTargets(fin):
targets = {}
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = GroupDictTarget(targets).start
with gopen(fin) as f:
p.ParseFile(f)
return targets
import xml.sax
from gzip import open as gopen
class GroupDictTarget(object):
# Same as above
def extractTargets(fin):
targets = {}
handler = xml.sax.handler.ContentHandler()
handler.startElement = GroupDictTarget(targets).start
with gopen(fin) as f:
xml.sax.parse(f, handler)
return targets
今天,我遇到了同样的问题,对我来说,删除“ tag”参数后,它可以正常工作:
context = etree.iterparse(xml)
for event, elem in context:
if elem.tag = "group":
targets[(elem.get("from"), elem.get("to"))] = elem.xpath("link/@target")
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
del context
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.