使用HTMLParser解析Unicode時出錯

Question

這是我為Python 3.3運行的代碼：

from html.parser import HTMLParser

class TableParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.in_table = False
        self.in_table_header = False
        self.in_table_header_field = False
        self.table_fields = []

    def handle_starttag(self, tag, attributes):
        if tag == 'table':
            for name, value in attributes:
                if name == 'id' and value == 'data_table':
                    self.in_table = True
        if self.in_table == True:
            if tag == 'thead':
                self.in_table_header = True
        if self.in_table_header == True and tag == 'th':
            self.in_table_header_field = True

    def handle_endtag(self, tag):
        if tag == 'table':
            self.in_table = False
        if tag == 'thead':
            self.in_table_header = False
        if tag == 'th':
            self.in_table_header_field = False            

    def handle_data(self, data):
        if self.in_table_header_field == True:
            self.table_fields.append(data)

parser = TableParser()
parser.feed('<table id="data_table"><thead><tr><th>Company</th><th>Rapport</th><th>Price</th><th>Development 1&#229;r</th></thead></table>')
print(parser.table_fields)

這是輸出：

['Company', 'Rapport', 'Price', 'Development 1', 'r']

我期待着：

['Company', 'Rapport', 'Price', 'Development 1år']

還是更好：

['Company', 'Rapport', 'Price', 'Development 1år']

我究竟做錯了什么？

Answer 1

您還需要為HTMLParser.handle_charref()方法添加一個處理程序：

def handle_charref(self, name):
    self.handle_data(self.unescape('&#{};'.format(name)))

Answer 2

使用lxml ：

>>> import lxml.html
>>> root = lxml.html.fromstring('<table id="data_table"><thead><tr><th>Company</th><th>Rapport</th><th>Price</th><th>Development 1&#229;r</th></thead></table>') 
>>> root.xpath('.//thead//th/text()')
['Company', 'Rapport', 'Price', 'Development 1år']