[英]Encoding Problems in python3
我正在關注使用 reddit 評論制作聊天機器人的 senddex 教程,並將一個月的評論下載為 .bz2 文件。
當我運行下面鏈接的代碼時,出現錯誤
Traceback (most recent call last):
File "/Users/my_user/Documents/ok.py", line 94, in <module>
for row in x:
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/codecs.py", line 322,
in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd9 in position 13: invalid continuation
byte
我的代碼如下:
import sqlite3
import json
import datetime
timeframe = '2015-01'
sql_transaction = []
connection = sqlite3.connect('{}.db'.format(timeframe))
c = connection.cursor()
def create_table():
c.execute("CREATE TABLE IF NOT EXISTS parent_reply(parent_id TEXT PRIMARY KEY, comment_id TEXT UNIQUE, parent TEXT, comment TEXT, subreddit TEXT, unix INT, score INT)")
def format_data(data):
data = data.replace('\n',' newlinechar ').replace('\r',' newlinechar ').replace('"',"'")
return data
def transaction_bldr(sql):
global sql_transaction
sql_transaction.append(sql)
if len(sql_transaction) > 1000:
c.execute('BEGIN TRANSACTION')
for s in sql_transaction:
try:
c.execute(s)
except:
pass
connection.commit()
sql_transaction = []
def sql_insert_replace_comment(commentid,parentid,parent,comment,subreddit,time,score):
try:
sql = """UPDATE parent_reply SET parent_id = ?, comment_id = ?, parent = ?, comment = ?, subreddit = ?, unix = ?, score = ? WHERE parent_id =?;""".format(parentid, commentid, parent, comment, subreddit, int(time), score, parentid)
transaction_bldr(sql)
except Exception as e:
print('s0 insertion',e)
def sql_insert_has_parent(commentid,parentid,parent,comment,subreddit,time,score):
try:
sql = """INSERT INTO parent_reply (parent_id, comment_id, parent, comment, subreddit, unix, score) VALUES ("{}","{}","{}","{}","{}",{},{});""".format(parentid, commentid, parent, comment, subreddit, int(time), score)
transaction_bldr(sql)
except Exception as e:
print('s0 insertion',e)
def sql_insert_no_parent(commentid,parentid,comment,subreddit,time,score):
try:
sql = """INSERT INTO parent_reply (parent_id, comment_id, comment, subreddit, unix, score) VALUES ("{}","{}","{}","{}",{},{});""".format(parentid, commentid, comment, subreddit, int(time), score)
transaction_bldr(sql)
except Exception as e:
print('s0 insertion',e)
def acceptable(data):
if len(data.split(' ')) > 50 or len(data) < 1:
return False
elif len(data) > 1000:
return False
elif data == '[deleted]':
return False
elif data == '[removed]':
return False
else:
return True
def find_parent(pid):
try:
sql = "SELECT comment FROM parent_reply WHERE comment_id = '{}' LIMIT 1".format(pid)
c.execute(sql)
result = c.fetchone()
if result != None:
return result[0]
else: return False
except Exception as e:
#print(str(e))
return False
def find_existing_score(pid):
try:
sql = "SELECT score FROM parent_reply WHERE parent_id = '{}' LIMIT 1".format(pid)
c.execute(sql)
result = c.fetchone()
if result != None:
return result[0]
else: return False
except Exception as e:
#print(str(e))
return False
if __name__ == '__main__':
create_table()
row_counter = 0
paired_rows = 0
with open("/Users/my_user/Downloads/reddit_data/{}/RC_{}.bz2".format(timeframe.split('-')[0],timeframe), buffering=1000) as x:
for row in x:
row_counter += 1
row = json.loads(row)
parent_id = row['parent_id']
body = format_data(row['body'])
created_utc = row['created_utc']
score = row['score']
comment_id = row['name']
subreddit = row['subreddit']
parent_data = find_parent(parent_id)
if score >= 2:
existing_comment_score = find_existing_score(parent_id)
if existing_comment_score:
if score > existing_comment_score:
if acceptable(body):
sql_insert_replace_comment(comment_id,parent_id,parent_data,body,subreddit,created_utc,score)
else:
if acceptable(body):
if parent_data:
sql_insert_has_parent(comment_id,parent_id,parent_data,body,subreddit,created_utc,score)
paired_rows += 1
else:
sql_insert_no_parent(comment_id,parent_id,body,subreddit,created_utc,score)
if row_counter % 100000 == 0:
print('Total Rows Read: {}, Paired Rows: {}, Time: {}'.format(row_counter, paired_rows, str(datetime.now())))
謝謝你的時間!
PS對不起,如果我問的是一個愚蠢的問題,我對sqlite和json很陌生
您正在嘗試將 bz2 壓縮文件讀取為文本文件。
import bz2
if __name__ == '__main__':
create_table()
paired_rows = 0
with bz2.open("/Users/my_user/Downloads/reddit_data/{}/RC_{}.bz2".format(timeframe.split('-')[0],timeframe)) as rows:
for row_counter, row in enumerate(rows, 1):
row = json.loads(row)
...
但這並不是唯一的問題。 切勿將參數格式化為 SQL 語句。 例如:
def find_parent(pid):
sql = "SELECT comment FROM parent_reply WHERE comment_id = ? LIMIT 1"
c.execute(sql, [pid])
result = c.fetchone()
if result is not None:
return result[0]
return False
但是函數應該有一個更好的名字,比如fetch_comment
,如果沒有注釋,它不應該返回False
, None
會更好。 comment_id
或parent_id
是否正確?
總之,這應該是這樣的:
import sqlite3
import json
import datetime
import bz2
TIMEFRAME = '2015-01'
def create_table(cursor):
cursor.execute("CREATE TABLE IF NOT EXISTS parent_reply(parent_id TEXT PRIMARY KEY, comment_id TEXT UNIQUE, parent TEXT, comment TEXT, subreddit TEXT, unix INT, score INT)")
def format_data(data):
return data.replace('\n',' newlinechar ').replace('\r',' newlinechar ').replace('"',"'")
def sql_insert_replace_comment(cursor, commentid,parentid,parent,comment,subreddit,time,score):
cursor.execute("""UPDATE parent_reply SET comment_id = ?, parent = ?, comment = ?, subreddit = ?, unix = ?, score = ? WHERE parent_id =?;""",
[commentid, parent, comment, subreddit, int(time), score, parentid])
def sql_insert(cursor, commentid,parentid,parent,comment,subreddit,time,score):
cursor.execute("""INSERT INTO parent_reply (parent_id, comment_id, parent, comment, subreddit, unix, score) VALUES (?,?,?,?,?,?,?);""",
[parentid, commentid, parent, comment, subreddit, int(time), score])
def acceptable(data):
return (
len(data.split(' ')) <= 50
and 1 <= len(data) <= 1000
and data not in ('[deleted]','[removed]')
)
def fetch_comment(cursor, pid):
# TODO: is this comment or parent data?
cursor.execute("SELECT comment FROM parent_reply WHERE comment_id = ? LIMIT 1", [pid])
result = cursor.fetchone()
return result and result[0]
def fetch_score(cursor, pid):
cursor.execute("SELECT score FROM parent_reply WHERE parent_id = ? LIMIT 1", [pid])
result = cursor.fetchone()
return result and result[0]
def main():
connection = sqlite3.connect('{}.db'.format(TIMEFRAME))
cursor = connection.cursor()
create_table(cursor)
paired_rows = 0
with bz2.open("/Users/my_user/Downloads/reddit_data/{}/RC_{}.bz2".format(timeframe.split('-')[0],timeframe)) as rows:
for row_counter, row in enumerate(rows, 1):
row = json.loads(row)
parent_id = row['parent_id']
body = format_data(row['body'])
created_utc = row['created_utc']
score = row['score']
comment_id = row['name']
subreddit = row['subreddit']
parent_data = fetch_comment(cursor, parent_id) # or is this comment?
if score >= 2 and acceptable(body):
existing_comment_score = fetch_score(cursor, parent_id)
if existing_comment_score is not None:
if score > existing_comment_score:
sql_insert_replace_comment(cursor, comment_id,parent_id,parent_data,body,subreddit,created_utc,score)
else:
if parent_data:
paired_rows += 1
sql_insert(cursor, comment_id,parent_id,parent_data,body,subreddit,created_utc,score)
if row_counter % 100000 == 0:
print('Total Rows Read: {}, Paired Rows: {}, Time: {}'.format(row_counter, paired_rows, str(datetime.now())))
if __name__ == '__main__':
main()
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.