将元组列表附加到数据框 python panda

Question

tlist = [("a1", "a2","a3"),("b1", "b2","b3"),("c1", "c2","c3")]

我想要的是：

df=pd.DataFrame([["a1","a2","a3"],["b1","b2","b3"],["c1","c2","c3"]])

我可以做这个：

df2 = pd.DataFrame(tlist, columns=['col1', 'col2', 'col3'])

然而，元组列表是从某个数据库中提取的，所以我有一个循环，并一次提取一个数据块，然后追加。

什么是最好的方法来做到这一点？

数据拉取现在可以达到 10 亿行，并且可以增长。

谢谢。

#very big table#
sql2 = "Select col1,col2,col3 from bigT"
#very big table#
try:
  cursor.execute (sql2)
except cx_Oracle.DatabaseError:
  print ('Failed \n'+sql2)

#need to do it in chunk as not enough memory and blow up!
while True:
   tlist = cursor.fetchmany()
   print(type(tlist))
   print (len(tlist))
   if rows == []:
        break;
   #I cannot get this one to work
   df.append([tlist],ignore_index=True)
   #I cannot get this one to work

Answer 1

我找到了一个更简单的解决方案！

import pandas as pd
print(con.version)
query = """select * from all_tab_columns"""
df_ora = pd.read_sql(query, con=con)

Answer 2

import cx_Oracle



conn_str="scott/tiger@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=db1.org.waaw.com)(PORT=1234))(CONNECT_DATA=(SERVICE_NAME=hatsx)))"


con = cx_Oracle.connect(conn_str)
cursor= con.cursor ()
cursor.arraysize = 10000

import pandas as pd
rowsx=[("xxxxxxxxxx","xxxxxxxxxx","xxxxxxxxxx","xxxxxxxxxx","xxxxxxxxxx")]
labels=['col1', 'col2','col3']
df = pd.DataFrame(rowsx, columns=labels)


#verify the connection
print (con.version)
#verify the connection

#very big table#
sql2 = """Select col1,col2,col3 from bigT"""
#very big table#

try:
  cursor.execute (sql2)
except cx_Oracle.DatabaseError:
  print ('Failed \n'+sql2)

#need to do it in chunk as not enough memory and blow up!
while True:
   rows = cursor.fetchmany()
   if rows == []:
        break;
   df2=pd.DataFrame.from_records(rows,columns=labels)    
   df=df.append(df2)