Read from multiple excel files and Insert them into a table in PosgreSQL DB

Question

I'm going to read multiple excel files located in a specific folder in PC and which are in the same construct and then insert them into the DB table in PostgreSQL. I'm reading excel files, however, I'm having an error during cur.execute section.

Example Excel file:

And My script:

import pandas as pd
import glob
import psycopg2
import numpy
from psycopg2.extensions import register_adapter, AsIs

def addapt_numpy_float64(numpy_float64):
    return AsIs(numpy_float64)
def addapt_numpy_int64(numpy_int64):
    return AsIs(numpy_int64)
register_adapter(numpy.float64, addapt_numpy_float64)
register_adapter(numpy.int64, addapt_numpy_int64)

def connect_db(frame):
    # database connection

    print("DB opened successfully")
    sqlpart(conn, frame)


def sqlpart(conn,frame):
    cur = conn.cursor()

    sql="""INSERT INTO actuals (timeslot,unit_id,actual) 
            VALUES(%s,%s,%s) 
            ON CONFLICT (timeslot,unit_id,actual)
            DO UPDATE SET  timeslot=%s, unit_id=%s,actual=%s"""

    conn.commit()
    values=[]
    da=[]
    for x in range(len(frame)):
        da=frame[0][x],  frame[1][x],  frame[2][x]
        values.append(da)
        print(values)
    cur.execute(sql, values)
    print("everything is done")

def patika():

    path = r'mypath'
    all_files = glob.glob(path + "/*.xlsx")

    li = []

    for filename in all_files:
        df = pd.read_excel(filename, index_col=None,header=None)
        li.append(df)

    frame = pd.concat(li, axis=0, ignore_index=True)
    #print(frame)--->> [(Timestamp('2020-03-01 00:00:00'), 200, -1000),....] values that comes from excel like this

    connect_db(frame)


patika()

Output:

cur.execute(sql, values)
TypeError: not all arguments converted during string formatting

Answer 1

Your query has six parameters that need to bind. As you verified in a comment, your binding tuples contain only three elements:

sql="""INSERT INTO actuals (timeslot,unit_id,actual) 
            VALUES(%s,%s,%s) 
            ON CONFLICT (timeslot,unit_id,actual)
            DO UPDATE SET  timeslot=%s, unit_id=%s,actual=%s"""

Please update your insert to:

sql="""INSERT INTO actuals (timeslot,unit_id,actual) 
            VALUES(%s,%s,%s) 
            ON CONFLICT (timeslot,unit_id,actual)
            DO NOTHING"""

In case your table is unique on (timeslot, unit_id) , then your insert would be:

sql="""INSERT INTO actuals (timeslot,unit_id,actual) 
            VALUES(%s,%s,%s) 
            ON CONFLICT (timeslot,unit_id)
            DO UPDATE SET actual = EXCLUDED.actual"""

UPDATE On reading more closely, you have multiple problems in your code:

def sqlpart(conn,frame):
    cur = conn.cursor()

    sql="""INSERT INTO actuals (timeslot,unit_id,actual) 
            VALUES(%s,%s,%s) 
            ON CONFLICT (timeslot,unit_id,actual)
            DO UPDATE SET  timeslot=%s, unit_id=%s,actual=%s"""

    conn.commit()
    values=[]
    da=[]
    for x in range(len(frame)):
        da=frame[0][x],  frame[1][x],  frame[2][x]
        values.append(da)
        print(values)
    cur.execute(sql, values)
    print("everything is done")

First, your conn.commit() is not being done after the execution of the insert .

Second, you are attempting to pass a list of tuple objects to your insert execution.

Please try something like this:

def sqlpart(conn,frame):
    cur = conn.cursor()

    sql="""INSERT INTO actuals (timeslot,unit_id,actual) 
            VALUES(%s,%s,%s) 
            ON CONFLICT (timeslot,unit_id,actual)
            DO NOTHING"""

    #values=[]
    #da=[]
    for x in range(len(frame)):
        da=frame[0][x],  frame[1][x],  frame[2][x]
        #values.append(da)
        #print(values)
        cur.execute(sql, da)
    conn.commit()
    print("everything is done")