Insert python pandas dataframe to a SQL Server table

Question

I had try insert a pandas dataframe into my SQL Server database.

The connections works fine, but when I try create a table is not ok.

My connection:

import pyodbc
cnxn = pyodbc.connect(
    Trusted_Connection='Yes',
    Driver= '{SQL Server}',
    Server='SVR-DGT',
    Database='DGTCAR',
    UID ='sa',  
    PWD ='DgT(FFCC)35'
)

So I am trying:

 df_clientes.to_sql('clientes', cnxn, if_exists='replace')

Error:

Error:

---------------------------------------------------------------------------
ProgrammingError                          Traceback (most recent call last)
C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in execute(self, *args, **kwargs)
   2055         try:
-> 2056             cur.execute(*args, **kwargs)
   2057             return cur

ProgrammingError: ('42S02', "[42S02] [Microsoft][ODBC SQL Server Driver][SQL Server]Nome de objeto 'sqlite_master' inválido. (208) (SQLExecDirectW); [42S02] [Microsoft][ODBC SQL Server Driver][SQL Server]Não foi possível preparar uma ou mais instruções. (8180)")

The above exception was the direct cause of the following exception:

DatabaseError                             Traceback (most recent call last)
C:\Users\ADMINI~1\AppData\Local\Temp\2/ipykernel_4944/338003934.py in <module>
      1 #df_clientes.to_sql('clientes', con=cnxn)
      2 
----> 3 df_clientes.to_sql('clientes', cnxn, if_exists='replace')
      4 
      5 #df_clientes.to_sql('clientes', cnxn, if_exists='append')

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\generic.py in to_sql(self, name, con, schema, if_exists, index, index_label, chunksize, dtype, method)
   2870         from pandas.io import sql
   2871 
-> 2872         sql.to_sql(
   2873             self,
   2874             name,

C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in to_sql(frame, name, con, schema, if_exists, index, index_label, chunksize, dtype, method, engine, **engine_kwargs)
    715         )
    716 
--> 717     pandas_sql.to_sql(
    718         frame,
    719         name,

C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in to_sql(self, frame, name, if_exists, index, index_label, schema, chunksize, dtype, method, **kwargs)
   2223             dtype=dtype,
   2224         )
-> 2225         table.create()
   2226         table.insert(chunksize, method)
   2227 

C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in create(self)
    854 
    855     def create(self):
--> 856         if self.exists():
    857             if self.if_exists == "fail":
    858                 raise ValueError(f"Table '{self.name}' already exists.")

C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in exists(self)
    838 
    839     def exists(self):
--> 840         return self.pd_sql.has_table(self.name, self.schema)
    841 
    842     def sql_schema(self):

C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in has_table(self, name, schema)
   2234         query = f"SELECT name FROM sqlite_master WHERE type='table' AND name={wld};"
   2235 
-> 2236         return len(self.execute(query, [name]).fetchall()) > 0
   2237 
   2238     def get_table(self, table_name: str, schema: str | None = None):

C:\ProgramData\Anaconda3\lib\site-packages\pandas\io\sql.py in execute(self, *args, **kwargs)
   2066 
   2067             ex = DatabaseError(f"Execution failed on sql '{args[0]}': {exc}")
-> 2068             raise ex from exc
   2069 
   2070     @staticmethod

DatabaseError: Execution failed on sql 'SELECT name FROM sqlite_master WHERE type='table' AND name=?;': ('42S02', "[42S02] [Microsoft][ODBC SQL Server Driver][SQL Server]Nome de objeto 'sqlite_master' inválido. (208) (SQLExecDirectW); [42S02] [Microsoft][ODBC SQL Server Driver][SQL Server]Não foi possível preparar uma ou mais instruções. (8180)")

Answer 1

Just use df.to_sql.

Create an in-memory SQLite database.

from sqlalchemy import create_engine

engine = create_engine('sqlite://', echo=False)

Create a table from scratch with 3 rows.

df = pd.DataFrame({'name' : ['User 1', 'User 2', 'User 3']})

df
     name
0  User 1
1  User 2
2  User 3

df.to_sql('users', con=engine)
3

engine.execute("SELECT * FROM users").fetchall()
[(0, 'User 1'), (1, 'User 2'), (2, 'User 3')]

An sqlalchemy.engine.Connection can also be passed to con:

with engine.begin() as connection:

    df1 = pd.DataFrame({'name' : ['User 4', 'User 5']})

    df1.to_sql('users', con=connection, if_exists='append')
2

This is allowed to support operations that require that the same DBAPI connection is used for the entire operation.

df2 = pd.DataFrame({'name' : ['User 6', 'User 7']})

df2.to_sql('users', con=engine, if_exists='append')
2

engine.execute("SELECT * FROM users").fetchall()
[(0, 'User 1'), (1, 'User 2'), (2, 'User 3'),
 (0, 'User 4'), (1, 'User 5'), (0, 'User 6'),
 (1, 'User 7')]

Overwrite the table with just df2.

df2.to_sql('users', con=engine, if_exists='replace',

           index_label='id')
2

engine.execute("SELECT * FROM users").fetchall()
[(0, 'User 6'), (1, 'User 7')]

Specify the dtype (especially useful for integers with missing values). Notice that while pandas is forced to store the data as floating point, the database supports nullable integers. When fetching the data with Python, we get back integer scalars.

df = pd.DataFrame({"A": [1, None, 2]})

df
     A
0  1.0
1  NaN
2  2.0

from sqlalchemy.types import Integer

df.to_sql('integers', con=engine, index=False,

          dtype={"A": Integer()})
3

engine.execute("SELECT * FROM integers").fetchall()
[(1,), (None,), (2,)]

https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html