How to create a sql server table variable in a query using sqlalchemy in python

Question

I'm trying to create a table variable in SQL Server, query it, and return the results to a pandas dataframe (see example). I want to do this so that I can aggregate data in the database prior to sending it to a pandas dataframe. I recall that setting NOCOUNT ON would allow for this to work since it wouldn't return anything as it executed each query. But this isn't working. So this is obviously an example code, but I've been able to recreate the error here. Following the suggested link gives you the documentating for ProgrammingErrors . I didn't find it very helpful.

import urllib
import sqlalchemy
import pandas as pd

quoted = urllib.parse.quote_plus('DRIVER={ODBC Driver 17 for SQL Server};Server=127.0.0.1;Database=mydb;UID=myuser;PWD=mypasswd;Port=1433;')
engine = sqlalchemy.create_engine('mssql+pyodbc:///?odbc_connect={}'.format(quoted))

query = """
SET NOCOUNT ON;

DECLARE @n_majors TABLE (id varchar(9), n_majors int)

INSERT INTO @n_majors
SELECT m.student_id_fk
, COUNT(DISTINCT dc.category) AS [N majors declared]
FROM msu_db.dbo.Majors AS m
JOIN department_categories AS dc
ON dc.dept_name = m.dept_name
WHERE m.Student_Level_Code = 'UN'
GROUP BY m.student_id_fk

DECLARE @grad_category TABLE (id varchar(9), category varchar(20))

INSERT INTO @grad_category
select m.student_id_fk
, MIN(dc.category)
from Majors AS m
join department_categories as dc
on dc.dept_name = m.dept_name
WHERE m.Student_Level_Code = 'UN'
and graduated = 'CONF'
GROUP BY m.student_id_fk

DECLARE @first_category TABLE (id varchar(9), category varchar(20))

INSERT INTO @first_category
select m.student_id_fk
, MIN(dc.category) as cat
from Majors AS m
join department_categories as dc
on dc.dept_name = m.dept_name
WHERE m.Student_Level_Code = 'UN'
and graduated IS NULL
GROUP BY m.student_id_fk

DECLARE @first_semester_grades TABLE (id varchar(9), avg_grade float, std_grade float, first_Semester_seq_id varchar(4))

INSERT INTO @first_semester_grades
SELECT c.student_id_fk
, AVG(c.Grade) AS [mean grade]
, STDEV(c.Grade) AS [stdev grade]
, MIN(c.Term_Seq_Id) AS Term_Seq_Id
FROM Courses AS c
WHERE c.Student_Level_Code = 'UN'
GROUP BY c.student_id_fk

SET NOCOUNT OFF;

SELECT  s.[student_id_fk]
      ,[gender]
      ,[ethnicity]
      ,[first_course_datetime]
      ,[hs_gpa]
      ,[math_placement_score]
      ,[math_act]
      ,[natsci_act]
      ,COUNT(c.[transfer institution name]) AS [N AP courses]
      , nm.n_majors AS [n-categories]
      , fc.category
      , gc.category AS [grad category]
      , fsg.avg_grade AS first_term_avg
      , fsg.std_grade AS first_term_std
      , fsg.first_Semester_seq_id
  FROM [msu_db].[dbo].[Students] AS s
  LEFT JOIN msu_db.dbo.Courses AS c
  ON s.student_id_fk = c.student_id_fk
  AND c.[transfer institution name] = 'Advanced Placement'

  LEFT JOIN @n_majors as nm
  ON s.student_id_fk = nm.id

  LEFT JOIN @grad_category as gc
  ON s.student_id_fk = gc.id

  LEFT JOIN @first_category AS fc
  ON s.student_id_fk = fc.id

  LEFT JOIN @first_semester_grades AS fsg
  ON s.student_id_fk = fsg.id

  WHERE s.first_course_datetime BETWEEN '1993' AND '2013'

  GROUP BY s.[student_id_fk]
      ,[gender]
      ,[ethnicity]
      ,[first_course_datetime]
      ,[hs_gpa]
      ,[math_placement_score]
      ,[math_act]
      ,[natsci_act]
      , nm.n_majors
      , fc.category
      , gc.category
      , fsg.avg_grade
      , fsg.std_grade
      , fsg.first_Semester_seq_id
    """
pd.read_sql_query(query, engine)

The error message that is output is as follows:

     --------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/engine/result.py in _fetchall_impl(self)
   1081         try:
-> 1082             return self.cursor.fetchall()
   1083         except AttributeError:

AttributeError: 'NoneType' object has no attribute 'fetchall'

During handling of the above exception, another exception occurred:

ResourceClosedError                       Traceback (most recent call last)
<ipython-input-3-2a0ea765a8e2> in <module>()
----> 1 df = pd.read_sql_query(query, engine)

~/anaconda3/envs/research/lib/python3.6/site-packages/pandas/io/sql.py in read_sql_query(sql, con, index_col, coerce_float, params, parse_dates, chunksize)
    312     return pandas_sql.read_query(
    313         sql, index_col=index_col, params=params, coerce_float=coerce_float,
--> 314         parse_dates=parse_dates, chunksize=chunksize)
    315 
    316 

~/anaconda3/envs/research/lib/python3.6/site-packages/pandas/io/sql.py in read_query(self, sql, index_col, coerce_float, parse_dates, params, chunksize)
   1070                                         parse_dates=parse_dates)
   1071         else:
-> 1072             data = result.fetchall()
   1073             frame = _wrap_result(data, columns, index_col=index_col,
   1074                                  coerce_float=coerce_float,

~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/engine/result.py in fetchall(self)
   1135             self.connection._handle_dbapi_exception(
   1136                 e, None, None,
-> 1137                 self.cursor, self.context)
   1138 
   1139     def fetchmany(self, size=None):

~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1414                 )
   1415             else:
-> 1416                 util.reraise(*exc_info)
   1417 
   1418         finally:

~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/util/compat.py in reraise(tp, value, tb, cause)
    185         if value.__traceback__ is not tb:
    186             raise value.with_traceback(tb)
--> 187         raise value
    188 
    189 else:

~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/engine/result.py in fetchall(self)
   1129 
   1130         try:
-> 1131             l = self.process_rows(self._fetchall_impl())
   1132             self._soft_close()
   1133             return l

~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/engine/result.py in _fetchall_impl(self)
   1082             return self.cursor.fetchall()
   1083         except AttributeError:
-> 1084             return self._non_result([])
   1085 
   1086     def _non_result(self, default):

~/anaconda3/envs/research/lib/python3.6/site-packages/sqlalchemy/engine/result.py in _non_result(self, default)
   1087         if self._metadata is None:
   1088             raise exc.ResourceClosedError(
-> 1089                 "This result object does not return rows. "
   1090                 "It has been closed automatically.",
   1091             )

ResourceClosedError: This result object does not return rows. It has been closed automatically.

It seems like as soon as the NoneType object gets passed, it fails. What I don't understand is why a NoneType object is being passed in the first place. Shouldn't the query results be passed?

Answer 1

You misspelled table in the variable declaration - it has a 1 instead of an l. If something isn't working that you believe should work, check your assumptions first.

Update:

import urllib
import sqlalchemy
import pandas as pd

quoted = urllib.parse.quote_plus('DRIVER={ODBC Driver 17 for SQL Server};Server=127.0.0.1;Database=mydb;UID=myuser;PWD=mypasswd;Port=1433;')
engine = sqlalchemy.create_engine('mssql+pyodbc:///?odbc_connect={}'.format(quoted))

query = """
SET NOCOUNT ON
DECLARE @table TABLE (id int, value float)
INSERT INTO @table VALUES (1, 2.7)
INSERT INTO @table VALUES (2, 4.5)
INSERT INTO @table VALUES (3, 1.2)

SELECT * FROM @table
"""
pd.read_sql_query(query, engine)

Answer 2

You have to turn NOCOUNT back off before returning your query result for the correct row(s) affected message to be returned from SQL Server:

import urllib
import sqlalchemy
import pandas as pd

quoted = urllib.parse.quote_plus('DRIVER={ODBC Driver 17 for SQL Server};Server=127.0.0.1;Database=mydb;UID=myuser;PWD=mypasswd;Port=1433;')
engine = sqlalchemy.create_engine('mssql+pyodbc:///?odbc_connect={}'.format(quoted))

query = """
SET NOCOUNT ON
DECLARE @table TABLE (id int, value float)
INSERT INTO @table VALUES (1, 2.7)
INSERT INTO @table VALUES (2, 4.5)
INSERT INTO @table VALUES (3, 1.2)

SET NOCOUNT OFF
SELECT * FROM @table
"""
pd.read_sql_query(query, engine)