Python script writes twice in to the tables

Question

I am trying to automate this scenario. I have 2 .sql files ( add1.sql and add2.sql ) which has 1 insert script each.

My goal is to write one record to table1 by executing lines from add1.sql and one record to cm.cl by executing lines from add2.sql , waiting for about 5 mins so a backend service runs. This service writes from DB1 to DB2 . I then connect to DB2 to see if the record from DB1 matches what was written to DB2 . Depending no the results, an email is sent.

Below is my code. Everything works just fine except that it writes twice to DB1 . So, basically 4 records are inserted instead of 2. Any idea why it writes 4 records?

import pypyodbc as pyodbc
import smtplib
sender = 'abc@abc.com'
receivers = ['abc@abc.com','xyz@abc.com']
import unittest
import time

class TestDB1(unittest.TestCase):


    def testing_master(self):

           Master_Conn = 'Driver=
 {SQLServer};Server=server\servername;Database=database;UID=userid;PWD=password'
           Master_db = pyodbc.connect(Master_Conn)
           Master_Cursor = Master_db.cursor()
           try:
               #Open, read and execute add_shell.sql
               file = open('C:\\aaa\\add1.sql', 'r')
               line = file.read()
               lines = line.replace('\n', ' ')
               file1 = open('C:\\aaa\\add2.sql', 'r')
               line1=file1.read()
               lines1=line1.replace('\n', ' ')
               Master_Cursor.execute(lines)
               time.sleep(1)
               Master_Cursor.execute(lines1)
               Master_db.commit()
               file.close()
               file1.close()

               #Get python object for latest record inserted in DB1
               Master_CID=Master_Cursor.execute("select col1 from tablename1 order by sequenceid desc").fetchone()
               #convert tuple to srting [0] gives first tuple element. 
               Master_CID_str=str(Master_CID[0])
               #Get GUID by stripping first 2 chars and last char.
               Master_CID_str=Master_CID_str[2:len(Master_CID_str)-1]
               Master_CLID=Master_Cursor.execute("select col2 from tablename2 order by sequenceid desc").fetchone()
               Master_CLID_str=str(Master_CLID[0])
               Master_CLID_str=Master_CLID_str[2:len(Master_CLID_str) - 1]



           # Wait for service that transfers data from one db to another DB to run
               time.sleep(310)
           finally:
                Master_Cursor.close()
                Master_db.close()

           return Master_CID,Master_CID_str,Master_CLID,Master_CLID_str


    def testing_int_instance(self):
           #unpacking return value of tuple from testing_master() function
           Master_CID,Master_CID_str,Master_CLID,Master_CLID_str=self.testing_master()
           print ("printing from testing_int_instance {0}".format(Master_CID))
           Int_Instance_Conn = 'Driver={SQL Server};Server=server2\servername2;Database=database2;UID=uid;PWD=password;'
           Int_db = pyodbc.connect(Int_Instance_Conn)
           Int_Cursor = Int_db.cursor()
           #return Int_db, Int_Cursor
           #execute select from  db where col matches that of one inserted in master db.
           Int_Instance_CID=Int_Cursor.execute("select col1 from table1 where cartridgemodelid = '%s'" %(Master_CID_str)).fetchone()
           print(Int_Instance_CID)
           smtpObj = smtplib.SMTP('22.101.1.333', 25)
           if (Master_CID==Int_Instance_CID):
               print("Matched")
               content="This email confirms successful  data transfer from Master to  Instance for col1: \n"
               message = "\r\n".join(["From:" + sender,"To:" + str(receivers[:]),"Subject: Test Result","",content +Master_CID_str])
               #smtpObj = smtplib.SMTP('22.101.2.222', 25)
               smtpObj.sendmail(sender, receivers, message)
           elif (Master_CID!=Int_Instance_CID):
               print("no match")
               content = "This email confirms failure of  data transfer from DB1 to DB2 for COL1: \n"
               message = "\r\n".join(["From:" + sender, "To:" + str(receivers[:]), "Subject: Test Result", "",content +Master_CID_str])
               smtpObj.sendmail(sender, receivers, message)
           Int_Instance_CLID=Int_Cursor.execute("select COL2 from table2 where col= '%s'" %(Master_CLID_str)).fetchone()
           print (Int_Instance_CLID)
           if (Master_CLID == Int_Instance_CLID):
               print ("Printing int_instance CLID {0}".format(Int_Instance_CLID))
               content = "This email confirms successful data transfer from DB1 to DB2 for COL: \n"
               message = "\r\n".join(
                  ["From:" + sender, "To:" + str(receivers[:]), "Subject: Test Result", "", content + Master_CLID_str])
               #smtpObj = smtplib.SMTP('22.101.2.222', 25)
               smtpObj.sendmail(sender, receivers, message)
               print ("Ids Matched")
           elif (Master_CLID != Int_Instance_CLID):
DB1 to DB2 for COL: \n"
               message = "\r\n".join(
                  ["From:" + sender, "To:" + str(receivers[:]), "Subject: Test Result", "", content + Master_CLID_str])
               #smtpObj = smtplib.SMTP('22.101.2.222', 25)
               smtpObj.sendmail(sender, receivers, message)
           smtpObj.quit()


           Int_db.close()

if name == ' main ': unittest.main()

add1.sql is:

DECLARE @Name VARCHAR(2000)
DECLARE @PartNumber VARCHAR(2000)
SELECT @Name='test'+convert(varchar,getdate(),108)
SELECT @PartNumber='17_00001_'+convert(varchar,getdate(),108)
DECLARE @XML XML
DECLARE @FileName VARCHAR(1000)
DECLARE @Id UNIQUEIDENTIFIER
SELECT @Id = NEWID()
SELECT @FileName = 'test.xml'
SELECT @XML='<model>
   <xml tags go here>
BEGIN
  INSERT INTO table1
         (ID,Name,Type,Desc,Number,Revision,Model,status,Modifiedby,Modifiedon)
   VALUES(@Id,@Name,'xyz','',@partnumber,'01',@XML,'A','453454-4545-4545-4543-345342343',GETUTCDATE())

add2.sql is:

DECLARE @XML XML
DECLARE @CM_Name VARCHAR(2000)
DECLARE @FileName VARCHAR(1000)
DECLARE @PartNumber VARCHAR(2000)
DECLARE @Id UNIQUEIDENTIFIER
SELECT @Id=NEWID()
DECLARE @Name VARCHAR(2000)
DECLARE @CMId VARCHAR(2000)
DECLARE @CM_PartName VARCHAR(2000)
DECLARE @CM_Partnumber VARCHAR(2000)
SELECT @Name='test'+convert(varchar,getdate(),108)
SELECT @PartNumber='test'+convert(varchar,getdate(),108)
DECLARE @RowCount INT
DECLARE @Message VARCHAR(100);

SELECT @FileName = 'test.xml' 
SELECT @CMId = CM.CMID,
    @CM_Name = CM.CMName,
    @CM_PN  = CM.PN
    FROM cm.Model CM
    WHERE CM.MName LIKE 'test%'
    ORDER BY CM.ModifiedBy DESC 
SELECT @XML='<Layout>
   other xml tags...

BEGIN
  INSERT INTO cm.CL(ID, ModelID, Layout, Description, PN, Revision, CLayout, Status, ModifiedBy, ModifiedOn)
    SELECT TOP 1 @Id, @CMId, @Name, '', @PartNumber, '01', @XML, 'A', '453454-345-4534-4534-4534543545', GETUTCDATE()
    FROM cm.table1 CM
    WHERE CM.Name=@CM_Name
    AND CM.Partnumber=@CM_Partnumber

Answer 1

Currently, you are calling test_master() twice! First as your named method and then in second method when you unpack the returned values. Below is a demonstration of defined methods outside of the Class object. If called as is, testing_master will run twice.

Consider also using a context manager to read .sql scripts using with() which handles open and close i/o operations shown below:

# FIRST CALL
def testing_master():
    #...SAME CODE...
    try:
        with open('C:\\aaa\\add1.sql', 'r') as file:
            lines = file.read().replace('\n', ' ')    
        Master_Cursor.execute(lines)
        Master_db.commit()

        time.sleep(1)

        with open('C:\\aaa\\add2.sql', 'r') as file1:
            lines1 = file1.read().replace('\n', ' ')    
        Master_Cursor.execute(lines1)
        Master_db.commit()

    #...SAME CODE...  
    return Master_CID, Master_CID_str, Master_CLID, Master_CLID_str


def testing_int_instance():
    # SECOND CALL
    Master_CID, Master_CID_str, Master_CLID, Master_CLID_str = testing_master()    
    #...SAME CODE...

if __name__ == "__main__": 
    testing_master()
    testing_int_instance()

Commenting out the time(310) seems like it works but as you mention the background Windows service does not effectively run and so interrupts database transfer.

To resolve, consider calling the second method at the end of the first by passing the values as parameters without any return and remove unpacking line. Then, in the main global environment, only run testing_master() . Of course qualify with self when inside a Class definition.

def testing_master():
    #...SAME CODE...
    testing_int_instance(Master_CID, Master_CID_str, Master_CLID, Master_CLID_str)

def testing_int_instance(Master_CID, Master_CID_str, Master_CLID, Master_CLID_str):
    #...SKIP UNPACK LINE 
    #...CONTINUE WITH SAME CODE...

if __name__ == "__main__": 
    testing_master()

Due to your unittest , consider slight adjustment to original setup where you qualify every variable with self :

def testing_master():
    ...
    self.Master_CID=Master_Cursor.execute("select col1 from tablename1 order by sequenceid desc").fetchone()
    self.Master_CID_str=str(Master_CID[0])
    self.Master_CID_str=Master_CID_str[2:len(Master_CID_str)-1]
    self.Master_CLID=Master_Cursor.execute("select col2 from tablename2 order by sequenceid desc").fetchone()
    self.Master_CLID_str=str(Master_CLID[0])
    self.Master_CLID_str=Master_CLID_str[2:len(Master_CLID_str) - 1]

def testing_int_instance(self):
    # NO UNPACK LINE
    # ADD self. TO EVERY Master_* VARIABLE
    ...