Python: Print standard output to multiple files after a set number of rows

Question

I have a database table that has 70 million records and another table that has 9 million records. I am trying to have my program print out the joined tables into multiple files after the loop has reached 1,000,000 records and continue to do so until the whole query dump finishes. Provided is my code:

from netaddr import *
import sys, csv, sqlite3, logging, time, os, errno

# Functions
s = time.strftime('%Y%m%d%H%M%S')

# Create file from standard output for database import
class Logger(object):
    def __init__(self):
        self.terminal = sys.stdout
        self.log = open((s) + "_" + sys.argv[1], "a")

    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)

def ExportVNEDeltas():
    sys.stdout = Logger() # Start screen capture to log file

    con = sqlite3.connect(sys.argv[1])
    cur = con.cursor()

    try:
        cur.execute('SELECT tbl70m.IP, tbl9m.ip, tbl0m.Cgroup, \
            TOTAL(case when tbl9m.vne = "internal" then 1 end), \
            TOTAL(case when tbl9m.vne = "external" then 1 end), \
            TOTAL(case when tbl9m.vne = "enterprise" then 1 end), \
            TOTAL(case when tbl9m.vne = "zone" then 1 end) \
            FROM tbl70m LEFT OUTER JOIN tbl9m ON tbl70m.IP=tbl9m.ip \
            GROUP BY tbl70m.IP \
            ORDER BY tbl70m.Cgroup, tbl70m.IP;')

        data = cur.fetchall()

        for row in data:
            print '"'+str(row[0])+'","'+str(row[1])+'","'+str(row[2])+'","'+str(row[3])+'","'+str(row[4])+'","'+str(row[5])+'","'+str(row[6])+'"'

    except (KeyboardInterrupt, SystemExit):
        raise

    con.close()
    sys.stdout = sys.__stdout__ # stops capturing data from database export.

# Primary function to execute
def main():
    ExportVNEDeltas()

if __name__=='__main__':
    main()

I cannot seem to figure out how to stop printing standard output once I reach 1,000,000 records and create a new file. The reason I am needing a hard stop on the 1,000,000 is so we can view this data in Microsoft Excel.

Answer 1

You could add this rule to your Logger :

class Logger(object):
    def __init__(self, maxlines=1000000):
        self.maxlines = maxlines
        self.lines = 0
        self.terminal = sys.stdout
        self.log = open((s) + "_" + sys.argv[1], "a")

    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)
        self.lines += 1
        if self.lines == self.maxlines:
            self.log.close()
            self.log = open(...)
            self.lines = 0