Is Teradata CLOB batch processing useless with JDBC?

Question

I think I know the answer to this question but I also want to confirm it with the experts here. I think the answer is: "Yes, because the batch size limit is 16, which is too little. So practically speaking batch processing is useless with Teradata CLOB."

Here is my reasoning. Here is the working Java code. I copy a table from one database connection to another using streaming

public class TestClob {

public void test() throws ClassNotFoundException, SQLException, IOException { 

Connection conn1, conn2; 
conn1 = DriverManager.getConnection(..., user, pass);
conn2 = DriverManager.getConnection(..., user, pass); 

Statement select = conn1.createStatement(); 
ResultSet rs = select.executeQuery("SELECT TOP 100 myClob FROM myTab " );

int totalRowNumber = 0; 

PreparedStatement ps = null; 
Clob clob = null; 
Reader clobReader = null; 

while (rs.next()) { 
totalRowNumber++; 
System.out.println(totalRowNumber);
clob = rs.getClob(1); 
clobReader = clob.getCharacterStream(); 
ps = conn2.prepareStatement("INSERT INTO myTab2 (myClob2) values (?) ");
ps.setCharacterStream(1, clobReader , clob.length() ); 
ps.execute(); // HERE I just execute the current row 
clob.free(); //  FREE the CLOB and READER objects 
clobReader.close(); 
} 

conn2.commit(); 
ps.close(); 
select.close(); 
rs.close(); 

Based on Teradata rules, I cannot have more than 16 object related to LOB open simultaneously.

Therefore I have to make sure that Clob clob and Reader clobReader are freed and closed respectively.

So I have two options

1) do the executeBatch() method and have up to 16 Clob clob and Reader clobReader objects at a time.

2) do the execute() method and close Clob clob and Reader clobReader objects right after that.

The conclusion: Teradata CLOB batch insert is useless with JDBC. One cannot set a batch size of more than 16 when trying to INSERT a Clob

Please help me and let me know if I understand this correctly

I don't see any other ways

Answer 1

You can find here attached an example of batch insert of more than 16 Clobs.

 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
 import java.security.GeneralSecurityException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;


 public class ClobBatch {

    public static void main(String[] args) throws GeneralSecurityException, IOException, SQLException {

        String databaseCredentials = ExternalData.getCredentials();
        Connection c1=DriverManager.getConnection(databaseCredentials);
        Connection c2=DriverManager.getConnection(databaseCredentials);

        String sql="create volatile table clob_test_input ( id bigint, longobj clob) no primary index on commit preserve rows;";
        Statement s=c1.createStatement();
        s.execute(sql);

        String sql2="create volatile table clob_test_target ( id bigint, longobj clob) no primary index on commit preserve rows;";
        Statement s2=c2.createStatement();
        s2.execute(sql2);

        System.out.println("Inserting test data");
        PreparedStatement ps=c1.prepareStatement("insert into clob_test_input (id, longobj) values (?,?);"); 
        for(int i=0; i<1000; i++) {
            String st=randomLargeString();
            ps.setInt(1, i);
            ps.setCharacterStream(2, new BufferedReader(new StringReader(st)), st.length());
            ps.addBatch();
        }
        ps.executeBatch();

        System.out.println("reading test data from input table");
        Statement select=c1.createStatement();
        ResultSet rs=select.executeQuery("select * from clob_test_input");


        PreparedStatement ps2=c2.prepareStatement("insert into clob_test_target (id, longobj) values (?,?);"); 
        List<Reader> readerToClose=new ArrayList<Reader>(); 
        System.out.println("start batch creation");
        while(rs.next()) {
            int pos=rs.getInt("id");
            Reader rdr=new BufferedReader(rs.getCharacterStream("longobj"));

            StringBuffer buffer=new StringBuffer();
            int c=0;
            while((c=rdr.read())!=-1) {
                buffer.append((char)c);
            }
            rdr.close();
            ps2.setInt(1, pos);
            Reader strReader= new StringReader(buffer.toString());
            ps2.setCharacterStream(2, strReader,buffer.length());
            readerToClose.add(strReader);
            ps2.addBatch();
        }
        System.out.println("start batch execution");
        ps2.executeBatch();
        rs.close();
        c1.commit();
        c2.commit();

        for(Reader r:readerToClose) r.close();

        Statement selectTest=c2.createStatement();
        ResultSet rsTest=selectTest.executeQuery("select * from clob_test_target");
        System.out.println("show results");
        int i=0;
        while(rsTest.next()) {
            BufferedReader is=new BufferedReader(rsTest.getCharacterStream("longobj"));
            StringBuilder sb=new StringBuilder();
            int c=0;
            while((c=is.read())!=-1) {
                sb.append((char)c);
            }
            is.close();
            System.out.println(""+rsTest.getInt("id")+' '+sb.toString().substring(0,80));
        }

        rsTest.close();
    }


    private static String randomLargeString() {
        StringBuilder sb=new StringBuilder();
        for(int i=0;i<10000; i++) {
            sb.append((char) (64+Math.random()*20));
        }
        return sb.toString();
    }
 } 

I've worked on some optimistic hypothesis (eg 10000 chars Clobs) but the approach could be made less memory intensive by using temporary files instead of StringBuffers.

The approach is basically find some "buffer" (be it in memory or on temp files) where to keep the data from the source database, so that you can close the input ClobReader. Then you can batch insert the data from the buffer where you don't have the limitation of 16 (you still have memory limitations).