java best way to write huge data to excel file

Question

I have huge data in 2 PostgreSQL database and I want to write it to excel file, I have worked code well but the problem is it takes a long time. Also, when I select period of data more than 3 months I will an "Java heap space" error in the server because it's more the 2 million rows. I need to more efficient way to write data to excel file, also, timestamp column I need to change it to DateTime column before write to the file. I make file I server and return file path to the client to download it.

please write an example code to solve my problem.

public String exportRmsValues(String path, String query, String query2) {
    System.out.println(query);
    System.out.println(query2);
    dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    Calendar cal = Calendar.getInstance();
    System.out.println("start exportRmsValues \n" + dateFormat.format(cal.getTime()));
    Connection c = null;
    Statement st = null;
    Connection c2 = null;
    Statement st2 = null;

    XSSFWorkbook wb = new XSSFWorkbook();
    XSSFSheet sheet = wb.createSheet("RMS Data Sheet 1");
    XSSFSheet sheet2 = wb.createSheet("RMS Data Sheet 2");
    XSSFSheet sheet3 = wb.createSheet("RMS Data Sheet 3");
    //XSSFCellStyle my_style = wb.createCellStyle();
    //XSSFFont my_font = wb.createFont();
    //my_font.setBoldweight(XSSFFont.BOLDWEIGHT_BOLD);
    //my_style.setFont(my_font);

    Row rowhead = sheet.createRow(0);       
    rowhead.createCell(0).setCellValue("Va");
    rowhead.createCell(1).setCellValue("Vb");
    rowhead.createCell(2).setCellValue("Vc");
    rowhead.createCell(3).setCellValue("Ia");
    rowhead.createCell(4).setCellValue("Ib");
    rowhead.createCell(5).setCellValue("Ic");
    rowhead.createCell(6).setCellValue("datatime");

    Row rowhead2 = sheet2.createRow(0);     
    rowhead2.createCell(0).setCellValue("Va");
    rowhead2.createCell(1).setCellValue("Vb");
    rowhead2.createCell(2).setCellValue("Vc");
    rowhead2.createCell(3).setCellValue("Ia");
    rowhead2.createCell(4).setCellValue("Ib");
    rowhead2.createCell(5).setCellValue("Ic");
    rowhead2.createCell(6).setCellValue("datatime");

    Row rowhead3 = sheet3.createRow(0);     
    rowhead3.createCell(0).setCellValue("Va");
    rowhead3.createCell(1).setCellValue("Vb");
    rowhead3.createCell(2).setCellValue("Vc");
    rowhead3.createCell(3).setCellValue("Ia");
    rowhead3.createCell(4).setCellValue("Ib");
    rowhead3.createCell(5).setCellValue("Ic");
    rowhead3.createCell(6).setCellValue("datatime");

    try {
        Class.forName("org.postgresql.Driver");
        String conString = "jdbc:postgresql://" + host + ":" + port + "/" + DBName
                + "?user=" + user + "&pass=" + pass;
        String conString1 = "jdbc:postgresql://" + host + ":" + port2 + "/" + DBName2
                + "?user=" + user + "&pass=" + pass;
        c = DriverManager.getConnection(conString);
        c2 = DriverManager.getConnection(conString1);
        st = c.createStatement();
        st2 = c2.createStatement();
        String file_name = "RMS_"+dateFormat2.format(cal.getTime())+".xlsx";
        //path = "/opt/jetty/files/"+file_name;
        path = path + file_name;
        List<ResultSet> resultSets = new ArrayList<>();
        resultSets.add(st.executeQuery(query));
        resultSets.add(st2.executeQuery(query2));
        ResultSets rs = new ResultSets(resultSets);
        int index = 1; // row index 0 for columns name
        int index2 = 1;
        int index3 = 1;
        int sheetCount = 1;
        Row row = null;
        Calendar calendar = Calendar.getInstance();
        //TimeZone tz = TimeZone.getDefault();
        while (rs.next()) {
            //XSSFRow row = sheet.createRow((long) index);
            if ( sheetCount <= 1000000 ) {
                row = sheet.createRow(index);
                index++;
            }
            if ( sheetCount > 1000000 && sheetCount <= 2000000) {
                row = sheet2.createRow(index2);
                index2++;
            }
            if ( sheetCount > 2000000 && sheetCount <= 3000000) {
                row = sheet3.createRow(index3);
                index3++;
            }

            /*else {
                row = sheet2.createRow(index2);
                index2++;
                if ( sheetCount >= 2000000 ) {
                    row = sheet3.createRow(index3);
                    index3++;
                }
            }*/

            row.createCell(0).setCellValue(rs.getDoubleValues("va"));
            row.createCell(1).setCellValue(rs.getDoubleValues("vb"));
            row.createCell(2).setCellValue(rs.getDoubleValues("vc"));
            row.createCell(3).setCellValue(rs.getDoubleValues("ia"));
            row.createCell(4).setCellValue(rs.getDoubleValues("ib"));
            row.createCell(5).setCellValue(rs.getDoubleValues("ic"));
            long datatime = rs.getLongValues("datatime");
            calendar.setTimeInMillis(datatime * 1000);
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            Date currenTimeZone = (Date) calendar.getTime();
            row.createCell(6).setCellValue(sdf.format(currenTimeZone));
            sheetCount++;
        }

        FileOutputStream fileOut = new FileOutputStream(path);
        wb.write(fileOut);
        fileOut.close();
        System.out.println("Data is saved in excel file.");
        rs.close();
        st.close();
        c.close();
        st2.close();
        c2.close();
    } catch (Exception e) {
        System.out.println(e);
        e.printStackTrace();
    }
    Calendar cal1 = Calendar.getInstance();
    System.out.println("finish exportRmsValues \n" + dateFormat.format(cal1.getTime()));
    return path;
}

Answer 1

I get good way and it's fast

query = "select name, to_timestamp(datatime) from x ";
CopyManager copyManager = new CopyManager((BaseConnection) c);
File file = new File(path);
FileOutputStream fileOutputStream = new FileOutputStream(file);

//and finally execute the COPY command to the file with this method:

copyManager.copyOut("COPY (" + query + ") TO STDOUT WITH (FORMAT CSV)", fileOutputStream);

This is will write all data from the query to the CSV file.

Answer 2

You can achieve this using BCP Utility . This is nothing but a command line query which copies data returned by a query into desired format. Run the below code snippet after installing BCP utility on your computer:

public class ExcelExport {

    public static void main(String[] args) {

        String excelFileName = "excel/myExcelFile.xls";  // Other formats also supported like .csv, .xlsx etc
        String query = "The SQL Query goes here";
        String databaseName = "DatabseName";
        String ServerUrl = "DatabaseServerUrl/IP";
        String userName = "DatabaseUsername";
        String password = "DatabasePassword";
        String bcpCommand = "bcp \"" + query + "\" queryout \"" + excelFileName + "\" -c -d \"" + databaseName
                + "\" -S " + ServerUrl + " -U \"" + userName + "\" -P \"" + password + "\"";
        System.out.println("BCP Command : " + bcpCommand);

        // Executes BCP command using command line
        ProcessBuilder processBuilder = new ProcessBuilder("cmd.exe", "/c", bcpCommand);

        try {

            Process process = processBuilder.start();
            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            String line;

            while (true) {
                line = reader.readLine();

                if (line == null) {
                    break;
                }

                System.out.println(line);
            }

            System.out.println("Excel File Created !!!");

        } catch (IOException ioException) {
            ioException.printStackTrace();
            System.out.println("Failed to export data to excel !!!");
        }
    }
}

NOTE: It doesn't format excel files beautifully like Apache POI does, but when it comes to exporting millions of data. It saves great amount of time. You can do formatting manually. The code copies all the data returned by query. So keep in mind your record count should not cross the excel file row limit which is 1,048,576 rows in current version and 65536 before Office 2003. So, In your case for 2 million rows make multiple files if required.