PostgreSQL: How do you insert rows of data with a loop into a database table?

Question

How can you read & insert rows from a text file containing data in [x, y, z] form for example:

1 google com (separated by tabs)

into a database table (without having to insert every single line manually)?

I'm new at programming!

Thank you for your time and help kind sirs and madams!

import java.sql.*;
import java.util.Scanner;
import java.io.*;

public class Database {

    public static Connection connectToDatabase(String user, String port, String database) {
        System.out.println("-------- PostgreSQL " + "JDBC Connection Testing ------------");
        try {
            Class.forName("org.postgresql.Driver");
        } catch (ClassNotFoundException e) {

            System.out.println("Where is your PostgreSQL JDBC Driver? " + "Include in your library path!");
            e.printStackTrace();
        }
        System.out.println("PostgreSQL JDBC Driver Registered!");

        Connection connection = null;
        try {
            connection = DriverManager.getConnection("jdbc:postgresql://localhost:" + port + "/" + database, user,
                    "doesn't matter!");
        } catch (SQLException e) {
            System.out.println("Connection Failed! Check output console");
            e.printStackTrace();
        }
        return connection;
    }

    public static ResultSet executeSelect(Connection connection, String query) {
        Statement st = null;
        try {
            st = connection.createStatement();
        } catch (SQLException e) {
            e.printStackTrace();
            return null;
        }

        ResultSet rs = null;
        try {
            rs = st.executeQuery(query);
            st.close();
        } catch (SQLException e) {
            e.printStackTrace();
            return null;
        }

        return rs;
    }

    public static void dropTable(Connection connection, String table) {
        Statement st = null;
        try {
            st = connection.createStatement();
            st.execute("DROP TABLE " + table);
            st.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    public static void createTable(Connection connection, String tableDescription) {
        Statement st = null;
        try {
            st = connection.createStatement();
            st.execute("CREATE TABLE " + tableDescription);
            st.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    public static int insertIntoTableFromFile(Connection connection, String table, String file) {

        BufferedReader br = null;
        int numRows = 0;
        try {
            Statement st = connection.createStatement();
            String sCurrentLine, brokenLine[], composedLine = "";
            br = new BufferedReader(new FileReader(file));

            while ((sCurrentLine = br.readLine()) != null) {
                // Insert each line to the DB
                brokenLine = sCurrentLine.split("\t");
                composedLine = "INSERT INTO dotcom VALUES (";
                int i;
                for (i = 0; i < brokenLine.length - 1; i++) {
                    composedLine += "'" + brokenLine[i] + "',";
                }
                composedLine += "'" + brokenLine[i] + "')";
                numRows = st.executeUpdate(composedLine);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            try {
                if (br != null)
                    br.close();
            } catch (IOException ex) {
                ex.printStackTrace();
            }
        }
        return numRows;
    }

    public static void main(String[] argv) throws SQLException, FileNotFoundException {

        @SuppressWarnings("resource")
        Scanner input = new Scanner(System.in);
        System.out.println("Please enter your Username:");
        String user = input.next();
        System.out.println("Please enter your Port ID:");
        String port = input.next();
        String database = "test";

        Connection connection = connectToDatabase(user, port, database);

        Statement st = connection.createStatement();

        if (connection != null) {
            System.out.println("You made it, take control your database now!");
        } else {
            System.out.println("Failed to make connection!");
            return;
        }
        // Now we're ready to work on the DB

        // read TopURLs file
        try {
            BufferedReader fileReader = new BufferedReader(new FileReader("TopURLs"));

            while (fileReader.readLine() != null) {

                st.execute("DROP TABLE IF EXISTS dotcom;");
                st.execute("CREATE TABLE dotcom (rank integer PRIMARY KEY, domainName varchar(128), domainType varchar(128));");
                st.execute("INSERT INTO dotcom VALUES(1, 'google', 'com');");
                //st.execute("INSERT INTO dotcom VALUES(2, 'facebook', 'com');");
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
        }

        /*
        try {
            BufferedReader fileReader = new BufferedReader(new FileReader("TopURLs"));

            while (fileReader.readLine() != null) {

                st.execute("DROP TABLE IF EXISTS dotcom;");
                st.execute("CREATE TABLE dotcom (rank integer PRIMARY KEY, domainName varchar(128), domainType varchar(128));");
                st.execute("INSERT INTO dotcom SELECT com, domainType FROM dotcom WHERE domainType = 'com'");
            }

        } catch (IOException e) {
            e.printStackTrace();
        } finally {
        }
        */

        /*
        Statement createStatement = null;
        PreparedStatement insertStatement = null;
        try {
            BufferedReader fileReader = new BufferedReader(new FileReader("TopURLs"));
            String line = null;
            createStatement = connection.createStatement();
            createStatement.executeUpdate("DROP TABLE IF EXISTS dotcom");
            createStatement.executeUpdate("CREATE TABLE dotcom (rank integer PRIMARY KEY, domainName varchar(128), domainType varchar(128))");

            connection.setAutoCommit(false);//commit whole batch at the end
            insertStatement = connection.prepareStatement("INSERT INTO dotcom VALUES (?, ?, ?)");

            while ( (line = fileReader.readLine()) != null) {
                line = fileReader.readLine();
                String[] urls = line.split("\t");//space or any other delimiter that you're using
                insertStatement.setInt(1, Integer.parseInt(urls[0]));
                insertStatement.setString(2, urls[1]);
                insertStatement.setString(3, urls[2]);
                //insertStatement.setString(4, urls[3]);

                insertStatement.addBatch();
            }
            insertStatement.executeBatch();
            connection.commit();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if(connection != null) {
                connection.setAutoCommit(true);
            }
            if(createStatement != null) {
                createStatement.close();
            }
            if(insertStatement != null) {
                insertStatement.close();
            }
        }
        */

        // connection is of type Connection (in JDBC)
        DatabaseMetaData dbm = connection.getMetaData();

        // check if table is there
        ResultSet tables = dbm.getTables(null, null, "table name", null);
        if (tables.next()) {
            System.out.println("Table exists");
        } else {
            System.out.println("Table does not exist");
        }

        // check if view is there?
        //"create view foo as select * from table;"
        //"select * from foo;"
        ResultSet views = dbm.getTables("catalog name", null, null, null);
        if (views.next()) {
            System.out.println("View exists");
        } else {
            System.out.println("View does not exist");
        }

        String query = "SELECT * FROM internet";
        ResultSet rs = executeSelect(connection, query);
        try {
            while (rs.next()) {
                System.out.print("Column 1 returned ");
                System.out.println(rs.getString(1));
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
        rs.close();

        dropTable(connection, "dotcom");
        createTable(connection,
                "dotcom (id int primary key, name varchar(15), type varchar(15));");
        int rows = insertIntoTableFromFile(connection, "dotcom", "TopURLs");
        System.out.println(rows + " rows inserted.");
    }
}

Answer 1

This is too long for a comment.

When loading data from a text file, I generally load into a staging table that consists of character column. Then, I can do validate the data, convert to the appropriate data types, and even reparse some columns, if necessary.

Admittedly, the Postgres COPY command often works quite well just reading in the data.

You would seem to have two options:

• Load the data into a staging table and do the data cleansing there.
• Read the data line-by-line in java and do the data cleansing there.

Because my SQL skills >> my java skills, it is a no-brainer which I would choose (SQL). But these would seem to be your most reasonable options.