定期从PostgreSQL获取数据并将其发送到ActiveMQ

Question

基本上，我在PostgreSQL中有一个数据表。 我想获取所有数据，并将其发送为ActiveMQ消息（表中1行有1条消息）。

但是数据表每5秒自动更新一次。 因此，表中每隔5秒就会有一些新数据。

如何在不重复发送已发送数据的情况下发送新数据？ 而且我认为我必须使用Thread.sleep（5000）使其无休止地循环吗？

这就是我得到的一切：

package testcode;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.JMSException;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
import org.apache.activemq.ActiveMQConnection;
import org.apache.activemq.ActiveMQConnectionFactory;


public class ProducerClear2 {

    public static String vardbserver;
    public static String vardbuser;
    public static String vardbpassword;
    public static String vardbname;

    public static void main (String[] args){
        vardbserver = "TestDBtoMQ";
        vardbuser = "postgresql";
        vardbpassword = "admin";

        ConnectionFactory factory = null;
        javax.jms.Connection connection = null;
        Session session = null;
        Destination destination = null;
        MessageProducer producer = null;
        Connection c = null;
        Statement stmt = null;
        try {
            Class.forName("org.postgresql.Driver");
            c = DriverManager.getConnection("jdbc:postgresql://localhost:5432/"+vardbserver, vardbuser, vardbpassword);
            c.setAutoCommit(false);
            System.out.println("----------------------------");

            stmt = c.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * FROM MESSAGES;");
            while (rs.next()) {
                String  message = rs.getString("MESSAGE");
                System.out.println("Message = " + message);
                try {
                    factory = new ActiveMQConnectionFactory(ActiveMQConnection.DEFAULT_BROKER_URL);
                    connection = factory.createConnection();
                    connection.start();
                    session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
                    destination = session.createQueue("TestQueue");
                    producer = session.createProducer(destination);
                    TextMessage mssg = session.createTextMessage(message);
                    System.out.println("Sent: " + mssg.getText());
                    producer.send(mssg);
                }
                catch (JMSException e) {
                    e.printStackTrace();
                }
            }

            rs.close();
            stmt.close();
            c.close();

        }
        catch (Exception e) {
            System.err.println(e.getClass().getName()+": "+ e.getMessage());
        }finally {
            if (session != null) {
                try {
                    session.close();
                } catch (JMSException ex) {
                    // ignore
                }
            }
            if (connection != null) {
                try {
                    connection.close();
                } catch (JMSException ex) {
                    // ignore
                }
            }
        }
        System.out.println("----------------------------");
        System.out.println("Message sent successfully");
    }
}

答案在这里检查

Answer 1

为防止消息重复，有一些可能的方法，例如：

1. ActiveMQ支持重复消息检测 ，您可以利用此功能。
2. 使用字段（布尔）表示该行是否发送。

关于这5秒钟，您可以休眠或使用调度程序来执行此操作，例如： Spring Scheduler ， Quartz

Answer 2

为了仅从postgres中获取新数据，您可以在存储特定行的事务ID的每一行上使用xmin系统列。

select xmin::varchar::bigint as xrow, * from messages;

然后，您可以跟踪上一次获取的交易，因此下次运行查询时，可以将该值用作限制

select xmin::varchar::bigint as xrow, * from messages 
    where xmin::varchar::bigint > :last_transaction_id_from_previous_run;

如果您使用的是Postgres 9.4或更高版本，则还可以使用逻辑复制来直接从数据库中流式传输数据。

有关逻辑复制的更多信息： https : //www.postgresql.org/docs/current/static/logicaldecoding.html

更新：

问题中的示例代码可以这样重写，为简洁起见，我排除了第一个try语句之外的所有内容

// Moved all the setup for the ActiveMQ connection outside the loops
// Otherwise they would create a new connection/sessions for each iteration without being closed
factory = new ActiveMQConnectionFactory(ActiveMQConnection.DEFAULT_BROKER_URL);
connection = factory.createConnection();
connection.start();
session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE);
destination = session.createQueue("TestQueue");
producer = session.createProducer(destination);

Class.forName("org.postgresql.Driver");
System.out.println("----------------------------");

// Made a try-with-resources block in order to auto close everything in case of failure, no need to remember to close them manually.
// Using prepared statements so that the query itself doesn't have to be parsed for each iteration
try (Connection c = DriverManager.getConnection("jdbc:postgresql://localhost:5432/" + vardbserver, vardbuser, vardbpassword);
     PreparedStatement stmt = c.prepareStatement("SELECT * FROM MESSAGES where xmin::varchar::bigint > ? and xmin::varchar::bigint < ? ");
     PreparedStatement max = c.prepareStatement("select max(xmin::varchar::bigint) as txid from messages")
) {
    c.setAutoCommit(false);

    // Keep track of the previous and next transaction id in order to provide windowed results
    Long previousTxId = 0L;
    Long nextTxId = 0L;

    while (true) { // Enter an infinite loop
        stmt.clearParameters(); // Remove any set parameters in the fetch query

        // Fetch the next transaction id by executing the prepared statement in max
        try (ResultSet rs = max.executeQuery()) {
            if (rs.next()) {
                nextTxId = rs.getLong(1);
            }
        }

        // Set the window in the messages query. 
        // Since the query has already been prepared, we only need to set the parameters
        stmt.setLong(1, previousTxId);
        stmt.setLong(2, nextTxId + 1);

        // Execute the statment fetching all messages that were created between previousTxId and nextTxId
        try (ResultSet rs = stmt.executeQuery()) {
            while (rs.next()) {
                String message = rs.getString("MESSAGE");
                System.out.println("Message = " + message);
                TextMessage mssg = session.createTextMessage(message);
                System.out.println("Sent: " + mssg.getText());
                producer.send(mssg);
            }
            // Update previousTxId for the next iteration
            previousTxId = nextTxId; 
        }
        Thread.sleep(5000);
    }
}