MQ Source in Spark Structured Streaming

Question

I have implemented the MQ source in Spark structured streaming. And I am using IBM MQ Core library and using Java 8

            <groupId>com.ibm.mq</groupId>
            <artifactId>com.ibm.mq</artifactId>
            <version>${mq.version}</version>
            <scope>provided</scope>

The issue is reading operation is too slow. I am running my streaming application with 4 executors, 4GB memory each and with 4 Cores. But my application takes 10 minutes to read 1000 messages from Queue. Each message size is around 3kb.

As of now my spark application reading the message one by one and processing the messages in parallel.

Is there a way we can read this messages in parallel as well. I have heard concurrent consumers in mq. But I couldnt get any proper documentation for that.

Can anyone share me some examples in Java?

I cant share any code because of my organizational restrictions. sorry for that. But I can give any information needed to provide some help.

Answer 1

But my application takes 10 minutes to read 1000 messages from Queue. Each message size is around 3kb.

Are you running your application on the same server as MQ queue manager or on a remote server? Is your application doing some backend work before looping to get the next message? What network performance testing have you done? Maybe you have a slow network.

Below is a simple Java/MQ that loops and retrieves all messages on the queue. I just put 1000 3KB messages on a queue and ran it. Here is the output:

2018/06/28 17:18:21.191 MQTest12L: testReceive: successfully connected to MQWT1
2018/06/28 17:18:21.207 MQTest12L: testReceive: successfully opened TEST.Q1
2018/06/28 17:18:21.448 MQTest12L: testReceive: read 1000 messages
2018/06/28 17:18:21.448 MQTest12L: testReceive: closed: TEST.Q1
2018/06/28 17:18:21.450 MQTest12L: testReceive: disconnected from MQWT1

To get 1000 3KB messages took 259 milliseconds . So, if your program takes 10 minutes to get 1000 3kb messages then there is something wrong with either your program or your network.

Here is the fully-functioning Java/MQ application called MQTest12L.java that will loop and retrieve all messages on the queue:

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Hashtable;

import com.ibm.mq.MQException;
import com.ibm.mq.MQMessage;
import com.ibm.mq.MQGetMessageOptions;
import com.ibm.mq.MQQueue;
import com.ibm.mq.MQQueueManager;
import com.ibm.mq.constants.CMQC;

/**
 * Program Name
 *  MQTest12L
 *
 * Description
 *  This java class will connect to a remote queue manager with the 
 *  MQ setting stored in a HashTable, loop to retrieve all messages on a queue
 *  then close and disconnect.
 *
 * Sample Command Line Parameters
 *  -m MQA1 -h 127.0.0.1 -p 1414 -c TEST.CHL -q TEST.Q1 -u UserID -x Password
 *
 * @author Roger Lacroix
 */
public class MQTest12L
{
   private static final SimpleDateFormat  lOGGER_TIMESTAMP = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS");

   private Hashtable<String,String> params;
   private Hashtable<String,Object> mqht;
   private String qMgrName;
   private String outputQName;

   /**
    * The constructor
    */
   public MQTest12L()
   {
      super();
      params = new Hashtable<String,String>();
      mqht = new Hashtable<String,Object>();
   }

   /**
    * Make sure the required parameters are present.
    * @return true/false
    */
   private boolean allParamsPresent()
   {
      boolean b = params.containsKey("-h") && params.containsKey("-p") &&
                  params.containsKey("-c") && params.containsKey("-m") &&
                  params.containsKey("-q") &&
                  params.containsKey("-u") && params.containsKey("-x");
      if (b)
      {
         try
         {
            Integer.parseInt((String) params.get("-p"));
         }
         catch (NumberFormatException e)
         {
            b = false;
         }
      }

      return b;
   }

   /**
    * Extract the command-line parameters and initialize the MQ HashTable.
    * @param args
    * @throws IllegalArgumentException
    */
   private void init(String[] args) throws IllegalArgumentException
   {
      int port = 1414;
      if (args.length > 0 && (args.length % 2) == 0)
      {
         for (int i = 0; i < args.length; i += 2)
         {
            params.put(args[i], args[i + 1]);
         }
      }
      else
      {
         throw new IllegalArgumentException();
      }

      if (allParamsPresent())
      {
         qMgrName = (String) params.get("-m");
         outputQName = (String) params.get("-q");

         try
         {
            port = Integer.parseInt((String) params.get("-p"));
         }
         catch (NumberFormatException e)
         {
            port = 1414;
         }

         mqht.put(CMQC.CHANNEL_PROPERTY, params.get("-c"));
         mqht.put(CMQC.HOST_NAME_PROPERTY, params.get("-h"));
         mqht.put(CMQC.PORT_PROPERTY, new Integer(port));
         mqht.put(CMQC.USER_ID_PROPERTY, params.get("-u"));
         mqht.put(CMQC.PASSWORD_PROPERTY, params.get("-x"));

         // I don't want to see MQ exceptions at the console.
         MQException.log = null;
      }
      else
      {
         throw new IllegalArgumentException();
      }
   }

   /**
    * Connect, open queue, loop and get all messages then close queue and disconnect.
    *
    * @throws MQException
    */
   private void testReceive()
   {
      MQQueueManager qMgr = null;
      MQQueue queue = null;
      int openOptions = CMQC.MQOO_INPUT_AS_Q_DEF + CMQC.MQOO_INQUIRE + CMQC.MQOO_FAIL_IF_QUIESCING;
      MQGetMessageOptions gmo = new MQGetMessageOptions();
      gmo.options = CMQC.MQGMO_NO_WAIT + CMQC.MQGMO_FAIL_IF_QUIESCING;
      MQMessage receiveMsg = null;
      int msgCount = 0;
      boolean getMore = true;

      try
      {
         qMgr = new MQQueueManager(qMgrName, mqht);
         MQTest12L.logger("successfully connected to "+ qMgrName);

         queue = qMgr.accessQueue(outputQName, openOptions);
         MQTest12L.logger("successfully opened "+ outputQName);

         while(getMore)
         {
            receiveMsg = new MQMessage();

            try
            {
               // get the message on the queue
               queue.get(receiveMsg, gmo);
               msgCount++;

               if (CMQC.MQFMT_STRING.equals(receiveMsg.format))
               {
                  String msgStr = receiveMsg.readStringOfByteLength(receiveMsg.getMessageLength());
//                  MQTest12L.logger("["+msgCount+"] " + msgStr);
               }
               else
               {
                  byte[] b = new byte[receiveMsg.getMessageLength()];
                  receiveMsg.readFully(b);
//                  MQTest12L.logger("["+msgCount+"] " + new String(b));
               }
            }
            catch (MQException e)
            {
               if ( (e.completionCode == CMQC.MQCC_FAILED) && 
                    (e.reasonCode == CMQC.MQRC_NO_MSG_AVAILABLE) )
               {
                  // All messages read.
                  getMore = false;
                  break;
               }
               else
               {
                  MQTest12L.logger("MQException: " + e.getLocalizedMessage());
                  MQTest12L.logger("CC=" + e.completionCode + " : RC=" + e.reasonCode);
                  getMore = false;
                  break;
               }
            }
            catch (IOException e)
            {
               MQTest12L.logger("IOException:" +e.getLocalizedMessage());
            }
         }
      }
      catch (MQException e)
      {
         MQTest12L.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
      }
      finally
      {
         MQTest12L.logger("read " + msgCount + " messages");

         try
         {
            if (queue != null)
            {
               queue.close();
               MQTest12L.logger("closed: "+ outputQName);
            }
         }
         catch (MQException e)
         {
            MQTest12L.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
         }
         try
         {
            if (qMgr != null)
            {
               qMgr.disconnect();
               MQTest12L.logger("disconnected from "+ qMgrName);
            }
         }
         catch (MQException e)
         {
            MQTest12L.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
         }
      }
   }

   /**
    * A simple logger method
    * @param data
    */
   public static void logger(String data)
   {
      String className = Thread.currentThread().getStackTrace()[2].getClassName();

      // Remove the package info.
      if ( (className != null) && (className.lastIndexOf('.') != -1) )
         className = className.substring(className.lastIndexOf('.')+1);

      System.out.println(lOGGER_TIMESTAMP.format(new Date())+" "+className+": "+Thread.currentThread().getStackTrace()[2].getMethodName()+": "+data);
   }

   /**
    * main line
    * @param args
    */
   public static void main(String[] args)
   {
      MQTest12L write = new MQTest12L();

      try
      {
         write.init(args);
         write.testReceive();
      }
      catch (IllegalArgumentException e)
      {
         System.err.println("Usage: java MQTest12L -m QueueManagerName -h host -p port -c channel -q QueueName -u UserID -x Password");
         System.exit(1);
      }

      System.exit(0);
   }
}

Answer 2

Based on your comments to my other post, I take it you are a newbie to MQ. You should be doing MQ only testing and Kafka only testing. This will help you to determine where any bottlenecks exist.

In case you didn't know, there is a really cheap conference called MQ Technical Conference . It has lots of sessions on IBM MQ plus sessions on both Kafka and MQTT. It is a cheap way to get some education!!

Ok, back to testing MQ. The MQTest12L program I posted can retrieve all messages on a queue. So, to create the other half of the test harness, I took one of my other programs that puts messages to a queue and simply altered it to put 1000 3KB messages. It is called MQTest10003KB.java (the name says it all).

I don't know if your messages are persistent or non-persistent. I set the code for non-persistent but just flip the comment in the code.

So, you run MQTest10003KB program first then run MQTest12L program. You will get the time for putting 1000 3KB messages and then the time for getting 1000 3KB messages.

Here are the results from my run:

2018/06/29 17:42:31.355 MQTest10003KB: testSend: successfully connected to MQWT1
2018/06/29 17:42:31.366 MQTest10003KB: testSend: successfully opened TEST.Q1
2018/06/29 17:42:31.552 MQTest10003KB: testSend: 1000 messages written to queue.
2018/06/29 17:42:31.552 MQTest10003KB: testSend: closed: TEST.Q1
2018/06/29 17:42:31.554 MQTest10003KB: testSend: disconnected from MQWT1

It took MQTest10003KB 199 milliseconds to put 1000 3KB non-persistent messages to a queue.

I don't know Kafka but you would need to do a similar thing then report your finding to your team and manager to determine if the project is feasible.

Here is the code for MQTest10003KB:

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Hashtable;

import com.ibm.mq.MQException;
import com.ibm.mq.MQMessage;
import com.ibm.mq.MQPutMessageOptions;
import com.ibm.mq.MQQueue;
import com.ibm.mq.MQQueueManager;
import com.ibm.mq.constants.CMQC;

/**
 * Program Name
 *  MQTest10003KB
 *
 * Description
 *  This java class will connect to a remote queue manager with the
 *  MQ setting stored in a HashTable and put 1000 3KB messages on a queue.
 *
 * Sample Command Line Parameters
 *  -m MQA1 -h 127.0.0.1 -p 1414 -c TEST.CHL -q TEST.Q1 -u UserID -x Password
 *
 * @author Roger Lacroix
 */
public class MQTest10003KB
{
   private static final SimpleDateFormat  lOGGER_TIMESTAMP = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS");

   private Hashtable<String,String> params;
   private Hashtable<String,Object> mqht;
   private String qMgrName;
   private String outputQName;

   /**
    * The constructor
    */
   public MQTest10003KB()
   {
      super();
      params = new Hashtable<String,String>();
      mqht = new Hashtable<String,Object>();
   }

   /**
    * Make sure the required parameters are present.
    * @return true/false
    */
   private boolean allParamsPresent()
   {
      boolean b = params.containsKey("-h") && params.containsKey("-p") &&
                  params.containsKey("-c") && params.containsKey("-m") &&
                  params.containsKey("-q") &&
                  params.containsKey("-u") && params.containsKey("-x");
      if (b)
      {
         try
         {
            Integer.parseInt((String) params.get("-p"));
         }
         catch (NumberFormatException e)
         {
            b = false;
         }
      }

      return b;
   }

   /**
    * Extract the command-line parameters and initialize the MQ HashTable.
    * @param args
    * @throws IllegalArgumentException
    */
   private void init(String[] args) throws IllegalArgumentException
   {
      int port = 1414;
      if (args.length > 0 && (args.length % 2) == 0)
      {
         for (int i = 0; i < args.length; i += 2)
         {
            params.put(args[i], args[i + 1]);
         }
      }
      else
      {
         throw new IllegalArgumentException();
      }

      if (allParamsPresent())
      {
         qMgrName = (String) params.get("-m");
         outputQName = (String) params.get("-q");

         try
         {
            port = Integer.parseInt((String) params.get("-p"));
         }
         catch (NumberFormatException e)
         {
            port = 1414;
         }

         mqht.put(CMQC.CHANNEL_PROPERTY, params.get("-c"));
         mqht.put(CMQC.HOST_NAME_PROPERTY, params.get("-h"));
         mqht.put(CMQC.PORT_PROPERTY, new Integer(port));
         mqht.put(CMQC.USER_ID_PROPERTY, params.get("-u"));
         mqht.put(CMQC.PASSWORD_PROPERTY, params.get("-x"));

         // I don't want to see MQ exceptions at the console.
         MQException.log = null;
      }
      else
      {
         throw new IllegalArgumentException();
      }
   }

   /**
    * Connect, open queue, write 1000 3KB messages, close queue and disconnect.
    *
    */
   private void testSend()
   {
      final String MSG_DATA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+-={}[]:;'<>?,./";
      final int requiredMsgSize = 3072;  // 3KB
      final int requiredNumOfMsgs = 1000;
      MQQueueManager qMgr = null;
      MQQueue queue = null;
      int openOptions = CMQC.MQOO_OUTPUT + CMQC.MQOO_FAIL_IF_QUIESCING;
      MQMessage sendmsg = null;
      int msgCount = 0;

      /**
       * Create a buffer that is 3KB (3072) in size
       */
      StringBuffer sb = new StringBuffer();
      for (int i=0; i < (requiredMsgSize/MSG_DATA.length()); i++)
         sb.append(MSG_DATA);

      // Add remaining characters to make it a 3KB message
      if (sb.length() < requiredMsgSize)
         sb.append(MSG_DATA.substring(0, (requiredMsgSize - sb.length())));

      try
      {
         qMgr = new MQQueueManager(qMgrName, mqht);
         MQTest10003KB.logger("successfully connected to "+ qMgrName);

         queue = qMgr.accessQueue(outputQName, openOptions);
         MQTest10003KB.logger("successfully opened "+ outputQName);

         /** 
          * Put 1000 3KB messages on the queue
          */
         for (int i=0; i < requiredNumOfMsgs; i++)
         {
            sendmsg = new MQMessage();
            sendmsg.format = CMQC.MQFMT_STRING;
            sendmsg.persistence = CMQC.MQPER_NOT_PERSISTENT;
//            sendmsg.persistence = CMQC.MQPER_PERSISTENT;

            // Write message data
            sendmsg.writeString(sb.toString());
            // put the message on the queue
            queue.put(sendmsg, new MQPutMessageOptions());
            msgCount++;
         }
      }
      catch (MQException e)
      {
         MQTest10003KB.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
      }
      catch (IOException e)
      {
         MQTest10003KB.logger("IOException:" +e.getLocalizedMessage());
      }
      finally
      {
         MQTest10003KB.logger(msgCount+" messages written to queue.");

         try
         {
            if (queue != null)
            {
               queue.close();
               MQTest10003KB.logger("closed: "+ outputQName);
            }
         }
         catch (MQException e)
         {
            MQTest10003KB.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
         }
         try
         {
            if (qMgr != null)
            {
               qMgr.disconnect();
               MQTest10003KB.logger("disconnected from "+ qMgrName);
            }
         }
         catch (MQException e)
         {
            MQTest10003KB.logger("CC=" +e.completionCode + " : RC=" + e.reasonCode);
         }
      }
   }

   /**
    * A simple logger method
    * @param data
    */
   public static void logger(String data)
   {
      String className = Thread.currentThread().getStackTrace()[2].getClassName();

      // Remove the package info.
      if ( (className != null) && (className.lastIndexOf('.') != -1) )
         className = className.substring(className.lastIndexOf('.')+1);

      System.out.println(lOGGER_TIMESTAMP.format(new Date())+" "+className+": "+Thread.currentThread().getStackTrace()[2].getMethodName()+": "+data);
   }

   /**
    * main line
    * @param args
    */
   public static void main(String[] args)
   {
      MQTest10003KB write = new MQTest10003KB();

      try
      {
         write.init(args);
         write.testSend();
      }
      catch (IllegalArgumentException e)
      {
         MQTest10003KB.logger("Usage: java MQTest10003KB -m QueueManagerName -h host -p port -c channel -q QueueName -u UserID -x Password");
         System.exit(1);
      }

      System.exit(0);
   }
}