How to run a step in parallel mode with Spring Batch

Question

I'am working on a spring batch. I have a partitioning step (of a list of objects) and then a slave step with Reader and Writer.

I want to execute the processStep in parallel mode. So, I want to have a specific instances of Reader-Writer for each partition .

For the moment, created partitions uses same instances of Reader-Writer. So, those operations are done in serial mode: Read and write the first partition and then do the same for the next one when the first is completed.

The spring boot configuration class:

@Configuration
@Import({ DataSourceConfiguration.class})
public class BatchConfiguration {

    private final static int COMMIT_INTERVAL = 1;

    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    @Autowired
    private StepBuilderFactory stepBuilderFactory;

   @Autowired
   @Qualifier(value="mySqlDataSource")
   private DataSource mySqlDataSource;

   public static int GRID_SIZE = 3;

   public static List<Pojo> myList;

   @Bean
   public Job myJob() throws UnexpectedInputException, ParseException, NonTransientResourceException, Exception {

      return jobBuilderFactory.get("myJob")
            .incrementer(new RunIdIncrementer())
            .start(partitioningStep())
            .build();
  }

  @Bean(name="partitionner")
  public MyPartitionner partitioner() {

    return new MyPartitionner();
  }

  @Bean
  public SimpleAsyncTaskExecutor taskExecutor() {

    SimpleAsyncTaskExecutor taskExecutor = new SimpleAsyncTaskExecutor();
    taskExecutor.setConcurrencyLimit(GRID_SIZE);
    return taskExecutor;
  }

  @Bean
  public Step partitioningStep() throws NonTransientResourceException, Exception {

    return stepBuilderFactory.get("partitioningStep")
              .partitioner("processStep", partitioner())
              .step(processStep())
              .taskExecutor(taskExecutor())
              .build();
  }

  @Bean
  public Step processStep() throws UnexpectedInputException, ParseException, NonTransientResourceException, Exception {

    return stepBuilderFactory.get("processStep")
            .<List<Pojo>, List<Pojo>> chunk(COMMIT_INTERVAL)
            .reader(processReader())
            .writer(processWriter())
            .taskExecutor(taskExecutor())
            .build();
  }

  @Bean
  public ProcessReader processReader() throws UnexpectedInputException, ParseException, NonTransientResourceException, Exception {

    return new ProcessReader();
  }

  @Bean
  public ProcessWriter processWriter() {

    return new ProcessWriter();
  }
}

The partitionner class

public class MyPartitionner implements Partitioner{

@Autowired
private IService service;

@Override
public Map<String, ExecutionContext> partition(int gridSize) {

    // list of 300 object partitionned like bellow
    ...
    Map<String, ExecutionContext> partitionData = new HashMap<String, ExecutionContext>();

    ExecutionContext executionContext0 = new ExecutionContext();
    executionContext0.putString("from", Integer.toString(0));
    executionContext0.putString("to", Integer.toString(100));
    partitionData.put("Partition0", executionContext0);

    ExecutionContext executionContext1 = new ExecutionContext();
    executionContext1.putString("from", Integer.toString(101));
    executionContext1.putString("to", Integer.toString(200));
    partitionData.put("Partition1", executionContext1);

    ExecutionContext executionContext2 = new ExecutionContext();
    executionContext2.putString("from", Integer.toString(201));
    executionContext2.putString("to", Integer.toString(299));
    partitionData.put("Partition2", executionContext2);

    return partitionData;
 }
}

The Reader class

    public class ProcessReader implements ItemReader<List<Pojo>>, ChunkListener {

    @Autowired
    private IService service;

    private StepExecution stepExecution;

    private static List<String> processedIntervals = new ArrayList<String>();

    @Override
    public List<Pojo> read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {

        System.out.println("Instance reference: "+this.toString());

        if(stepExecution.getExecutionContext().containsKey("from") && stepExecution.getExecutionContext().containsKey("to")){

            Integer from = Integer.valueOf(stepExecution.getExecutionContext().get("from").toString());
            Integer to = Integer.valueOf(stepExecution.getExecutionContext().get("to").toString());

            if(from != null && to != null && !processedIntervals.contains(from + "" + to) && to < BatchConfiguration.myList.size()){
                processedIntervals.add(String.valueOf(from + "" + to));
                return BatchConfiguration.myList.subList(from, to);
            }
        }

        return null;
    }

    @Override
    public void beforeChunk(ChunkContext context) {

        this.stepExecution = context.getStepContext().getStepExecution();
    }

    @Override
    public void afterChunk(ChunkContext context) { }

    @Override
    public void afterChunkError(ChunkContext context) { }

    }
  }

The writer class

 public class ProcessWriter implements ItemWriter<List<Pojo>>{

    private final static Logger LOGGER = LoggerFactory.getLogger(ProcessWriter.class);

    @Autowired
    private IService service;

    @Override
    public void write(List<? extends List<Pojo>> pojos) throws Exception {

        if(!pojos.isEmpty()){
            for(Pojo item : pojos.get(0)){
                try {
                    service.remove(item.getId());
                } catch (Exception e) {
                    LOGGER.error("Error occured while removing the item [" + item.getId() + "]", e);
                }
            }
        }
    }
 }

Can you please tell me what is wrong with my code?

Answer 1

Resolved by adding @StepScope to my reader and writer beans declaration:

@Configuration
@Import({ DataSourceConfiguration.class})
public class BatchConfiguration {

   ...

   @Bean
   @StepScope
    public ProcessReader processReader() throws UnexpectedInputException, ParseException, NonTransientResourceException, Exception {

      return new ProcessReader();
   }

   @Bean
   @StepScope
   public ProcessWriter processWriter() {

     return new ProcessWriter();
   }

   ...

}

By this way, you I have an different instance of the chunck (Reader-Writer) for each partition.