從slaveStep不調用Spring Batch JDBCPagingItemReader

Question

我創建了一個POC項目，其中使用Spring批處理本地分區步驟將Employee表10的記錄移到NewEmployee表。 我已經配置了4個線程來運行此批處理過程。 當我運行此批處理過程時，我看到從屬步驟未調用pageingItemReader（）方法。由於未調用此OraclePagingQueryProvider。 我注意到丟失（未移動）的記錄數等於配置的線程數。 我已經從以下鏈接獲得了指導，開發了該POC：-https: //github.com/mminella/LearningSpringBatch/tree/master/src/localPartitioning

請注意，當我用不涉及多線程的普通讀取，處理和寫入邏輯替換主代碼和從代碼時，以下代碼可以正常工作。

DB中的BATCH_STEP_EXECUTION表還顯示僅移動了8條記錄（此處再次丟失2條記錄，這等於線程數）。 DB Record如下：

STEP_NAME狀態COMMIT_COUNT READ_COUNT WRITE_COUNT EXIT_CODE從屬步驟：partition1已完成1 4 4已完成slaveStep：partition0已完成1 4 4已完成masterStep已完成2 8 8已完成

Configuration類的代碼段

         @Bean
                public JobRegistryBeanPostProcessor jobRegistrar() throws Exception{
                    JobRegistryBeanPostProcessor registrar=new JobRegistryBeanPostProcessor();
                    registrar.setJobRegistry(this.jobRegistry);
                    registrar.setBeanFactory(this.applicationContext.getAutowireCapableBeanFactory());
                    registrar.afterPropertiesSet();
                    return registrar;
                }

                @Bean
                public JobOperator jobOperator() throws Exception{
                    SimpleJobOperator simpleJobOperator=new SimpleJobOperator();
                    simpleJobOperator.setJobLauncher(this.jobLauncher);
                    simpleJobOperator.setJobParametersConverter(new DefaultJobParametersConverter());
                    simpleJobOperator.setJobRepository(this.jobRepository);
                    simpleJobOperator.setJobExplorer(this.jobExplorer);
                    simpleJobOperator.setJobRegistry(this.jobRegistry);

                    simpleJobOperator.afterPropertiesSet();
                    return simpleJobOperator;

                }

                @Bean
                public ColumnRangePartitioner partitioner() {
                    ColumnRangePartitioner partitioner = new ColumnRangePartitioner();
                    partitioner.setColumn("id");
                    partitioner.setDataSource(this.dataSource);
                    partitioner.setTable("Employee");
                    LOGGER.info("partitioner---->"+partitioner);
                    return partitioner;
                }

                @Bean
                public Step masterStep() {
                    return stepBuilderFactory.get("masterStep")
                            .partitioner(slaveStep().getName(), partitioner())
                            .step(slaveStep())
                            .gridSize(gridSize)
                            .taskExecutor(taskExecutorConfiguration.taskExecutor())
                            .build();
                }

                @Bean
                public Step slaveStep() {
                    return stepBuilderFactory.get("slaveStep")
                            .<Employee, NewEmployee>chunk(chunkSize)
                            .reader(pagingItemReader(null,null))
                            .processor(employeeProcessor())
                            .writer(employeeWriter.customItemWriter())
                            .build();
                }

                @Bean
                public Job job() {
                    return jobBuilderFactory.get("FR")
                            .start(masterStep())
                            .build();
                }

                @Bean
                public ItemProcessor<Employee, NewEmployee> employeeProcessor() {
                    return new EmployeeProcessor();
                }

                @Override
                public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
                    this.applicationContext=applicationContext;
                }


                 */

                @Bean
                @StepScope
                public JdbcPagingItemReader<Employee> pagingItemReader(@Value("#{stepExecutionContext['minValue']}") Long minvalue,
                        @Value("#{stepExecutionContext['maxValue']}") Long maxvalue) {

                    JdbcPagingItemReader<Employee> reader = new JdbcPagingItemReader<Employee>();
                    reader.setDataSource(this.dataSource);
                    // this should be equal to chunk size for the performance reasons.
                    reader.setFetchSize(chunkSize);
                    reader.setRowMapper((resultSet, i) -> {
                        return new Employee(resultSet.getLong("id"), 
                                resultSet.getString("firstName"),
                                resultSet.getString("lastName"));
                    });

                    OraclePagingQueryProvider provider = new OraclePagingQueryProvider();
                    provider.setSelectClause("id, firstName, lastName");
                    provider.setFromClause("from Employee");
                    LOGGER.info("min-->"+minvalue);
                    LOGGER.info("max-->"+maxvalue);
                    provider.setWhereClause("where id<=" + minvalue + " and id > " + maxvalue);

                    Map<String, Order> sortKeys = new HashMap<>(1);
                    sortKeys.put("id", Order.ASCENDING);
                    provider.setSortKeys(sortKeys);

                    reader.setQueryProvider(provider);
                    LOGGER.info("reader--->"+reader);
                    return reader;
                }

      @Override
        public Map<String, ExecutionContext> partition(int gridSize) {
            int min = jdbcTemplate.queryForObject("SELECT MIN(" + column + ") from " + table, Integer.class);
            int max = jdbcTemplate.queryForObject("SELECT MAX(" + column + ") from " + table, Integer.class);
            int targetSize = (max - min) / gridSize + 1;

            Map<String, ExecutionContext> result = new HashMap<String, ExecutionContext>();
            int number = 0;
            int start = min;
            int end = start + targetSize - 1;

            while (start <= max) {
                ExecutionContext value = new ExecutionContext();
                result.put("partition" + number, value);

                if (end >= max) {
                    end = max;
                }
                LOGGER.info("Start-->" + start);
                LOGGER.info("end-->" + end);
                value.putInt("minValue", start);
                value.putInt("maxValue", end);
                start += targetSize;
                end += targetSize;
                number++;
            }

            return result;
        }

ColumnRangePartitioner類的代碼段：-

 int min = jdbcTemplate.queryForObject("SELECT MIN(" + column + ") from " + table, Integer.class);
    int max = jdbcTemplate.queryForObject("SELECT MAX(" + column + ") from " + table, Integer.class);
    int targetSize = (max - min) / gridSize + 1;

    Map<String, ExecutionContext> result = new HashMap<String, ExecutionContext>();
    int number = 0;
    int start = min;
    int end = start + targetSize - 1;

    while (start <= max) {
        ExecutionContext value = new ExecutionContext();
        result.put("partition" + number, value);

        if (end >= max) {
            end = max;
        }
        LOGGER.info("Start-->" + start);
        LOGGER.info("end-->" + end);
        value.putInt("minValue", start);
        value.putInt("maxValue", end);
        start += targetSize;
        end += targetSize;
        number++;
    }

    return result;

Answer 1

我找到了解決這個問題的方法。 我們必須在分區器之后的masterStep中添加partitionHandler。 在partitionHandler中，我們定義slaveStep和其他配置。 以下是代碼段。

MasterStep：-在此處添加partitionHandler代碼，

     stepBuilderFactory
            .get("userMasterStep")
            .partitioner(userSlaveStep().getName(), userPartitioner())
            .partitionHandler(userMasterSlaveHandler())
            .build();

定義另一個名為partitionHandler的bean並在此處調用從屬步驟

   @Bean
public PartitionHandler userMasterSlaveHandler() throws Exception {
    TaskExecutorPartitionHandler handler = new TaskExecutorPartitionHandler();
    handler.setGridSize(gridSize);
    handler.setTaskExecutor(taskExecutorConfiguration.taskExecutor());
    handler.setStep(userSlaveStep());
    handler.afterPropertiesSet();
    return handler;
}