Spring 批处理 Hive Script tasklet 脚本执行错误

Question

I'm having trouble executing this step where a Hive script is fired. Even if change the script to a simple show databases; it still gives the same error.

 <hdp:hive-tasklet id="addPartitionToActiveItemsTableScriptTasklet"
                      scope="step">
        <hdp:script>
            <![CDATA[
            ALTER TABLE active_items DROP IF EXISTS PARTITION
            (batch_date="#{jobExecutionContext['formattedOperationDate']}");
            ALTER TABLE active_items ADD PARTITION
            (batch_date="#{jobExecutionContext['formattedOperationDate']}")
            LOCATION
            "${hdfs.data.path}/${hdfs.data.resource.active.items}/#{jobExecutionContext['formattedOperationDate']}";
        ]]>
        </hdp:script>
    </hdp:hive-tasklet>

Weirdly enough, when I execute a jdbc query that is linked to Hive Server 2 using Java codes instead of XML, it works. It's just this tasklet that doesn't work. So I'm not sure whether it is a hive configuration problem or something else. The error returned during the tasklet script execution says

2022-09-08 18:24:46.095 ERROR AbstractStep:229 - Encountered an error executing step addPartitionToActiveItemsTableScriptTaskletStep in job floorPriceActiveItemsJob
java.lang.NoClassDefFoundError: org/apache/hadoop/hive/service/HiveServerException
    at org.springframework.data.hadoop.hive.HiveTemplate$1.doInHive(HiveTemplate.java:267) ~[spring-data-hadoop-hive-2.5.0.RELEASE.jar:2.5.0.RELEASE]
    at org.springframework.data.hadoop.hive.HiveTemplate$1.doInHive(HiveTemplate.java:264) ~[spring-data-hadoop-hive-2.5.0.RELEASE.jar:2.5.0.RELEASE]
    at org.springframework.data.hadoop.hive.HiveTemplate.execute(HiveTemplate.java:83) ~[spring-data-hadoop-hive-2.5.0.RELEASE.jar:2.5.0.RELEASE]
    at org.springframework.data.hadoop.hive.HiveTemplate.executeScript(HiveTemplate.java:264) ~[spring-data-hadoop-hive-2.5.0.RELEASE.jar:2.5.0.RELEASE]
    at org.springframework.data.hadoop.hive.HiveExecutor.executeHiveScripts(HiveExecutor.java:53) ~[spring-data-hadoop-hive-2.5.0.RELEASE.jar:2.5.0.RELEASE]
    at org.springframework.data.hadoop.batch.hive.HiveTasklet.execute(HiveTasklet.java:33) ~[spring-data-hadoop-hive-2.5.0.RELEASE.jar:2.5.0.RELEASE]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_202]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_202]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_202]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_202]
    at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:333) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.aop.support.DelegatingIntroductionInterceptor.doProceed(DelegatingIntroductionInterceptor.java:133) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.aop.support.DelegatingIntroductionInterceptor.invoke(DelegatingIntroductionInterceptor.java:121) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213) ~[spring-aop-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at com.sun.proxy.$Proxy58.execute(Unknown Source) ~[?:?]
    at org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:406) ~[spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:330) ~[spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133) ~[spring-tx-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.batch.core.step.tasklet.TaskletStep$2.doInChunkContext(TaskletStep.java:271) ~[spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.scope.context.StepContextRepeatCallback.doInIteration(StepContextRepeatCallback.java:81) ~[spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.repeat.support.RepeatTemplate.getNextResult(RepeatTemplate.java:374) ~[spring-batch-infrastructure-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.repeat.support.RepeatTemplate.executeInternal(RepeatTemplate.java:215) ~[spring-batch-infrastructure-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.repeat.support.RepeatTemplate.iterate(RepeatTemplate.java:144) ~[spring-batch-infrastructure-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.step.tasklet.TaskletStep.doExecute(TaskletStep.java:257) ~[spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.step.AbstractStep.execute(AbstractStep.java:200) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.SimpleStepHandler.handleStep(SimpleStepHandler.java:148) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.flow.JobFlowExecutor.executeStep(JobFlowExecutor.java:64) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.flow.support.state.StepState.handle(StepState.java:67) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.flow.support.SimpleFlow.resume(SimpleFlow.java:169) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.flow.support.SimpleFlow.start(SimpleFlow.java:144) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.flow.FlowJob.doExecute(FlowJob.java:134) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.job.AbstractJob.execute(AbstractJob.java:306) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.batch.core.launch.support.SimpleJobLauncher$1.run(SimpleJobLauncher.java:135) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) [spring-core-4.3.25.RELEASE.jar:4.3.25.RELEASE]
    at org.springframework.batch.core.launch.support.SimpleJobLauncher.run(SimpleJobLauncher.java:128) [spring-batch-core-3.0.7.RELEASE.jar:3.0.7.RELEASE]
    at jp.co.rakuten.adplats.cpn.batch.job.AbstractCpnJob.execute(AbstractCpnJob.java:69) [classes/:?]
    at jp.co.rakuten.adplats.cpn.batch.ActiveItemsJob.main(ActiveItemsJob.java:9) [classes/:?]
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hive.service.HiveServerException
    at java.net.URLClassLoader.findClass(URLClassLoader.java:382) ~[?:1.8.0_202]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_202]
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) ~[?:1.8.0_202]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_202]
    ... 40 more

Here are my hadoop and hive configurations.

spring-hadoop.xml

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:util="http://www.springframework.org/schema/util" xmlns:context="http://www.springframework.org/schema/context"
    xmlns:c="http://www.springframework.org/schema/c" xmlns:p="http://www.springframework.org/schema/p"
    xmlns:batch="http://www.springframework.org/schema/batch" xmlns:task="http://www.springframework.org/schema/task"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hdp="http://www.springframework.org/schema/hadoop"
    xsi:schemaLocation="http://www.springframework.org/schema/context
                http://www.springframework.org/schema/context/spring-context.xsd
                http://www.springframework.org/schema/batch
                http://www.springframework.org/schema/batch/spring-batch.xsd
                http://www.springframework.org/schema/beans 
                http://www.springframework.org/schema/beans/spring-beans.xsd
                http://www.springframework.org/schema/util
                http://www.springframework.org/schema/util/spring-util.xsd
                http://www.springframework.org/schema/hadoop
                http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">

    <hdp:configuration id="hdpConfiguration">
        fs.defaultFS=hdfs://localhost:9000
    </hdp:configuration>

    <bean id="hiveDriver" class="org.apache.hive.jdbc.HiveDriver" />
    <bean id="hiveDataSource"
        class="org.springframework.jdbc.datasource.SimpleDriverDataSource">
        <constructor-arg name="driver" ref="hiveDriver" />
        <constructor-arg name="url" value="${hive.url}" />
        <constructor-arg name="username" value="hiveuser" />
        <constructor-arg name="password" value="hivepassword" />
    </bean>
    <bean id="namedParameterJdbcHiveTemplate" name="namedParameterJdbcHiveTemplate" class="org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate">
        <constructor-arg ref="hiveDataSource"/>
    </bean>

    <hdp:hive-client-factory hive-data-source-ref="hiveDataSource" />

</beans>

Answer 1

Solved by downgrading spring-data-hadoop and hive-jdbc maven dependency versions.