Unable to run camel:quartz2 in clustering mode using spring boot application

Question

I'm trying to implement camel:quartz2 scheduler in clustering mode. Here is my two Routes

    from("quartz2://Ingestion/ruleExecuteFirstSequence?cron=" + rulesExecutionSeviceImpl.getSequenceCronExpression(1) + "")
            .log("Start executing firstSequence Rule").bean(RulesExecutor.class, "getExecuteRuleWithSequence(1)")
            .log("Completed executing firstSequence Rule").end();

    from("quartz2://Ingestion/ruleExecuteSecondSequence?cron=" + rulesExecutionSeviceImpl.getSequenceCronExpression(2) + "")
            .log("Start executing secondSequence Rule").bean(RulesExecutor.class, "getExecuteRuleWithSequence(2)")
            .log("Completed executing secondSequence Rule").end();

My application.properties look like below in

spring.quartz.properties.org.quartz.scheduler.instanceName = ClusteredSchedular
spring.quartz.properties.org.quartz.scheduler.instanceId = AUTO
spring.quartz.properties.org.quartz.scheduler.jobFactory.class = org.quartz.simpl.SimpleJobFactory
spring.quartz.properties.org.quartz.scheduler.skipUpdateCheck = true
spring.quartz.properties.org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
spring.quartz.properties.org.quartz.jobStore.driverDelegateClass =org.quartz.impl.jdbcjobstore.MSSQLDelegate
spring.quartz.properties.org.quartz.jobStore.tablePrefix = QRTZ_
spring.quartz.properties.org.quartz.jobStore.isClustered=true
spring.quartz.properties.org.quartz.jobStore.clusterCheckinInterval = 20000
spring.quartz.properties.org.quartz.jobStore.useProperties=true
spring.quartz.properties.org.quartz.jobStore.misfireThreshold = 60000

spring.quartz.properties.org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
spring.quartz.properties.org.quartz.threadPool.threadCount = 10
spring.quartz.properties.org.quartz.org.quartz.threadPool.threadPriority = 5
spring.quartz.job-store-type=jdbc

I'm using the sqlserver dataSource which is already existing in my application.{envi}.properties for which i'm using spring.quartz.job-store-type=jdbc property.

I created db tables using quartz-2.3.0-distribution quartz-core\\src\\main\\resources\\org\\quartz\\impl\\jdbcjobstore\\tables_sqlServer.sql

Here are my dependencies in build.gradle

buildscript {
    ext {
        springBootVersion = '2.0.4.RELEASE'
        //springKafkaVersion = "2.2.0.RELEASE"
        avroVersion = "1.8.2"
        confluentVersion = "5.0.0"
        gradleAvroPluginVersion = "0.16.0"
        gradleOwaspVersion = "3.3.0"
        camelVersion = '2.22.1'
    }
    repositories {
        mavenCentral()
        maven { url "http://packages.confluent.io/maven/" }
        maven { url 'https://plugins.gradle.org/m2/' }
    }
    dependencies {
        classpath "org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}"
        classpath "com.commercehub.gradle.plugin:gradle-avro-plugin:${gradleAvroPluginVersion}"
        classpath "org.owasp:dependency-check-gradle:${gradleOwaspVersion}"
    }
}


apply plugin: 'groovy'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'org.springframework.boot'
apply plugin: 'io.spring.dependency-management'
apply plugin: 'com.commercehub.gradle.plugin.avro'
//apply plugin: 'org.owasp.dependencycheck'
apply from: 'gradle/common.gradle'
//apply from: 'gradle/codenarc.gradle'
apply from: 'gradle/checkstyle.gradle'
apply from: 'gradle/jacoco.gradle'
apply from: 'gradle/test.gradle'

group = 'com.tcfbank.risk'
version = '0.0.1-SNAPSHOT'
sourceCompatibility = 1.8
targetCompatibility = 1.8

repositories {
    mavenCentral()
    maven { url "http://packages.confluent.io/maven/" }
}


dependencies { compileOnly 'org.projectlombok:lombok:1.18.4' }

sourceSets {
    main{
        java {
            srcDir 'src/main/java'
            srcDir 'src/main/avro'
        }
    }
}
avro {
    createSetters = false
    fieldVisibility = "PRIVATE"
}

task generateAvro(type: com.commercehub.gradle.plugin.avro.GenerateAvroJavaTask) {
    source("src/main/resources/avro")
    outputDir = file("src/main/avro")
}
compileJava.source(generateAvro.outputs)

dependencies {
    compile('org.springframework.boot:spring-boot-starter-web')
    compile('org.springframework.boot:spring-boot-starter-actuator')
    compile 'org.springframework.kafka:spring-kafka'
    compile("org.springframework.boot:spring-boot-starter-data-jpa")
    compile 'org.codehaus.groovy:groovy-all:2.4.13'

    compile("io.confluent:kafka-avro-serializer:$confluentVersion"){exclude group:'com.fasterxml.jackson.core', module :'jackson-databind'}
    compile "org.apache.avro:avro:${avroVersion}"
    compile("commons-io:commons-io:2.6")
    compile("org.apache.commons:commons-lang3:3.5");
    compile("org.apache.commons:commons-collections4:4.1")
    compile("com.googlecode.json-simple:json-simple:1.1.1")
    compile 'com.microsoft.sqlserver:mssql-jdbc:6.4.0.jre8'
    //runtime('com.microsoft.sqlserver:mssql-jdbc')

    // camel routing deps
    compile "org.apache.camel:camel-jetty:${camelVersion}"
    compile "org.apache.camel:camel-spring-boot-starter:${camelVersion}"
    compile "org.apache.camel:camel-spring:${camelVersion}"
    compile "org.apache.camel:camel-jackson:${camelVersion}"

    // https://mvnrepository.com/artifact/org.apache.camel/camel-quartz2-starter
    compile "org.apache.camel:camel-quartz2-starter:${camelVersion}"

    // https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-quartz
    compile group: 'org.springframework.boot', name: 'spring-boot-starter-quartz', version: '2.1.3.RELEASE'

    // https://mvnrepository.com/artifact/org.apache.camel/camel-core
    compile group: 'org.apache.camel', name: 'camel-core', version: '2.22.1'

    compile group: 'org.springframework', name: 'spring-context-support', version: '4.3.12.RELEASE'


    annotationProcessor("org.projectlombok:lombok:1.16.20")
    compileOnly("org.projectlombok:lombok:1.16.20")

    testImplementation('org.springframework.boot:spring-boot-starter-test')
    //testImplementation("org.springframework.boot:spring-kafka-test")
    testCompile group: 'junit', name: 'junit', version: '4.12'
    //Groovy depends on spock version

    testCompile("org.codehaus.groovy:groovy-all:2.4.5")
    testCompile group: 'org.spockframework', name: 'spock-spring', version: '1.1-groovy-2.4'
    // https://mvnrepository.com/artifact/org.apache.camel/camel-quartz2
    testCompile group: 'org.apache.camel', name: 'camel-quartz2', version: '2.12.1'
    // https://mvnrepository.com/artifact/org.apache.camel/camel-test-spring
    testCompile group: 'org.apache.camel', name: 'camel-test-spring', version: '2.23.1'
    // https://mvnrepository.com/artifact/com.h2database/h2
    testCompile group: 'com.h2database', name: 'h2', version: '1.4.197'

}

when I run with the above code locally, logs look like it is not running in clustered mode

15:41:14.628 [main] INFO  org.quartz.impl.StdSchedulerFactory - Using default implementation for ThreadExecutor
15:41:14.647 [main] INFO  o.quartz.core.SchedulerSignalerImpl - Initialized Scheduler Signaller of type: class org.quartz.core.SchedulerSignalerImpl
15:41:14.647 [main] INFO  org.quartz.core.QuartzScheduler - Quartz Scheduler v.2.3.0 created.
15:41:14.648 [main] INFO  org.quartz.core.QuartzScheduler - JobFactory set to: org.quartz.simpl.SimpleJobFactory@5bb39285
15:41:14.683 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - Detected usage of MSSQLDelegate class - defaulting 'selectWithLockSQL' to 'SELECT * FROM {0}LOCKS WITH (UPDLOCK,ROWLOCK) WHERE SCHED_NAME = {1} AND LOCK_NAME = ?'.
15:41:14.683 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - Using db table-based data access locking (synchronization).
15:41:14.686 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - JobStoreCMT initialized.
15:41:14.687 [main] INFO  org.quartz.core.QuartzScheduler - Scheduler meta-data: Quartz Scheduler (v2.3.0) 'quartzScheduler' with instanceId 'PCC-016098LTA641551649274629'
  Scheduler class: 'org.quartz.core.QuartzScheduler' - running locally.
  NOT STARTED.
  Currently in standby mode.
  Number of jobs executed: 0
  Using thread pool 'org.quartz.simpl.SimpleThreadPool' - with 10 threads.
  Using job-store 'org.springframework.scheduling.quartz.LocalDataSourceJobStore' - which supports persistence. and is clustered.

15:41:14.687 [main] INFO  org.quartz.impl.StdSchedulerFactory - Quartz scheduler 'quartzScheduler' initialized from an externally provided properties instance.
15:41:14.687 [main] INFO  org.quartz.impl.StdSchedulerFactory - Quartz scheduler version: 2.3.0
15:41:14.688 [main] INFO  org.quartz.core.QuartzScheduler - JobFactory set to: org.springframework.boot.autoconfigure.quartz.AutowireCapableBeanJobFactory@1acc768
.
.
.
15:41:15.727 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - ClusterManager: detected 1 failed or restarted instances.
15:41:15.727 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - ClusterManager: Scanning for instance "PCC-016098LTA641551649203578"'s failed in-progress jobs.
.
.
.
15:41:15.908 [main] INFO  org.quartz.core.QuartzScheduler - Scheduler quartzScheduler_$_PCC-016098LTA641551649274629 started.
.
.
.
.
15:41:17.130 [main] INFO  o.a.c.spring.boot.RoutesCollector - Loading additional Camel XML routes from: classpath:camel/*.xml
15:41:17.131 [main] INFO  o.a.c.spring.boot.RoutesCollector - Loading additional Camel XML rests from: classpath:camel-rest/*.xml
15:41:17.135 [main] INFO  o.a.camel.spring.SpringCamelContext - Apache Camel 2.22.1 (CamelContext: camel-1) is starting
15:41:17.137 [main] INFO  o.a.c.m.ManagedManagementStrategy - JMX is enabled
15:41:17.336 [main] INFO  o.a.c.c.quartz2.QuartzComponent - Setting org.quartz.scheduler.jmx.export=true to ensure QuartzScheduler(s) will be enlisted in JMX.
15:41:17.336 [main] INFO  o.a.c.c.quartz2.QuartzComponent - Create and initializing scheduler.
15:41:17.345 [main] INFO  org.quartz.impl.StdSchedulerFactory - Using default implementation for ThreadExecutor
15:41:17.345 [main] INFO  org.quartz.simpl.SimpleThreadPool - Job execution threads will use class loader of thread: main
15:41:17.348 [main] INFO  o.quartz.core.SchedulerSignalerImpl - Initialized Scheduler Signaller of type: class org.quartz.core.SchedulerSignalerImpl
15:41:17.349 [main] INFO  org.quartz.core.QuartzScheduler - Quartz Scheduler v.2.3.0 created.
15:41:17.349 [main] INFO  org.quartz.simpl.RAMJobStore - RAMJobStore initialized.
15:41:17.357 [main] INFO  org.quartz.core.QuartzScheduler - Scheduler meta-data: Quartz Scheduler (v2.3.0) 'DefaultQuartzScheduler-camel-1' with instanceId 'NON_CLUSTERED'
  Scheduler class: 'org.quartz.core.QuartzScheduler' - running locally.
  NOT STARTED.
  Currently in standby mode.
  Number of jobs executed: 0
  Using thread pool 'org.quartz.simpl.SimpleThreadPool' - with 10 threads.
  Using job-store 'org.quartz.simpl.RAMJobStore' - which does not support persistence. and is not clustered.

15:41:17.357 [main] INFO  org.quartz.impl.StdSchedulerFactory - Quartz scheduler 'DefaultQuartzScheduler-camel-1' initialized from an externally provided properties instance.
15:41:17.357 [main] INFO  org.quartz.impl.StdSchedulerFactory - Quartz scheduler version: 2.3.0
15:41:17.686 [main] INFO  o.a.camel.spring.SpringCamelContext - StreamCaching is not in use. If using streams then its recommended to enable stream caching. See more details at http://camel.apache.org/stream-caching.html
15:41:17.708 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteFirstSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:30 CST 2019
15:41:17.778 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteSecondSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:32 CST 2019
15:41:17.791 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteThirdSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:17 CST 2019
15:41:17.800 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteForthSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:18 CST 2019
15:41:17.813 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteFifthSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:19 CST 2019
15:41:17.830 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteSixthSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:20 CST 2019
15:41:17.842 [main] INFO  o.a.c.c.quartz2.QuartzEndpoint - Job fraudIngestion.ruleExecuteSequence (triggerType=CronTriggerImpl, jobClass=CamelJob) is scheduled. Next fire date is Sun Mar 03 15:41:50 CST 2019
15:41:18.025 [main] INFO  org.eclipse.jetty.util.log - Logging initialized @21750ms to org.eclipse.jetty.util.log.Slf4jLog
15:41:18.067 [main] INFO  o.a.c.c.quartz2.QuartzComponent - Starting scheduler.
15:41:18.067 [main] INFO  org.quartz.core.QuartzScheduler - Scheduler DefaultQuartzScheduler-camel-1_$_NON_CLUSTERED started.
15:41:18.069 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route1 started and consuming from: quartz2://fraudIngestion/ruleExecuteFirstSequence?cron=0%2F15+*+*+*+*+%3F
15:41:18.071 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route2 started and consuming from: quartz2://fraudIngestion/ruleExecuteSecondSequence?cron=0%2F16+*+*+*+*+%3F
15:41:18.074 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route3 started and consuming from: quartz2://fraudIngestion/ruleExecuteThirdSequence?cron=0%2F17+*+*+*+*+%3F
15:41:18.076 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route4 started and consuming from: quartz2://fraudIngestion/ruleExecuteForthSequence?cron=0%2F18+*+*+*+*+%3F
15:41:18.077 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route5 started and consuming from: quartz2://fraudIngestion/ruleExecuteFifthSequence?cron=0%2F19+*+*+*+*+%3F
15:41:18.080 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route6 started and consuming from: quartz2://fraudIngestion/ruleExecuteSixthSequence?cron=0%2F20+*+*+*+*+%3F
15:41:18.083 [main] INFO  o.a.camel.spring.SpringCamelContext - Route: route7 started and consuming from: quartz2://fraudIngestion/ruleExecuteSequence?cron=0%2F50+*+*+*+*+%3F

I tried a lot and looks like I'm missing something. I even didn't find much info with the combination of camel:quartz2 schedular in clustering mode using spring boot application and using sqlserver datasource.

the below two line in my logs states that clusterManager failed.

15:41:15.727 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - ClusterManager: detected 1 failed or restarted instances.
15:41:15.727 [main] INFO  o.s.s.quartz.LocalDataSourceJobStore - ClusterManager: Scanning for instance "PCC-016098LTA641551649203578"'s failed in-progress jobs.
.

Can anyone help me in giving a solution or suggestion. This would be a great help since we are going live with this application in couple of weeks.

Answer 1

You should use the Camel Spring Boot starter JARs when you use Spring Boot with Camel. In your dependencies use camel-quartz2-starter , camel-jackson-starter . This allows Camel components to support/use spring boot auto-configuration.

Answer 2

Thanks for responding Claus but looks like I already have camel-quartz2-starter and camel-jackson-starter. But changing my application.propeties with below is working for me:

# quartz scheduler configurations
camel.component.quartz2.properties.org.quartz.scheduler.skipUpdateCheck = true
camel.component.quartz2.properties.org.quartz.scheduler.instanceName = ClusteredSchedular
camel.component.quartz2.properties.org.quartz.scheduler.instanceId = AUTO
camel.component.quartz2.properties.org.quartz.scheduler.jobFactory.class = org.quartz.simpl.SimpleJobFactory

# Jobstore configurations
camel.component.quartz2.properties.org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
camel.component.quartz2.properties.org.quartz.jobStore.driverDelegateClass =org.quartz.impl.jdbcjobstore.MSSQLDelegate
camel.component.quartz2.properties.org.quartz.jobStore.tablePrefix = QRTZ_
camel.component.quartz2.properties.org.quartz.jobStore.isClustered=true
camel.component.quartz2.properties.org.quartz.jobStore.clusterCheckinInterval = 20000
camel.component.quartz2.properties.org.quartz.jobStore.misfireThreshold = 60000
camel.component.quartz2.properties.org.quartz.jobStore.useProperties=true
camel.component.quartz2.properties.org.quartz.jobStore.dataSource=QRTZDS

# Instructs the Scheduler whether or not the Job should be re-executed if a 'recovery' or 'fail-over' situation is encountered
camel.component.quartz2.properties.org.quartz.JobBuilder.requestRecovery= true

# ThreadPool settings
camel.component.quartz2.properties.org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
camel.component.quartz2.properties.org.quartz.threadPool.threadCount = 25

# Using sqlserver
camel.component.quartz2.properties.org.quartz.dataSource.QRTZDS.driver=com.microsoft.sqlserver.jdbc.SQLServerDriver
camel.component.quartz2.properties.org.quartz.dataSource.QRTZDS.maxConnections=30
camel.component.quartz2.properties.org.quartz.dataSource.QRTZDS.URL=${spring.datasource.url}
camel.component.quartz2.properties.org.quartz.dataSource.QRTZDS.user=${spring.datasource.username}