TestNG retry test in multithreaded environment

Question

I have been using TestNG retry mechanism as following -

public class TestRetryAnalyzer implements IRetryAnalyzer {

public static final int MAX_RETRY_COUNT = 3;
private static final AtomicInteger count = new AtomicInteger(MAX_RETRY_COUNT);

public static void resetCount() {
    count.set(MAX_RETRY_COUNT);
}

public int getCount() {
    return count.get();
}

private boolean isRetryAvailable() {
    return (count.get() > 0);
}

@Override
public boolean retry(ITestResult result) {
    boolean retry = false;
    if (isRetryAvailable()) {
        System.out.println("Going to retry test case: " + result.getMethod() + ", " + (((MAX_RETRY_COUNT - count.get()) + 1)) + " out of " + MAX_RETRY_COUNT);
        retry = true;
        count.decrementAndGet();
    }
    return retry;
}

}

public class TestRetryListener implements IAnnotationTransformer {

@Override
public void transform(final ITestAnnotation annotation, final Class testClass, final Constructor testConstructor,
                      final Method testMethod) {
    IRetryAnalyzer retryAnalyzer = annotation.getRetryAnalyzer();
    if (retryAnalyzer == null) {
        annotation.setRetryAnalyzer(TestRetryAnalyzer.class);
    }
}

and a Test -

public class SimpleTest {

private int count = 0;

@Test
public void test() {
    count++;
    if (count % 3 != 0) {
        Assert.fail("Injected Failure");
    }
    count = 0;
}

And testng.xml file -

<suite name="Suite1" verbose="1">
<listeners>
    <listener class-name="main.java.TestRetryListener" />
    <listener class-name="org.uncommons.reportng.HTMLReporter" />
    <listener class-name="org.uncommons.reportng.JUnitXMLReporter" />
</listeners>
<test name="Sample" >
    <classes>
        <class name="main.java.SimpleTest" />
    </classes>
</test>
<test name="Sample2" >
    <classes>
        <class name="main.java.SimpleTest" />
    </classes>
</test>
<test name="Sample3" >
    <classes>
        <class name="main.java.SimpleTest" />
    </classes>
</test>
<test name="Sample4" >
    <classes>
        <class name="main.java.SimpleTest" />
    </classes>
</test>
<test name="Sample5" >
    <classes>
        <class name="main.java.SimpleTest" />
    </classes>
</test>

When I run only one test then retry mechanism works well (1 pass and 2 skipped). But when I run 5 tests as mentioned in above testng.xml file then they begin to fail. Do I run in to concurrency issues without even running tests in parallel? How do I get rid of it?

I am using testNG 6.9.10

Answer 1

TestRetryListener is called only one time by TestNG framework so there will be only one instance of TestRetryAnalyzer will be present in the heap. Instance variable "count" (instance of AtomicInteger class) is being shared across multiple tests. It is not reset to 3 for every test that means AtomicInteger object is instantiated only once and the same reference is used by TestRetryAnalyzer class.

To overcome the above problem, we have to reset the count to 3 after every test. To achieve this, you have to make the following changes

public class SimpleTest {

private int count = 0;

@Test
public void test() {
    count++;
    System.out.println("Test Count ... " + count);
    if (count % 4 != 0) {
        Assert.fail("Injected Failure");
    }
    count = 0;
}

@AfterTest
public void afterTest() {
    TestRetryAnalyzer.resetCount();
}

}

public class TestRetryAnalyzer implements IRetryAnalyzer {

private static int MAX_RETRY_COUNT = 3;

private static AtomicInteger count = new AtomicInteger(MAX_RETRY_COUNT);

public static void resetCount() {
    count.set(MAX_RETRY_COUNT);
}

private boolean isRetryAvailable() {
    return (count.intValue() > 0);
}

@Override
public boolean retry(ITestResult result) {
    boolean retry = false;
    if (isRetryAvailable()) {
        retry = true;
        count.decrementAndGet();
        System.out.println("Retry Analyzer count is : " + count);
    }
    return retry;
}

}

Hope this helps :)

Answer 2

I am using a ThreadLocal instance for dealing with this issue. This solution works perfectly.

public class RetryAnalyzer implements IRetryAnalyzer {

    private ThreadLocal<Integer> count = ThreadLocal.withInitial(() -> 0);

    private static final int maxTry = TestRunner.getRunConfig().getRerun();
    private static final Logger logger = LogManager.getLogger(RetryAnalyzer.class);

    @Override
    public synchronized boolean retry(ITestResult result) {
        if (!result.isSuccess()) {                      //Check if test not succeed
            if (count.get() < maxTry) {                 //Check if maxtry count is reached
                count.set(count.get() + 1);             //Increase the maxTry count by 1
                result.setStatus(ITestResult.FAILURE);  //Mark test as failed
                logger.debug("Now going to retry " + result.getMethod().getMethodName());
                logger.debug("Retry count is " + count);
                return true;                            //Tells TestNG to re-run the test
            } else {
                result.setStatus(ITestResult.FAILURE);  //If maxCount reached,test marked as failed
                count.set(0);                           //Reset count to 0 for next set of tests
                logger.debug("Reset the retry count for " + result.getMethod().getMethodName());
            }
        } else {
            result.setStatus(ITestResult.SUCCESS);      //If test passes, TestNG marks it as passed
        }
        return false;
    }
}