第一次失败后开玩笑停止测试套件

Question

I am using Jest for testing.

What I want, is to stop executing the current test suite when a test in that test suite fails.

The --bail option is not what I need, since it will stop other test suites after one test suite fails.

Answer 1

I've made some kludge but it works for me.

stopOnFirstFailed.js :

/**
 * This is a realisation of "stop on first failed" with Jest
 * @type {{globalFailure: boolean}}
 */

module.exports = {
    globalFailure: false
};

// Injects to jasmine.Spec for checking "status === failed"
!function (OriginalSpec) {
    function PatchedSpec(attrs) {
        OriginalSpec.apply(this, arguments);

        if (attrs && attrs.id) {
            let status = undefined;
            Object.defineProperty(this.result, 'status', {
                get: function () {
                    return status;
                },
                set: function (newValue) {
                    if (newValue === 'failed') module.exports.globalFailure = true;
                    status = newValue;
                },
            })
        }
    }

    PatchedSpec.prototype = Object.create(OriginalSpec.prototype, {
        constructor: {
            value: PatchedSpec,
            enumerable: false,
            writable: true,
            configurable: true
        }
    });

    jasmine.Spec = PatchedSpec;
}(jasmine.Spec);

// Injects to "test" function for disabling that tasks
test = ((testOrig) => function () {
    let fn = arguments[1];

    arguments[1] = () => {
        return module.exports.globalFailure ? new Promise((res, rej) => rej('globalFailure is TRUE')) : fn();
    };

    testOrig.apply(this, arguments);
})(test);

Imports that file before all tests (before first test(...) ), for ex my index.test.js :

require('./core/stopOnFirstFailed'); // before all tests

test(..., ()=>...);
...

That code marks all next tests failed with label globalFailure is TRUE when first error happens.

If you want to exclude failing , for ex. some cleanup tests you can do like this:

const stopOnFirstFailed = require('../core/stopOnFirstFailed');

describe('some protected group', () => {
    beforeAll(() => {
        stopOnFirstFailed.globalFailure = false
    });
    test(..., ()=>...);
    ...

It excludes whole group from failing .

Tested with Node 8.9.1 and Jest 23.6.0

Answer 2

Thanks to this comment on github I was able to resolve this with a custom testEnvironment . For this to work jest-circus need to be installed via npm / yarn .
It's worth noting that jest will set jest-circus to the default runner with jest v27 .

First of all jest configuration needs to be adapted:

jest.config.js

module.exports = {
  rootDir: ".",
  testRunner: "jest-circus/runner",
  testEnvironment: "<rootDir>/NodeEnvironmentFailFast.js",
}

Then you need to implement a custom environment, which is already referenced by the config above:

NodeEnvironmentFailFast.js

const NodeEnvironment = require("jest-environment-node")

class NodeEnvironmentFailFast extends NodeEnvironment {
  failedDescribeMap = {}
  registeredEventHandler = []

  async setup() {
    await super.setup()
    this.global.testEnvironment = this
  }

  registerTestEventHandler(registeredEventHandler) {
    this.registeredEventHandler.push(registeredEventHandler)
  }

  async executeTestEventHandlers(event, state) {
    for (let handler of this.registeredEventHandler) {
      await handler(event, state)
    }
  }

  async handleTestEvent(event, state) {
    await this.executeTestEventHandlers(event, state)

    switch (event.name) {
      case "hook_failure": {
        const describeBlockName = event.hook.parent.name

        this.failedDescribeMap[describeBlockName] = true
        // hook errors are not displayed if tests are skipped, so display them manually
        console.error(`ERROR: ${describeBlockName} > ${event.hook.type}\n\n`, event.error, "\n")
        break
      }
      case "test_fn_failure": {
        this.failedDescribeMap[event.test.parent.name] = true
        break
      }
      case "test_start": {
        if (this.failedDescribeMap[event.test.parent.name]) {
          event.test.mode = "skip"
        }
        break
      }
    }

    if (super.handleTestEvent) {
      super.handleTestEvent(event, state)
    }
  }
}

module.exports = NodeEnvironmentFailFast

---

NOTE

I added registerTestEventHandler functionality which is not necessary for the fail fast feature, but I thought it's quite useful, especially if you used jasmine.getEnv() before and it works with async / await !
You can register custom handler inside of your tests (eg beforeAll hook) like so:

// testEnvironment is globally available (see above NodeEnvironmentFailFast.setup)
testEnvironment.registerTestEventHandler(async (event) => {
  if (event.name === "test_fn_failure") {
    await takeScreenshot()
  }
})

---

When one test fails, other test statements in the same describe will be skipped. This also works for nested describe blocks, but the describe blocks must have different names.

Executing following test:

describe("TestJest 3 ", () => {
  describe("TestJest 2 ", () => {
    describe("TestJest 1", () => {
      beforeAll(() => expect(1).toBe(2))
      test("1", () => {})
      test("1.1", () => {})
      test("1.2", () => {})
    })

    test("2", () => expect(1).toBe(2))
    test("2.1", () => {})
    test("2.2", () => {})
  })

  test("3", () => {})
  test("3.1", () => expect(1).toBe(2))
  test("3.2", () => {})
})

will produce following log:

 FAIL  suites/test-jest.spec.js
  TestJest 3 
    ✓ 3
    ✕ 3.1 (1 ms)
    ○ skipped 3.2
    TestJest 2 
      ✕ 2
      ○ skipped 2.1
      ○ skipped 2.2
      TestJest 1
        ○ skipped 1
        ○ skipped 1.1
        ○ skipped 1.2

  ● TestJest 3  › TestJest 2  › TestJest 1 › 1

    expect(received).toBe(expected) // Object.is equality

    Expected: 2
    Received: 1

      2 |   describe("TestJest 2 ", () => {
      3 |     describe("TestJest 1", () => {
    > 4 |       beforeAll(() => expect(1).toBe(2))
        |                                 ^
      5 |       test("1", () => {})
      6 |       test("1.1", () => {})
      7 |       test("1.2", () => {})

      at suites/test-jest.spec.js:4:33

  ● TestJest 3  › TestJest 2  › TestJest 1 › 1.1

    expect(received).toBe(expected) // Object.is equality

    Expected: 2
    Received: 1

      2 |   describe("TestJest 2 ", () => {
      3 |     describe("TestJest 1", () => {
    > 4 |       beforeAll(() => expect(1).toBe(2))
        |                                 ^
      5 |       test("1", () => {})
      6 |       test("1.1", () => {})
      7 |       test("1.2", () => {})

      at suites/test-jest.spec.js:4:33

  ● TestJest 3  › TestJest 2  › TestJest 1 › 1.2

    expect(received).toBe(expected) // Object.is equality

    Expected: 2
    Received: 1

      2 |   describe("TestJest 2 ", () => {
      3 |     describe("TestJest 1", () => {
    > 4 |       beforeAll(() => expect(1).toBe(2))
        |                                 ^
      5 |       test("1", () => {})
      6 |       test("1.1", () => {})
      7 |       test("1.2", () => {})

      at suites/test-jest.spec.js:4:33

  ● TestJest 3  › TestJest 2  › 2

    expect(received).toBe(expected) // Object.is equality

    Expected: 2
    Received: 1

       8 |     })
       9 | 
    > 10 |     test("2", () => expect(1).toBe(2))
         |                               ^
      11 |     test("2.1", () => {})
      12 |     test("2.2", () => {})
      13 |   })

      at Object.<anonymous> (suites/test-jest.spec.js:10:31)

  ● TestJest 3  › 3.1

    expect(received).toBe(expected) // Object.is equality

    Expected: 2
    Received: 1

      14 | 
      15 |   test("3", () => {})
    > 16 |   test("3.1", () => expect(1).toBe(2))
         |                               ^
      17 |   test("3.2", () => {})
      18 | })
      19 | 

      at Object.<anonymous> (suites/test-jest.spec.js:16:31)

Test Suites: 1 failed, 1 total
Tests:       2 failed, 6 skipped, 1 passed, 9 total
Snapshots:   0 total
Time:        0.638 s, estimated 1 s

Answer 3

This was my solution -- if there are major downsides please let me know, for my purposes it seems to work as intended

I only have one top-level describe block, for my purposes I want the entire test file to fail when one test fails

export class FailEarly {
  msg: string | undefined;
  failed: boolean = false;
  jestIt: jest.It;

  constructor(jestIt: jest.It) {
    this.jestIt = jestIt;
  }

  test = (name: string, fn: jest.EmptyFunction, timeout?: number) => {
    const failEarlyFn = async () => {
      if (this.failed) {
        throw new Error(`failEarly: ${this.msg}`);
      }

      try {
        await fn();
      } catch (error) {
        this.msg = name;
        this.failed = true;
        throw error;
      }
    };

    this.jestIt(name, failEarlyFn, timeout);
  };
}

Gives me a context (class attributes) to store global-esq variables

const failEarlyTestRunner = new FailEarly(global.it);

const test = failEarlyTestRunner.test;
const it = failEarlyTestRunner.test;

overloads the test and it functions with my class' method (thus accessing the class attributes)

describe('my stuff', () => {
  it('passes', async () => {
    expect(1).toStrictEqual(1);
  })

  test('it fails', async () => {
    expect(1).toStrictEqual(2);
  })

  it('is skipped', async () => {
    expect(1).toStrictEqual(1);
  })
})

results in:

my stuff
  ✓ can create a sector (2 ms)
  ✕ it fails (2 ms)
  ✕ is skipped (1 ms)


  ● my stuff › it fails

    expect(received).toStrictEqual(expected) // deep equality

    Expected: 2
    Received: 1

    > ### |       expect(1).toStrictEqual(2);
          |                 ^
      ### |     });


  ● my stuff › is skipped

    failEarly: it fails

      69 |     const failEarlyFn = async () => {
      70 |       if (this.failed) {
    > 71 |         throw new Error(`failEarly: ${this.msg}`);
         |               ^
      72 |       }
      73 |
      74 |       try {

Where each skipped test is failed w/ an error indicating the upstream, failing test

As others have pointed out -- you've got to run jest with the --runInBand flag

Hope this helps someone -- if there are meaningful drawbacks or better ways please comment; I'm always happy to learn

Answer 4

hack the global.jasmine.currentEnv_.fail works for me.

      describe('Name of the group', () => {

        beforeAll(() => {

          global.__CASE_FAILED__= false

          global.jasmine.currentEnv_.fail = new Proxy(global.jasmine.currentEnv_.fail,{
            apply(target, that, args) {
              global.__CASE__FAILED__ = true
              // you also can record the failed info...
              target.apply(that, args)
              }
            }
          )

        })

        afterAll(async () => {
          if(global.__CASE_FAILED__) {
            console.log("there are some case failed");
            // TODO ...
          }
        })

        it("should xxxx", async () => {
          // TODO ...
          expect(false).toBe(true)
        })
      });

Answer 5

I guess that other tests in that suite depend on all previous tests be successful. This is a bad practice in unit testing. Try using beforeEach and afterEach to decouple individual test cases within a suite, so that they don't rely on each other.