Karma Coverage + RequireJS: Misleading coverage reports

Question

Today I've integrated Karma Coverage to my existing RequireJS application. I added the karma-requirejs plugin and I was able to successfully get the coverage reports.

Initially the report was very good, almost 100% coverage. When I carefully analyzed the results I noticed a lot of missing files from my "src" folder which led to this extremely good report.

It turns out that the coverage is just applying the analysis for the "src" files that have a corresponding spec (because I use require ('someFileFromSrcFolder') inside that spec).

Question : Is there a way for the coverage to analyze all the files from "src" folder?

Karma-conf.js

module.exports = function (config) {
    config.set({
        basePath: '../',
        autoWatch: true,
        singleRun: false,
        frameworks: ['jasmine', 'requirejs'],
        browsers: ['PhantomJS', 'Chrome'],
        logLevel: config.LOG_ERROR,
        files: [
            {pattern: 'vendor/**/*.js', included: false},
            {pattern: 'vendor/**/*.html', included: false},
            {pattern: 'src/**/*.js', included: false},
            {pattern: 'src/**/*.html', included: false},
            {pattern: 'tests/mock/**/*.js', included: false},
            {pattern: 'tests/**/*Specs.js', included: false},

            'tests/test-require.js'
        ],

        // list of files to exclude
        exclude: [
            'src/app-require.js'
        ],

        reporters: ['progress', 'coverage'],

        preprocessors: {
            'src/**/*.js': ['coverage']
        }
    });
};

Test-require.js

var allTestFiles = [];
var TEST_REGEXP = /Specs\.js$/;

Object.keys(window.__karma__.files).forEach(function (file) {
    if (TEST_REGEXP.test(file)) {
        allTestFiles.push(file);
    } 
});

require.config({

    baseUrl: '/base/',

    paths: {
       ...
    },

    deps: allTestFiles,

    callback: window.__karma__.start(),

waitSeconds: 20
});

Answer 1

Ok,

After trying for a little bit I was able to come to a solution that seems to fix the problem. I'm still not sure if this is the best solution but I will post here what I did so that I can gather your feedbacks.

Basically, I had to change the load mechanism in test-require.js to include all my packages by default. The updated test-require.js should look like follows:

Updated Test-require.js

var allTestFiles = [];
var modules = [];
var TEST_REGEXP = /Specs\.js$/;
var SRC_REGEXP = /src\//;
var JS_REGEXP = /\.js$/;

/**
* This function converts a given js path to requirejs module
*/
var jsToModule = function (path) {
    return path.replace(/^\/base\//, '').replace(/\.js$/, '');
};

Object.keys(window.__karma__.files).forEach(function (file) {
    if (TEST_REGEXP.test(file)) {
        allTestFiles.push(file);
    } else if (SRC_REGEXP.test(file) && JS_REGEXP.test(file)) {
        modules.push(jsToModule(file));
    }
});

var startTest = function () {
    //loading all the existing requirejs src modules before
    //triggering the karma test
    require(modules, function () { 
        window.__karma__.start();
    });
};

require.config({

    baseUrl: '/base/',

    paths: {
       ....
    },

    // dynamically load all test files
    deps: allTestFiles,

    callback: startTest,

    waitSeconds: 20
});

Now, when I run test tests, the coverage includes all the src modules and I can finally get a consistent and accurate reports.

Answer 2

I recently has the same problem. I have a solution in my project angular-require-lazy , which I will describe. Although it required a lot of custom stuff, it works in the end.

Summary

1. Pre-instrument your code, keeping the baseline
2. Make the Karma server sent the pre-instrumented sources
3. Collect the coverage result with a reporter that takes the baseline into account

1. Pre-Instrumenting

First of all, I did NOT manage to use the coverage preprocessor. Since RequireJS loads sources dynamically, the existing preprocessor is inadequate for our needs. Instead, I first run the instrumenation phase manually with Istanbul, using a custom Grunt plugin:

(check it out here )

module.exports = function(grunt) {
    grunt.registerMultiTask("instrument", "Instrument with istanbul", function() {
        var istanbul = require("istanbul"),
            instrumenter,
            options,
            instrumenterOptions,
            baselineCollector;

        options = this.options({
        });

        if( options.baseline ) {
            baselineCollector = new istanbul.Collector();
        }

        instrumenterOptions = {
            coverageVariable: options.coverageVariable || "__coverage__",
            embedSource: options.embedSource || false,
            preserveComments: options.preserveComments || false,
            noCompact: options.noCompact || false,
            noAutoWrap: options.noAutoWrap || false,
            codeGenerationOptions: options.codeGenerationOptions,
            debug: options.debug || false,
            walkDebug: options.walkDebug || false
        };

        instrumenter = new istanbul.Instrumenter(instrumenterOptions);

        this.files.forEach(function(f) {
            if( f.src.length !== 1 ) {
                throw new Error("encountered src with length: " + f.src.length + ": " + JSON.stringify(f.src));
            }
            var filename = f.src[0],
                code = grunt.file.read(filename, {encoding: grunt.file.defaultEncoding}),
                result = instrumenter.instrumentSync(code, filename),
                baseline,
                coverage;

            if( options.baseline ) {
                baseline = instrumenter.lastFileCoverage();
                coverage = {};
                coverage[baseline.path] = baseline;
                baselineCollector.add(coverage);
            }

            grunt.file.write(f.dest, result, {encoding: grunt.file.defaultEncoding});
        });

        if( options.baseline ) {
            grunt.file.write(options.baseline, JSON.stringify(baselineCollector.getFinalCoverage()), {encoding: grunt.file.defaultEncoding});
        }
    });
};

It is used as:

grunt.initConfig({
    instrument: {
        sources: {
            files: [{
                expand: true,
                cwd: "..the base directory of your sources...",
                src: ["...all your sources..."],
                dest: "...where to put the instrumented files..."
            }],
            options: {
                baseline: "build-coverage/baseline.json" // IMPORTANT!
            }
        }
    },
    ...

It is important to keep the baseline for later.

If not using Grunt I think you can still get ideas from this code. Actually the Istanbul API is pretty decent to work manually, so go ahead and use it if needed.

2. Configuring the Karma server to send the pre-instrumented files

For starters, configure the preprocessor to use your pre-instrumented files ( NOTE we will be using a custom preprocessor and reporter, the code comes at the end):

    ...
    preprocessors: {
        '...all your sources...': 'preInstrumented'
    },

    preInstrumentedPreprocessor: {
        basePath: '!!!SAME AS GRUNT'S dest!!!',
        stripPrefix: '...the base prefix to strip, same as Grunt's cwd...'
    },
    ...

3. Tweaking the reporter to use the baseline

The coverage reporter has to take the baseline into account. Unfortunately the pristine does not, so I tweaked it a bit. The configuration is:

    ...
    reporters: [
        'progress', 'coverage'
    ],

    coverageReporter: {
        type: 'lcov',
        dir: 'build-coverage/report',
        baseLine: '!!!SAME AS GRUNT'S options.baseline!!!'
    },
    ...

Code

To activate my custom Karma plugins, I included this:

    plugins: [
        ...
        require('./build-scripts/karma')
    ],

Where the folder ./build-scripts/karma contains these files:

index.js :

module.exports = {
    "preprocessor:preInstrumented": ["factory", require("./preInstrumentedPreprocessor")],
    "reporter:coverage": ["type", require("./reporter")]
};

preInstrumentedPreprocessor.js :

var path = require("path"),
    fs = require("fs");

createPreInstrumentedPreprocessor.$inject = ["args", "config.preInstrumentedPreprocessor", "config.basePath", "logger", "helper"];
function createPreInstrumentedPreprocessor(args, config, basePath, logger, helper) {
    var STRIP_PREFIX_RE = new RegExp("^" + path.join(basePath, config.stripPrefix).replace(/\\/g, "\\\\"));

    function instrumentedFilePath(file) {
        return path.join(basePath, config.basePath, path.normalize(file.originalPath).replace(STRIP_PREFIX_RE, ""));
    }

    return function(content, file, done) {
        fs.readFile(instrumentedFilePath(file), {encoding:"utf8"}, function(err, instrumentedContent) {
            if( err ) throw err;
            done(instrumentedContent);
        });
    };
}

module.exports = createPreInstrumentedPreprocessor;

reporter.js :

(Check out this issue for the reasons that made me "fork" it.)

// DERIVED FROM THE COVERAGE REPORTER OF KARMA, https://github.com/karma-runner/karma-coverage/blob/master/lib/reporter.js
var path = require('path');
var fs = require('fs');
var util = require('util');
var istanbul = require('istanbul');
var dateformat = require('dateformat');


var Store = istanbul.Store;

var BasePathStore = function(opts) {
    Store.call(this, opts);
    opts = opts || {};
    this.basePath = opts.basePath;
    this.delegate = Store.create('fslookup');
};
BasePathStore.TYPE = 'basePathlookup';
util.inherits(BasePathStore, Store);

Store.mix(BasePathStore, {
    keys : function() {
    return this.delegate.keys();
    },
    toKey : function(key) {
    if (key.indexOf('./') === 0) { return path.join(this.basePath, key); }
    return key;
    },
    get : function(key) {
    return this.delegate.get(this.toKey(key));
    },
    hasKey : function(key) {
    return this.delegate.hasKey(this.toKey(key));
    },
    set : function(key, contents) {
    return this.delegate.set(this.toKey(key), contents);
    }
});


// TODO(vojta): inject only what required (config.basePath, config.coverageReporter)
var CoverageReporter = function(rootConfig, helper, logger) {
    var log = logger.create('coverage');
    var config = rootConfig.coverageReporter || {};
    var basePath = rootConfig.basePath;
    var reporters = config.reporters;
    var baseLine;

    if (config.baseLine) {
    baseLine = JSON.parse(fs.readFileSync(path.join(basePath, config.baseLine), {encoding:"utf8"}));
    }

    if (!helper.isDefined(reporters)) {
    reporters = [config];
    }

    this.adapters = [];
    var collectors;
    var pendingFileWritings = 0;
    var fileWritingFinished = function() {};

    function writeEnd() {
    if (!--pendingFileWritings) {
        // cleanup collectors
        Object.keys(collectors).forEach(function(key) {
        collectors[key].dispose();
        });
        fileWritingFinished();
    }
    }

    /**
    * Generate the output directory from the `coverageReporter.dir` and
    * `coverageReporter.subdir` options.
    *
    * @param {String} browserName - The browser name
    * @param {String} dir - The given option
    * @param {String|Function} subdir - The given option
    *
    * @return {String} - The output directory
    */
    function generateOutputDir(browserName, dir, subdir) {
    dir = dir || 'coverage';
    subdir = subdir || browserName;

    if (typeof subdir === 'function') {
        subdir = subdir(browserName);
    }

    return path.join(dir, subdir);
    }

    this.onRunStart = function(browsers) {
    collectors = Object.create(null);

    // TODO(vojta): remove once we don't care about Karma 0.10
    if (browsers) {
        browsers.forEach(function(browser) {
        collectors[browser.id] = new istanbul.Collector();
        });
    }
    };

    this.onBrowserStart = function(browser) {
    var collector = new istanbul.Collector();
    if( baseLine ) {
        collector.add(baseLine);
    }
    collectors[browser.id] = collector;
    };

    this.onBrowserComplete = function(browser, result) {
    var collector = collectors[browser.id];

    if (!collector) {
        return;
    }

    if (result && result.coverage) {
        collector.add(result.coverage);
    }
    };

    this.onSpecComplete = function(browser, result) {
    if (result.coverage) {
        collectors[browser.id].add(result.coverage);
    }
    };

    this.onRunComplete = function(browsers) {
    reporters.forEach(function(reporterConfig) {
        browsers.forEach(function(browser) {

        var collector = collectors[browser.id];
        if (collector) {
            pendingFileWritings++;

            var outputDir = helper.normalizeWinPath(path.resolve(basePath, generateOutputDir(browser.name,
                                                                                            reporterConfig.dir || config.dir,
                                                                                            reporterConfig.subdir || config.subdir)));

            helper.mkdirIfNotExists(outputDir, function() {
            log.debug('Writing coverage to %s', outputDir);
            var options = helper.merge({}, reporterConfig, {
                dir : outputDir,
                sourceStore : new BasePathStore({
                basePath : basePath
                })
            });
            var reporter = istanbul.Report.create(reporterConfig.type || 'html', options);
            try {
                reporter.writeReport(collector, true);
            } catch (e) {
                log.error(e);
            }
            writeEnd();
            });
        }

        });
    });
    };

    this.onExit = function(done) {
    if (pendingFileWritings) {
        fileWritingFinished = done;
    } else {
        done();
    }
    };
};

CoverageReporter.$inject = ['config', 'helper', 'logger'];

// PUBLISH
module.exports = CoverageReporter;

This was A LOT of code, I know. I wish there was a simpler solution (ideas anybody?). Anyway you can check out how it works with angular-require-lazy to experiment. I hope it helps...