How can I read all JSON files from a folder in Meteor?

Question

I want to read all json files inside a folder in my Meteor application.

I have the following structure:

/server  
-- /methods  
-- -- file1.json  
-- -- file2.json  

I try to read all JSON files using the following code:

var fs = Npm.require('fs');
var path = Npm.require('path');
var base = path.resolve('.');

try {
    var files = fs.readdirSync(base + '/methods/*.json');
    console.log(files);
} catch (e) {
    console.dir(e)
}

But this doesn't work and it shows me an error saying that the directory or file doesn't exist.

Am I making some mistake? Or is there another way to do that?

Answer 1

First of all, please be careful with finding Meteor's project root, because the outcome of path.resolve('.') or even process.env.PWD may change in different deployment setups.

Secondly, the argument path of fs.readdirSync(path) needs a directory. As a result, the proper call would be var files = fs.readdirSync(base + '/server/methods/'); .

However, I recommend to use Assets . Just move your JSON files to your private directory and access them on the server via Assets.getText(assetPath, [asyncCallback]) or Assets.getBinary(assetPath, [asyncCallback]) .

For example:

if (Meteor.isServer) {
    Meteor.startup(function() {
        var example1 = JSON.parse(Assets.getText('methods/example1.json'));
        var example2 = JSON.parse(Assets.getText('methods/example2.json'));
        console.log(example1);
        console.log(example2);
    });
}

If you want to read all JSON files, you may need the following workaround:

if (Meteor.isServer) {
    Meteor.startup(function() {
        var exec = Npm.require('child_process').exec;
        var files = [],
            fileNames = [];
        exec('ls -m assets/app/methods | tr -d \' \n\' ', Meteor.bindEnvironment(function(error, stdout, stderr) {
            if (error !== null) {
                console.log('exec error: ' + error);
            }
            fileNames = stdout.split(',');
            /* Print all file names. */
            console.log("File names:");
            console.log(fileNames);
            _.each(fileNames, function(fileName) {
                /* Check if file has proper extension. */
                if (fileName.split('.').pop() == 'json') files.push(JSON.parse(Assets.getText('methods/' + fileName)));
            });
            /* Print all JSON files. */
            _.each(files, function(file) {
                console.log(file);
            });
        }));
    });
}

If you want to make the exec call synchronously, you may need to use Meteor.wrapAsync(func, [context]) :

if (Meteor.isServer) {
    var exec = Npm.require('child_process').exec;
    var files = [], fileNames = [];

    var execAsync = function (options, callback) {
        console.log("execAsync()");
        exec('ls -m assets/app/methods | tr -d \' \n\' ', Meteor.bindEnvironment(function (error, stdout, stderr) {
            if (error !== null) {
                console.log('exec error: ' + error);
            }
            fileNames = stdout.split(',');
            /* Print all file names. */
            console.log("File names:");
            console.log(fileNames);
            _.each(fileNames, function (fileName) {
                /* Check if file has proper extension. */
                if (fileName.split('.').pop() == 'json') files.push(JSON.parse(Assets.getText('methods/' + fileName)));
            });
            callback(null, options.callback);
        }));
    };

    function postProcessing(callback) {
        console.log("postProcessing()");
        /* Print all JSON files. */
        _.each(files, function (file) {
            console.log(file);
        });
        callback();
    }

    Meteor.startup(function () {
        /*  Wrap asynchronous exec function, in order to call it in a synchronous style. */
        var execSync = Meteor.wrapAsync(execAsync);
        var refToPostProcessing = execSync({callback: postProcessing});
        var postProcessingSync = Meteor.wrapAsync(refToPostProcessing);
        postProcessingSync();
    });

}

Here is my server output:

I20150919-09:27:09.189(2)? execAsync()        
I20150919-09:27:09.210(2)? File names:
I20150919-09:27:09.213(2)? [ 'example1.json', 'example2.json' ]
I20150919-09:27:09.215(2)? postProcessing()
I20150919-09:27:09.217(2)? { name: 'Product',
I20150919-09:27:09.217(2)?   properties: 
I20150919-09:27:09.218(2)?    { id: 
I20150919-09:27:09.218(2)?       { type: 'number',
I20150919-09:27:09.218(2)?         description: 'Product identifier',
I20150919-09:27:09.218(2)?         required: true },
I20150919-09:27:09.218(2)?      name: 
I20150919-09:27:09.218(2)?       { description: 'Name of the product',
I20150919-09:27:09.219(2)?         type: 'string',
I20150919-09:27:09.219(2)?         required: true },
I20150919-09:27:09.219(2)?      price: { type: 'number', minimum: 0, required: true },
I20150919-09:27:09.219(2)?      tags: { type: 'array', items: [Object] } } }
I20150919-09:27:09.220(2)? { red: '#f00',
I20150919-09:27:09.221(2)?   green: '#0f0',
I20150919-09:27:09.221(2)?   blue: '#00f',
I20150919-09:27:09.221(2)?   cyan: '#0ff',
I20150919-09:27:09.221(2)?   magenta: '#f0f',
I20150919-09:27:09.221(2)?   yellow: '#ff0',
I20150919-09:27:09.221(2)?   black: '#000' }

Assuming you have the following structure:

your-meteor-project
├── .meteor
├── server
├── private
│   └── methods
│       └── example1.json
│       └── example2.json
└── …

Answer 2

Based on Matthias Eckhart's answer above, I wrote a method to load all csv files in a folder, convert the contents to JSON and return data to the client as an object with a child object per csv file. I post this here in case it helps somebody else: there are a couple of wrinkles using Assets and methods, and when converting csv to JSON.

import CSVToJSON from 'csvtojson';

if (Meteor.isServer) {
    const getParameterFilenames = (options, callback) => {
        // read the contents of the 'private/parameters' folder
        const { exec } = Npm.require('child_process');

        // "ls -m" will return directories as well as folders, so make sure to filter the results before loading files
        exec('ls -m assets/app/parameters | tr -d \' \n\' ', Meteor.bindEnvironment((error, stdout, stderr) => {
            if (error !== null) {
                console.log(`Error in getParameterFilenames: ${error}`);
            }
            const filenames = stdout.split(',');
            callback(null, filenames);
        }));
    };

    Meteor.methods({
        'loadParameters'() {
            const syncFunc = Meteor.wrapAsync(getParameterFilenames);
            const filenames = syncFunc({});
            // load parameters from csv files in 'private/parameters'
            // this will be assets/app/parameters in the built app

            // csv file contains key / value pairs
            // first row is key, second row is value
            // first key must be propertyName which will be the key for this sheet's child object in parameters, e.g.:

            /*
            "propertyName","value1","value2"
            "map",10,20
            */

            const promises = [];
            const filesData = [];

            // although Assets.getText is used synchronously, the files must be retrieved before creating the promises
            filenames.forEach((filename) => {
                if (filename.split('.').pop() === 'csv') {
                    filesData.push(Assets.getText(`parameters/${filename}`));
                }
            });

            filesData.forEach((data) => {
                promises.push(CSVToJSON().fromString(data));
            });

            // Meteor will wait for Promise.all to resolve before returning the result to the client
            return Promise.all(promises)
                .then((results) => {
                    // aggregate results into an object
                    const parameters = {};

                    results.forEach((result) => {
                        const data = result[0];
                        const parameter = { ...data };
                        delete parameter.propertyName;
                        parameters[data.propertyName] = parameter;
                    });

                    return parameters;
                });
        },
    });
}