How to properly walk a disk directory recursively and stream all the paths in relative format into a json using node.js?

Question

I'm trying to read directory tree and save it into a JSON file using node.js with the following logic:

1. Read specified directory
2. Transform (or get in the first place if possible?) the paths into relative format. Something like this:

Directory: C:/test

{
{"C:/test folder 1": [
  {"folder 1": ["file1.txt", "file2.txt"]}, 
  "file1.txt",
  "file2.txt",
  ...
]},
{"C:/test folder 2": [
  {"folder 1": ["file1.txt", "file2.txt"]}, 
  "file1.txt",
  "file2.txt",
  ...
]},
"file1.txt",
"file2.txt",
"file3.txt",
...
}

3. Stream that object into a JSON file (I need to stream it because the object is quite big)

I was trying to use different npm modules like:

walkdir , walker , node-klaw + fs-extra.writeJson , json-object-stream , etc.

But I keep getting different errors trying to stream it into a json.

Answer 1

Following script should work for you, just replace __dirname with absolute path of location that you want to list. And update filename where to write it

I am not checking here if first time call actually gets valid folder, and there might be thing with / regarding on OS

removeNamespace flag here is only so that first level keeps absolute path and not in following

var fs = require("fs");

function getContentForLocation(location, removeNamespace) {
    let result = [];
    let content = fs.readdirSync(location);
    content.forEach(c => {
        if(fs.statSync(`${location}/${c}`).isFile()) result.push(c);
        else {
            let name = removeNamespace ? c : `${location}/${c}`;
            result.push({
                [`${name}`]: getContentForLocation(`${location}/${c}`, true)
            })
        }
    });
    return result;
}
let tree = JSON.stringify(getContentForLocation(__dirname))
fs.writeFileSync(`${__dirname}/test.json`, tree)

Async version:

var fs = require("fs");

function getContentForLocation(location, callback, removeNamespace) {
    let result = [];

    let folderNames = [];
    fs.readdir(location, (err, content) => {
        content.forEach(c => {
            if(fs.statSync(`${location}/${c}`).isFile()) result.push(c);
            else folderNames.push(c);
        });
        if(folderNames.length === 0) callback(result)
        else folderNames.forEach(folder => {
            let name = removeNamespace ? folder : `${location}/${folder}`;
            getContentForLocation(`${location}/${folder}`, resolveCallback.bind(this, name, folder),true)
        })
    });

    function resolveCallback(name, folder, content) {
        result.push({ [name]: content });
        folderNames = folderNames.filter(f => f !== folder);
        if(folderNames.length === 0) callback(result);
    }
}

getContentForLocation(__dirname, results => {
    console.log("resolved", results)
    fs.writeFile(`${__dirname}/test.json`, JSON.stringify(results), (err) => { console.log(err) })
});