通过github API异步递归以获取文件

Question

I'm using the github API to traverse a repo and get a list of all files in it. This structure is called a "tree". A tree is basically a subdirectory. So if I want see the contents of a tree, I need to make a GET request to the ID of that tree. The response will be an array of objects representing items in that tree. But some of these items will be trees too, so I'll have to make another get request to that tree. A repo may look like this:

|src
    app.jsx
    container.jsx
    |client
        index.html
readme.md

This structure would be represented by the following objects

[
    { name:'src', type:'tree', id:43433432 },
    { name:'readme.md', type:'md', id:45489898 }
]
//a GET req to the id of the first object would return the following array:
[
    { name:'app.jsx', type:'file', id:57473738 },
    { name:'contain.jsx', type:'file', id:748433454 },
    { name:'client', type:'tree', id:87654433 }
]
//a GET req to the id of the third object would return the following  array:
[
    { name:'index.html', type:'file', id:44444422 }
]

What I need to do is write a function that will return an array of the names of all files. This gets pretty tricky, as I'm trying to combine async calls with recursion. This is my attempt so far:

function treeRecurse(tree) {
  let promArr = [];  

  function helper(tree) {    
    tree.forEach(file => {      

      let prom = new Promise((resolve, reject) => {
        if (file.type == `tree`) {
          let uri = treeTrunk + file.sha + `?access_token=${config.ACCESS_TOKEN}`;          

          request({ uri, method: 'GET' })
            .then(res => {
              let newTree = JSON.parse(res.body).tree;              
              resolve(helper(newTree));              
            });

          } else resolve(promArr.push(file.path));
          promArr.push(prom);
      });
    });
  };
  helper(tree);
  Promise.all(promArr)
    .then(resArr => console.log(`treeRecurse - resArr:`, resArr));
};

It's crawling through everything, but the promArr is resolving too quickly. Also, I'm not sure what to pass into resolve. Halp me.

Answer 1

Solution 1:

let username = 'YOUR_USERNAME';
let reponame = 'YOUR_REPONAME';
let access_token = 'YOUR_ACCESS_TOKEN';

const axios = require('axios');

let tree = [];
let fileNames = [];
function start() {
    axios.get(`https://api.github.com/repos/${username}/${reponame}/git/trees/master?access_token=${access_token}`)
      .then(
        function(res) {
          tree = tree.concat(res.data.tree);
          getFilesNameRecur();
        },
        function(err) {
          console.log(err);
        }
      );
}


function getFilesNameRecur() {
  if (tree.length !== 0) {

    let firstObjectOfTree = tree.pop();
    if (firstObjectOfTree.type === 'tree') {
      axios.get(firstObjectOfTree.url + `?access_token=${access_token}`)
        .then(
          function(response) {
            tree = tree.concat(response.data.tree);
            getFilesNameRecur();
          },
          function(err) {
            console.log(err);
          }
        );
    } else if (firstObjectOfTree.type === 'blob') {
      fileNames.push(firstObjectOfTree.path);
      getFilesNameRecur();
    }
  } else {
    // Finished fetching all file names
    console.log(fileNames);
  }
}

start();

---

Solution 2 (preferred):

Use async and await keywords of ES2017.

Documentation: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function

import axios from 'axios';

let username = 'YOUR_USERNAME';
let reponame = 'YOUR_REPONAME';
let access_token = 'YOUR_ACCESS_TOKEN';
let tree = [];
let fileNames = [];
async function start() {
  try {
    let res = await axios.get(`https://api.github.com/repos/${username}/${reponame}/git/trees/master?access_token=${access_token}`);
    tree = tree.concat(res.data.tree);
    while (tree.length !== 0) {
      await getFilesNameRecur();
    }
    console.log(fileNames);
  } catch(e) {
    console.log(e);
  }
}

async function getTreeFromGithub(url) {
  try{
    let response = await axios.get(url + `?access_token=${access_token}`);
    return response.data.tree;
  } catch (e) {
    console.log(e);
    throw e;
  }
}


async function getFilesNameRecur() {
  let firstObjectOfTree = tree.pop();
  if (firstObjectOfTree.type === 'tree') {
    let subTree = await getTreeFromGithub(firstObjectOfTree.url);
    tree = tree.concat(subTree);
  } else if (firstObjectOfTree.type === 'blob') {
    fileNames.push(firstObjectOfTree.path);
  }
}

start();

Answer 2

Interesting problem. The reason why promArr is resolving too quickly, as you might have guessed, is because as soon as you push one Promise into it, Promise.all passes its condition and it doesn't wait around for the other Promises to populate in the array.

I would try re-writing it so that your recursive function helper accepts two params, tree and arr - with arr being your array of Promises. You start by calling the function with helper(tree, []) - and inside, you populate the array with the necessary promises and re-call helper with helper(newTree, updatedArray) . Add some logic that identifies when you are done populating promises into updatedArray , and on that condition, just return the updatedArray full of your Promises.

Then just call Promise.all(helper(tree, [])).then(...) and it should work as intended.

Kinda just talked through it but happy to implement some code if you need it.

Answer 3

Lets mimic the Git database.

var gitFake = {0       : [{ name:'src', type:'tree', id:43433432 },
                          { name:'readme.md', type:'md', id:45489898 }
                         ],
               43433432: [ { name:'app.jsx', type:'file', id:57473738 },
                           { name:'contain.jsx', type:'file', id:748433454 },
                           { name:'client', type:'tree', id:87654433 }
                         ],
               87654433: [ { name:'index.html', type:'file', id:44444422 }
                         ],
               getDir  : function(id,cb){ setTimeout(cb, 250, !this[id] && "Error: No such directory..!", this[id])}
              };

There is also a getDir method included in this library which is asynchronous and would return a directory in say 250ms. I assume gitFake.getDir(id,cb) , where the callback it takes is error first type like cb(err,data) , is not promisified. Lets invent a promisifier for async functions those accept error first type callback;

function promisify(f){
  return data => new Promise((v,x) => f(data, (err,res) => err ? x(err) : v(res)));
}

Now lets create our recursive asynchronous function getAllDirs to list all nested directories;

 function promisify(f){ // utility function to promisify the async functions taking error first callback return data => new Promise((v,x) => f(data, (err,res) => err ? x(err) : v(res))); } function getAllDirs(root = 0){ gd(root).then(function(ds){ ds.length && (console.log(ds), ds.filter( d => d.type === "tree") .forEach(d => getAllDirs(d.id))); }) .catch(e => console.log(e)); } var gitFake = {0 : [{ name:'src', type:'tree', id:43433432 }, { name:'readme.md', type:'md', id:45489898 } ], 43433432: [ { name:'app.jsx', type:'file', id:57473738 }, { name:'contain.jsx', type:'file', id:748433454 }, { name:'client', type:'tree', id:87654433 } ], 87654433: [ { name:'index.html', type:'file', id:44444422 } ], getDir : function(id,cb){ setTimeout(cb, 250, !this[id] && "Error: No such directory..!", this[id])} }, gd = promisify(gitFake.getDir.bind(gitFake)); getAllDirs(); 

 .as-console-wrapper { max-height: 100% !important; top: 0; }