How do i implement callback/await to my code?

Question

So, I have been working on this code for the past few days and tried to implement callback/await/whatever is needed to no avail.

The question is, how do i wait with response until i get lets say a callback from both functions? (and how would i implement that)

In short what I want to do is:

1. POST happens, getpTracks()
2. From getpTracks() we go on to dbChecks() for each track we got
3. Check from db if track is found, if it is push to final array.
4. If not, go to scrapeUrl()
5. After both functions are done running, response to POST with final array.

I've band-aided the code with some funky code checking every 250ms if the amount of tracks requested matches the amount of tracks in final array. But that isn't really what i'm looking for since if theres more than one POST request at a time it gets really broken.

Current code:

app.post("/spotify/playlist", (req, res) => {
  pTracks = [];
  let playlistId = req.body.id;
  t0 = performance.now();
  getpTracks(playlistId);
  let waitTime = setInterval(waitTillReady, 250);
  function waitTillReady() {
    counterLoop++;
    console.log("a: " + countera + "||" + pTracks.length);
    console.log("b: " + counterb + "||" + pTracks.length);
    console.log("a + b: " + (countera + counterb) + "||" + pTracks.length);
    console.log("Loop: " + counterLoop);
    // REPLACE WITH ASYNC OR CALLBACK!!
    if (
      countera == pTracks.length ||
      counterb == pTracks.length ||
      countera + counterb == pTracks.length ||
      counterLoop == 35 // 75 items scraping took on avg 4.8sec
    ) {
      countera = 0;
      counterb = 0;
      clearInterval(waitTime);
      res.send(pTracks);
      t1 = performance.now();
      console.log("Call took " + (t1 - t0) + " milliseconds.");
      pTracks = [];
    }
  }
});

function getpTracks(args) {
  spotifyApi.getPlaylistTracks(args, { limit: 75 }).then(function(data) {
    let temp = data.body.items;
    for (let b = 0; b < temp.length; b++) {
      let trackName = temp[b].track.name;
      for (let e = 0; e < temp[b].track.artists.length; e++) {
        var trackArtist = temp[b].track.artists[e].name;
      }
      dbChecks(trackName, trackArtist);
      //let trackId = temp[b].track.id + ",";
      //spotifyApi.getAudioFeaturesForTracks([trackId]).then(function(data) { // bpm, key etc
    }
  });
}

function dbChecks(trackName, trackArtist) {
  url = "https://www.songsterr.com/a/wa/bestMatchForQueryString?s=";
  //console.log(trackArtist + '|||' + trackName);
  //console.log(url)
  dbSongsterr
    .findOne({ artist: trackArtist, track: trackName }) // get results from mongo
    .then(response => {
      if (
        //if we find results,
        response != null &&
        response.artist == trackArtist &&
        response.track == trackName
      ) {
        countera++;
        pTracks.push({
          //push them into array
          artist: response.artist,
          name: response.track,
          url: response.url,
          tuning: response.tuning
        });
      } else if (response == null) {
        //if no results found, go webscrape
        urli = url + trackName + "&a=" + trackArtist; // url constructor
        pTracks.push({
          artist: trackArtist,
          name: trackName,
          url: urli
        });
        scrapeUrl(urli, trackName, trackArtist);
      }
    })
    .catch(error => {
      console.log("Error: " + error);
    });
}

function scrapeUrl(url, track, artist) {
  url = url;
  console.log(artist + "|||" + track);
  rp({
    url: url,
    resolveWithFullResponse: true,
    transform: function(body) {
      return cheerio.load(body);
    }
  })
    .then(async res => {
      counterb++;
      tempartist = artist.replace(/\s+/g, "");
      artistSongsterr = await res(".artist-1u304B") // tab page artist name
        .text()
        .replace(/\s+/g, "");
      //console.log(artistSongsterr);
      if (
        artistSongsterr != "" &&
        artistSongsterr.toLowerCase() == tempartist.toLowerCase()
      ) {
        // maybe add check for song aswell
        tuning = res(".tuning-1cQdvc").text(); // tab page tuning
        //console.log(tuning);
        if (tuning != "") {
          for (let y = 0; y < pTracks.length; y++) {
            if (pTracks[y].name == track && pTracks[y].tuning == null) {
              pTracks[y] = { ...pTracks[y], ...{ tuning: tuning } };
              dbSongsterr.insert({
                artist: artist,
                track: track,
                url: url,
                tuning: tuning
              });
            }
          }
        }
      } else {
        dbSongsterr.insert({
          // if didnt pass artist name check then
          artist: artist,
          track: track,
          url: false,
          tuning: false
        });
        //console.log('Artist check fail')
      }
    })
    .catch(err => {
      counterb++;
      console.log("Site crawl fail");
      pTracks.push({
        artist: track,
        name: track,
        url: false,
        tuning: false
      });
      dbSongsterr.insert({
        artist: artist,
        track: track,
        url: false,
        tuning: false
      });
    });
}

Answer 1

A few general advices:

• You usually never need globals (so declare them properly), functions should return the results they produce, if variables should beshared between multiple functions pass them explicitly.

• Don't mix .then chains with await s, it looks ugly, is confusing and might introduce subtle bugs.

That said your functions should be async:

async function scrapeUrl(url, track, artist, result = []) {
  console.log(artist + "|||" + track);

  try {

   const res = await rp({
    url,
    resolveWithFullResponse: true,
    transform: function(body) {
      return cheerio.load(body);
    },
   });

   const tempartist = artist.replace(/\s+/g, "");
   const artistSongsterr = await res(".artist-1u304B") // tab page artist name
          .text()
          .replace(/\s+/g, "");

      if (artistSongsterr  && artistSongsterr.toLowerCase() == tempartist.toLowerCase()) {
        // maybe add check for song aswell
        const tuning = res(".tuning-1cQdvc").text(); // tab page tuning

        if (tuning) {
          for (const el of pTracks) {
            if (el.name == track && !el.tuning) {
              el.tuning = tuning;

              result.push({ url, track, artist, tuning });
            }
          }
        }
      } else {
        result.push({
          // if didnt pass artist name check then
          artist,
          track,
          url: false,
          tuning: false
        });
        //console.log('Artist check fail')
      }
    }
   } catch(error) {
      console.log("Site crawl fail");
      result.push({
        artist: track,
        name: track,
        url: false,
        tuning: false
      });

   }

   return result;
}


async function dbChecks(trackName, trackArtist, result = []) {
  const url = "https://www.songsterr.com/a/wa/bestMatchForQueryString?s=";

  try {
    const response = await dbSongsterr
      .findOne({ artist: trackArtist, track: trackName }) // get results from mongo

    if (
      //if we find results,
      response &&
      response.artist == trackArtist &&
      response.track == trackName
    ) {

      result.push({
        //push them into array
        artist: response.artist,
        name: response.track,
        url: response.url,
        tuning: response.tuning
      });
  } else {
    //if no results found, go webscrape
    const urli = url + trackName + "&a=" + trackArtist; // url constructor
    result.push({
      artist: trackArtist,
      name: trackName,
      url: urli
    });
    await scrapeUrl(urli, trackName, trackArtist, result);
  }
 } catch(error) {
  console.log("Error: " + error);
 }

 return result;
}


async function getpTracks(args) {
  const result = [];

  const data = await spotifyApi.getPlaylistTracks(args, { limit: 75 });
  let temp = data.body.items;

  for (const { track: { name: trackName, artists  }} of  temp) {

    const trackArtist = artists[artists.length - 1].name;
    // TODO: use Promise.all to parallelize
    await dbChecks(trackName, trackArtist, result);

  }

  return result;
}

That can be used in the endpoint as:

app.post("/spotify/playlist", async (req, res) => {
  res.send(await getpTracks(req.body.id));
});