Parsing JSON from DB and displaying on the front-end

Question

Okay, so basically I have a web scraper, and I save the web scraper information into a local database called "db.json"

I was wondering at how about to tackle into implementing it and having the information display in the view. I'm used to working in like PSQL, so doing this is different for me.. I appreciate the help and I plan on setting up like cron jobs and stuff like that as well but I'll do that later on.

The current partial-jobs maps through out PSQL database and grabs example information right now so it will be changed..

I was also having trouble getting it to the grab the jobs title for more.. organized json but for exmaple

job: jobs.title

which I figured would get the jobs title didn't get anything is it because I have it outside of the function?

Here is the code :

The pushing of the information to the DB is located at the bottom of the searchJobs function.

webScraper.js :

debug = require("../models/conn");
const puppeteer = require("puppeteer");
const axios = require("axios");
const cheerio = require("cheerio");
const db = require("../public/scripts/db")

async function searchJobs(i) {
  const url = await axios
    .get("https://indeed.com/jobs?q=Web+Developer&l=Atlanta&fromage=last")
    .then(response => response)

    .then(res => {
      const jobs = [];
      const $ = cheerio.load(res.data);

      $(".result").each((index, element) => {
        const title = $(element)
          .children(".title")
          .text();
        const linkToJob = $(element)
          .children(".title")
          .children("a")
          .attr("href");
        const body = $(element)
          .children(".summary")
          .text();
        jobs[index] = { title, linkToJob, body };
      });
      console.log(jobs);
      // Push jobs to JSON DB
      db.get('jobs').push({
        job: jobs
      }).write();
      return jobs;
    });
  return url;
}

This is the script that writes the information to a json file.

db.js :

low = require("lowdb"),
    FileSync = require("lowdb/adapters/FileSync");


const adapter = new FileSync('db.json')
const db = low(adapter)

db.defaults({ jobs: [], body: []})
    .write()


module.exports = db;

module.exports = searchJobs;

This is the jobs route

jobs.js :

    const express = require("express"),
  router = express.Router();
jobModel = require("../models/jobModel");

// gets job page
router.get("/", async function(req, res) {
  const jobData = await jobModel.getAllJobs();

  console.log(jobData);

  res.render("template", {
    locals: {
      title: "jobs",
      jobData: jobData
    },
    partials: {
      partial: "partial-jobs"
    }
  });
});

module.exports = router;

Answer 1

You don't need to create any kind of models structure for such a simple task using lowdb .

Your searchJobs sets jobs wrong inside db.json , and that's because at the end of the scraping you push the whole jobs array to a field named job that belongs to the jobs . That will obviously end up with data like this:

{
  "jobs": [
    {
      "job": [
        {
          "title": "...",
          "linkToJob": "...",
          "body": "..."
        },
        {
          "title": "...",
          "linkToJob": "...",
          "body": "..."
        },
        ...
    }
  ],
  "body": []
}

which is not what you want. So, instead of using:

db.get('jobs').push({
  job: jobs
}).write();

you must use:

db.set('jobs', jobs).write();

and then you'll have the data JSON format like this:

{
  "jobs": [
    {
      "title": "...",
      "linkToJob": "...",
      "body": "..."
    },
    {
      "title": "...",
      "linkToJob": "...",
      "body": "..."
    },
    ...
  ],
  "body": []
}

and now you have a proper jobs collection which you can use it to display the data.

Express server has a template engine ( "Using template engines with Express" ) which supports ( among others ) EJS templates. You can use an EJS template and get/pass jobs to it inside the root route:

Code for express server file server.js

const low = require("lowdb");
const FileSync = require("lowdb/adapters/FileSync");
const express = require('express');
const app = express();
const port = 3000;

const adapter = new FileSync('db.json');
const db = low(adapter);

// Set express views template engine to EJS
app.set('view engine', 'ejs');

app.get('/', (req, res) => {
  // Get the jobs collection
  const jobs = db.get('jobs').value();

  // Render the jobs EJS template by passing the jobs
  res.render("jobs", { jobs });
});

app.listen(port, () => console.log(`Listening on port ${port}!`))

EJS template for rendering jobs jobs.ejs :

...
<body>
  <section id="jobs">
  <% for(const job of jobs) {%>
    <div class="job">
      <a href="<%= job.linkToJob %>"><h3><%= job.title %></h3></a>
      <p><%= job.body %></p>
    </div>
  <% } %>
  </section>
</body>
...

And the final webScraper.js :

const axios = require("axios");
const cheerio = require("cheerio");
const db = require("./db");

async function searchJobs() {
  const url = await axios
    .get("https://indeed.com/jobs?q=Web+Developer&l=Atlanta&fromage=last")
    .then(response => response)

    .then(res => {
      const jobs = [];
      const $ = cheerio.load(res.data);

      $(".result").each((index, element) => {
        const title = $(element)
          .children(".title")
          .text();
        const linkToJob = $(element)
          .children(".title")
          .children("a")
          .attr("href");
        const body = $(element)
          .children(".summary")
          .text();
        jobs.push({ title, linkToJob, body });
      });

      // Push jobs to JSON DB
      db.set('jobs', jobs).write();

    });
}

Now if you start the express server and visit the root route, you'll see something like this ( after you've run webScraper.js of course ):

 .job { margin-bottom: 10px; border: 1px grey solid; padding: 10px; border-radius: 5px; background-color: lightgray; }

 <section id="jobs"> <div class="job"> <a href="/rc/clk?jk=45633fe1e5f39cc8&amp;fccid=0e1982cac02545cc&amp;vjs=3"> <h3>Freelance Web Developer</h3> </a> <p>Extensive knowledge of HTML, CSS, Javascript/jQuery.</p> </div> <div class="job"> <a href="/rc/clk?jk=b554d8be38d65cba&amp;fccid=8c101aef95dbfdf6&amp;vjs=3"> <h3>Web Developer</h3> </a> <p>VenU is looking for a talented and reliable developer to join an elite development team. Applicants must be proficient in Responsive Design, with experience…</p> </div> <div class="job"> <a href="/rc/clk?jk=8edc3f88b6ec3083&amp;fccid=9899b2a8ca7c5e21&amp;vjs=3"> <h3>Web Developer</h3> </a> <p>We&#39;re looking for a web developer with an excellent eye for design as well as strong HTML &amp; CSS skills. The web developer will be responsible for creating new… </p> </div> </section>