Reshape JSON using lodash and reduce

Question

I want to reshape a JSON from a parsed CSV file downloaded from an URL.

I used 'csvtojson' module to parse CSV and create a JSON that need to be reshaped.

Below my full code:

const _ = require('lodash')
// to handle datetimes
const moment = require('moment')
// to convert csv to json
const csv = require('csvtojson')

// to save files to disk
const fs = require('fs')
const fsPromises = fs.promises;
const path = require('path')

// to load data stream from url
const axios = require('axios')

// to schedule downloading
const schedule = require('node-schedule');
//let t = moment('2020-01-30 02:59:59+01')
// console.log(t.format())
// console.log(t.toDate())

const urlBaseFile = 'http://cemec.arpacampania.it/meteoambientecampania/php/downloadFileDati.php?path=/var/www/html/meteoambientecampania/prodotti/aria/arpac_dati_centraline'

const path_fileJSON = path.resolve(__dirname, '../data.json')
const path_fileCSV = path.resolve(__dirname, '../data.csv')

const downloadCSV = async (date) => {

    const url = urlBaseFile + '_' +
        (!date ? moment().format('YYYYMMDD') : date) + '.csv'

    const writerCSV = fs.createWriteStream(path_fileCSV)

    try {
        let response = await axios.get(url, {
            responseType: 'stream'
        })
        await response.data.pipe(writerCSV)
        const json_from_csv = await response.data.pipe(csv())
        const json_from_csv_filtered = json_from_csv.filter(d => d.descrizione == 'Portici Parco Reggia').map(d => {
            return {
                // convert to Date object UTC format
                // moment(d.data_ora).format()
                datetime: moment(d.data_ora).toISOString(),
                [d.inquinante]: parseFloat(d.valore),
                // pollutant: d.inquinante,
                // value: parseFloat(d.valore), // convert to float
                //unit: d.um
            }
        })

        await fsPromises.writeFile(path_fileJSON, JSON.stringify(json_from_csv_filtered, null, 4))

        const res = json_from_csv_filtered.reduce((acc, val) => {
            const find = acc.find(el => el.date === val.datetime.substring(0, 10));
            const {
                datetime,
                ...obj
            } = val;

            if (find) {
                find.values[0] = {
                    ...find.values[0],
                    ...obj
                };
            } else {
                acc.push({
                    date: datetime.substring(0, 10),
                    values: [{
                        t: datetime,
                        ...obj
                    }]
                });
            }
            return acc;
        }, []);
        console.log(JSON.stringify(res, null, 4))



    } catch (err) {

        console.error(err)
    }



}

downloadCSV()

I obtain:

[
    {
        "date": "2020-06-28",
        "values": [
            {
                "t": "2020-06-28T23:59:59.000Z",
                "Benzene": 0.5,
                "NO2": 44.7,
                "O3": 23.2
            }
        ]
    },
    {
        "date": "2020-06-29",
        "values": [
            {
                "t": "2020-06-29T00:59:59.000Z",
                "Benzene": 4.3,
                "NO2": 11.5,
                "O3": 67.8
            }
        ]
    }
]

The overall JSON schema is correct but I should obtain additional items for date 2020-06-29 while only 1 item is returned.

What's wrong?

Thanks.

Answer 1

From the post and your comments I assumed your response data will be something like below and as far as I understand from your comments, you want to group the values by date from datetime and then having the multiple objects inside the values array of a group according to different hour basis. I tried it with lodash groupBy function. For solution, I approached with two grouping

1. group by date from datetime
2. group values of times of each group date

 let data = [ { "datetime": "2020-06-29T23:59:59.000Z", "Benzene": 1.9 }, { "datetime": "2020-06-30T00:59:59.000Z", "Benzene": 0.6 }, { "datetime": "2020-06-30T01:59:59.000Z", "Benzene": 5.7 }, { "datetime": "2020-06-30T02:59:59.000Z", "Benzene": 5.5 }, { "datetime": "2020-06-30T03:59:59.000Z", "Benzene": 5.2 }, { "datetime": "2020-06-30T04:59:59.000Z", "Benzene": 1 }, { "datetime": "2020-06-30T05:59:59.000Z", "Benzene": 2.1 }, { "datetime": "2020-06-30T06:59:59.000Z", "Benzene": 4.2 }, { "datetime": "2020-06-30T07:59:59.000Z", "Benzene": 3.4 }, { "datetime": "2020-06-30T00:59:59.000Z", "NO2": 16.3 }, { "datetime": "2020-06-30T01:59:59.000Z", "NO2": 12.6 }, { "datetime": "2020-06-30T02:59:59.000Z", "NO2": 11.7 }, { "datetime": "2020-06-30T03:59:59.000Z", "NO2": 35.5 }, { "datetime": "2020-06-30T04:59:59.000Z", "NO2": 44.6 }, { "datetime": "2020-06-30T05:59:59.000Z", "NO2": 19.9 }, { "datetime": "2020-06-30T06:59:59.000Z", "NO2": 11.2 }, { "datetime": "2020-06-30T07:59:59.000Z", "NO2": 8.2 }, { "datetime": "2020-06-29T23:59:59.000Z", "O3": 25.6 }, { "datetime": "2020-06-30T01:59:59.000Z", "O3": 31.7 }, { "datetime": "2020-06-30T02:59:59.000Z", "O3": 35.1 }, { "datetime": "2020-06-30T03:59:59.000Z", "O3": 11.4 }, { "datetime": "2020-06-30T04:59:59.000Z", "O3": 4.9 }, { "datetime": "2020-06-30T05:59:59.000Z", "O3": 32.8 }, { "datetime": "2020-06-30T06:59:59.000Z", "O3": 46.4 }, { "datetime": "2020-06-30T07:59:59.000Z", "O3": 55.6 } ]; const groupsByDate = _.groupBy(data, val => val.datetime.split('T')[0]); const res = Object.entries(groupsByDate).map(([date, vals]) => { const groupsByTime = _.groupBy(vals, val => val.datetime.split('T')[1]); return { date, values: Object.values(groupsByTime).map(arr => Object.assign({}, ...arr)) }; }); console.log(JSON.stringify(res, null, 2));

 <script src="https://cdn.jsdelivr.net/npm/lodash@4.17.15/lodash.min.js"></script>

I also tried to put my solution into your current approach

 const json_from_csv_filtered = [ { "datetime": "2020-06-29T23:59:59.000Z", "Benzene": 1.9 }, { "datetime": "2020-06-30T00:59:59.000Z", "Benzene": 0.6 }, { "datetime": "2020-06-30T01:59:59.000Z", "Benzene": 5.7 }, { "datetime": "2020-06-30T02:59:59.000Z", "Benzene": 5.5 }, { "datetime": "2020-06-30T03:59:59.000Z", "Benzene": 5.2 }, { "datetime": "2020-06-30T04:59:59.000Z", "Benzene": 1 }, { "datetime": "2020-06-30T05:59:59.000Z", "Benzene": 2.1 }, { "datetime": "2020-06-30T06:59:59.000Z", "Benzene": 4.2 }, { "datetime": "2020-06-30T07:59:59.000Z", "Benzene": 3.4 }, { "datetime": "2020-06-30T00:59:59.000Z", "NO2": 16.3 }, { "datetime": "2020-06-30T01:59:59.000Z", "NO2": 12.6 }, { "datetime": "2020-06-30T02:59:59.000Z", "NO2": 11.7 }, { "datetime": "2020-06-30T03:59:59.000Z", "NO2": 35.5 }, { "datetime": "2020-06-30T04:59:59.000Z", "NO2": 44.6 }, { "datetime": "2020-06-30T05:59:59.000Z", "NO2": 19.9 }, { "datetime": "2020-06-30T06:59:59.000Z", "NO2": 11.2 }, { "datetime": "2020-06-30T07:59:59.000Z", "NO2": 8.2 }, { "datetime": "2020-06-29T23:59:59.000Z", "O3": 25.6 }, { "datetime": "2020-06-30T01:59:59.000Z", "O3": 31.7 }, { "datetime": "2020-06-30T02:59:59.000Z", "O3": 35.1 }, { "datetime": "2020-06-30T03:59:59.000Z", "O3": 11.4 }, { "datetime": "2020-06-30T04:59:59.000Z", "O3": 4.9 }, { "datetime": "2020-06-30T05:59:59.000Z", "O3": 32.8 }, { "datetime": "2020-06-30T06:59:59.000Z", "O3": 46.4 }, { "datetime": "2020-06-30T07:59:59.000Z", "O3": 55.6 } ]; const groupBy = (xs, f) => { return xs.reduce((acc, val) => { const key = f(val); const values = [...(acc[key] || []), val]; acc[key] = values; return acc; }, {}); } const groupsByDate = groupBy(json_from_csv_filtered, v => v.datetime.split("T")[0]); const res = Object.entries(groupsByDate).map(([date, values]) => { const groupsByTime = groupBy(values, v => v.datetime.split("T")[1]); return { date, values: Object.values(groupsByTime).map(v => Object.assign({}, ...v)) }; }); console.log(JSON.stringify(res, null, 2));

Check the approach and I hope it will solve your problem as per my understanding.