Mongoose：按对象 id 分组并推入一个数组，按 createdAt 排序，填充所有对象并对其进行分页

Question

This is an example of getList of Arrivals by 10 items per page:

router.get('/arrivals', isAuthenticated, async (request, response, next) => {
  jsonPreProcessor.response = response;

  const resPerPage = 10;
  const page = request.query.page || 1;

  Arrival.find({})
    .populate({
      path: 'product',
      populate: {
        path: 'type'
      }
    })
    .select('-__v')
    .skip((resPerPage * page) - resPerPage)
    .limit(resPerPage).then(arrivals => {
      Arrival.countDocuments({}).then(numberOfResults => {
        return jsonPreProcessor.paginate(arrivals, page, Math.ceil(numberOfResults / resPerPage), numberOfResults);
      }).catch(error => {
        return jsonPreProcessor.error(error.message);
      });
    }).catch(error => {
      return jsonPreProcessor.error(error.message);
    });
});

And this is an output:

{
  "time": "2020-01-16T10:11:22.588Z",
  "message": "success",
  "success": true,
  "data": {
    "list": [
      {
        "quantity": 1,
        "discount": 0,
        "_id": "5e0db80a37dd4437b4329960",
        "product": {
          "_id": "5e0cecaaa9a5cc2c7c62e379",
          "title": "Ортопедический",
          "type": {
            "_id": "5ddcbc4685e53838dc564a44",
            "title": "fsdkjhfs",
            "createdAt": "2019-11-26T05:46:46.797Z",
            "updatedAt": "2019-11-26T05:46:46.797Z",
            "alt": "fsdkjhfs",
            "__v": 0
          },
          ...
          "mode": "simple",
          "createdAt": "2020-01-01T19:02:02.840Z",
          "updatedAt": "2020-01-01T19:02:02.840Z",
          "alt": "ortopedicheskij",
          "__v": 0
        },
        ...
        "sellPrice": 6,
        "minSellPrice": 0,
        "createdAt": "2020-01-02T09:29:46.688Z",
        "updatedAt": "2020-01-13T09:30:26.126Z"
      },
      ... // And other 9 items
    ],
    "currentPage": 1,
    "pages": 2,
    "numberOfResults": 16,
    "incompleteResults": true
  },
  "type": null
}

All those are arrivals, and as you see arrivals have product. Some arrivals can have same product (by id)
My problem is to group arrivals which have same product. (sorted by first item of arrivals array) populate all objects.. and paginate. I use aggregate, and unfortunately I don't have any idea how to use it (first of all aggregate's output is random) with pagination and of course output is not what I need. I used this link as solving of my problem.

router.get('/arrivalls', isAuthenticated, async (request, response, next) => {
  jsonPreProcessor.response = response;

  Arrival.aggregate(
    [
      // This is not working (may be it's not working in array)
      // {
      //   "$sort": {
      //     "createdAt": 1
      //   }
      // },
      {
        "$group": {
          "_id": "$product",
          "arrivals": {
            "$push": "$$ROOT"
          }
        }
      },
      // {
      //   "$lookup": {
      //     "from": "arrivals",
      //     "localField": "product",
      //     "foreignField": "_id",
      //     "as": "product"
      //   }
      // },
      // {
      //   "$unwind": {
      //     "path": "$arrivals"
      //   }
      // }
    ]
  ).then(arrivals => {
      // console.log(arrivals);
      return jsonPreProcessor.success(arrivals);
    }).catch(error => {
      return jsonPreProcessor.error(error.message);
    });
});

Output should be like:

  "data": {
    "list": [
    {
      "_id": "5e1d5dba611485397cfb0386",
      "arrivals": [
        {
          "_id": "5e1d5e26611485397cfb0387",
          "quantity": 6,
          "discount": 0,
          "product": {
            "_id": "5e1d5dba611485397cfb0386",
            ... // etc
          },
          "sellPrice": 5000,
          "minSellPrice": 4500,
          "createdAt": "2020-01-14T06:22:30.366Z",
          "updatedAt": "2020-01-14T09:14:13.824Z",
          "__v": 0
        },
        {
          "_id": "5e1ff4d15d059430e8405f94",
          "quantity": 2,
          "discount": 0,
          "product": {
            "_id": "5e1d5dba611485397cfb0386",
            ... // etc
          },
          "sellPrice": 7000,
          "minSellPrice": 6000,
          "comment": "",
          "createdAt": "2020-01-16T05:29:53.907Z",
          "updatedAt": "2020-01-16T05:29:53.907Z",
          "__v": 0
        }
      ]
    },
    {
      "_id": "5e1d84884d387d2334a7e9d9",
      "arrivals": [
        {
          // etc...
        }
      ]
    }
  ],
  "currentPage": 1,
  "pages": 2,
  "numberOfResults": 16,
  "incompleteResults": true
},
"type": null

}

Answer 1

I solved my problem.. And here is grouping, sorting, populating and paginating of list.

  const resPerPage = 10;
  const page = request.query.page || 1;

  Arrival.aggregate(
    [
      {
        $lookup: {
          from: 'products',
          localField: 'product',
          foreignField: '_id',
          as: 'product'
        }
      },
      {
        $unwind: {
          path: "$product",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'branchdans',
          localField: 'branch',
          foreignField: '_id',
          as: 'branch'
        }
      },
      {
        $unwind: {
          path: "$branch",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'colors',
          localField: 'color',
          foreignField: '_id',
          as: 'color'
        }
      },
      {
        $unwind: {
          path: "$color",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'types',
          localField: 'product.type',
          foreignField: '_id',
          as: 'product.type'
        }
      },
      {
        $unwind: {
          path: "$product.type",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'subcategories',
          localField: 'product.subcategory',
          foreignField: '_id',
          as: 'product.subcategory'
        }
      },
      {
        $unwind: {
          path: "$product.subcategory",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'categories',
          localField: 'product.subcategory.category',
          foreignField: '_id',
          as: 'product.subcategory.category'
        }
      },
      {
        $unwind: {
          path: "$product.subcategory.category",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'countries',
          localField: 'product.country',
          foreignField: '_id',
          as: 'product.country'
        }
      },
      {
        $unwind: {
          path: "$product.country",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $lookup: {
          from: 'manufacturers',
          localField: 'product.manufacturer',
          foreignField: '_id',
          as: 'product.manufacturer'
        }
      },
      {
        $unwind: {
          path: "$product.manufacturer",
          preserveNullAndEmptyArrays: true
        }
      },
      {
        $group: {
          "_id": "$product._id",
          "sizes": {
            "$first": "$product.sizes"
          },
          "title": {
            "$first": "$product.title"
          },
          "type": {
            "$first": "$product.type"
          },
          "subcategory": {
            "$first": "$product.subcategory"
          },
          "country": {
            "$first": "$product.country"
          },
          "manufacturer": {
            "$first": "$product.manufacturer"
          },
          "description": {
            "$first": "$product.description"
          },
          "comment": {
            "$first": "$product.comment"
          },
          "mode": {
            "$first": "$product.mode"
          },
          "createdAt": {
            "$first": "$product.createdAt"
          },
          "updatedAt": {
            "$first": "$product.updatedAt"
          },
          "alt": {
            "$first": "$product.alt"
          },
          arrivals: {
            $push: "$$ROOT"
          },
          "date": {
            $last: "$createdAt"
          }
        },
      },
      {
        $unset: "arrivals.product"
      },
      {
        $sort: {
          "date": 1
        }
      },
      {
        $skip: (resPerPage * page) - resPerPage
      },
      {
        $limit: resPerPage
      }
    ]
  ).then(arrivals => {

      Arrival.aggregate([
        {
          $group: {
            "_id": "$product",
            arrivals: {
              $push: "$$ROOT"
            },
            "date": {
              "$last": "$createdAt"
            }
          },
        },
        {
          $sort: {
            "date": 1
          }
        },
        {
          $count: "numberOfResults"
        }
      ]).then(countArrivals => {
        if(countArrivals.length === 0){
          return jsonPreProcessor.error('Ошибка при высчитывании прибытий товаров');
        }
        // Todo make more practise
        return jsonPreProcessor.paginate(arrivals, page, Math.ceil(countArrivals[0].numberOfResults / resPerPage), countArrivals[0].numberOfResults);
      }).catch(error => {
        return jsonPreProcessor.error(error.message);
      });

      // return jsonPreProcessor.success(arrivals);
    }).catch(error => {
      return jsonPreProcessor.error(error.message);
    });