Compare arrays and filter, using MongoDB aggregation

Question

For my DB, I wrote the following pipeline:

let orders = await Order.aggregate(
        {
          $unwind: "$candidate",
        },
        {
          $lookup: {
            from: "groups",
            localField: "candidate.groupId",
            foreignField: "_id",
            as: "groupData",
          },
        },
        {
          $lookup: {
            from: "users",
            let: {
              id: "$candidate.groupId",
            },
            pipeline: [
              { $match: { groupId: { $ne: null } } },
              {
                $match: {
                  $expr: { $in: ["$$id", "$groupId"] },
                },
              },
              { $project: { name: 1, email: 1, _id: 1 } },
            ],
            as: "members",
          },
        },
        { $match: { "members._id": new ObjectId(req.userId) } },
        {
          $lookup: {
            from: "users",
            let: { ids: "$candidate.autonomousId" },
            pipeline: [
              {
                $match: {
                  $expr: { $in: ["$_id", "$$ids"] },
                },
              },
              { $project: { name: 1, email: 1, _id: 1 } },
            ],
            as: "candidate",
          },
        },
        {
          $project: {
            groupData: 1,
            members: 1,
            candidate: 1,
            stillAvailable: 1,
            _id: 0,
          },
        }
      ).toArray();

The output was the expected...

{ candidate:
    [ { _id: 601817dc2eeecd17db3a68f6,
        name: 'Maria' },
      { _id: 601817ef2eeecd17db3a68f7,
        name: 'Jose' } ],
   groupData:
    [ { _id: 606632403fffb851b8c41d12,
        name: 'Giraia' } ],
   members:
    [ { _id: 601817dc2eeecd17db3a68f6,
        name: 'Maria' },
      { _id: 601817ef2eeecd17db3a68f7,
        name: 'Jose' },
      { _id: 60182cbb2b654330d2458f89,
        name: 'Jonas'} ] } 

The last step in the pipeline would be to compare the arrays, filter which members were not candidates and add them to the array stillAvailable . I tried in many ways but I couldn't achieve my goal with aggregation. The only solution I could find was to process the result of the incomplete pipeline on my backend. The code is:

  orders.forEach(
    (order) =>
      (order.stillAvailable = order.members.filter(
        (autonomous) =>
          !order.candidate.some((el) => {
            return el._id.toString() === autonomous._id.toString();
          })
      ))
  );

With that, I reach the expected output...

 { candidate:
    [ { _id: 601817dc2eeecd17db3a68f6,
       name: 'Maria' },
      { _id: 601817ef2eeecd17db3a68f7,
       name: 'Jose' } ],
   groupData:
    [ { _id: 606632403fffb851b8c41d12,
        name: 'Giraia' ],
   members:
    [ { _id: 601817dc2eeecd17db3a68f6,
        name: 'Maria' },
      { _id: 601817ef2eeecd17db3a68f7,
        name: 'Jose' },
      { _id: 60182cbb2b654330d2458f89,
        name: 'Jonas' ],
   stillAvailable:
    [ { _id: 60182cbb2b654330d2458f89,
        name: 'Jonas' ] }

The problem is to better compartmentalize my code, it would be necessary to realize the last step (done with javascript on my backend) as one more step on the pipeline. Does anyone have an idea how to reach that?

Answer 1

After I wrote the question here, somehow the idea was better structured and I achieved the result, using $map and one more level of $lookup . I left the answer documented here in case someone falls into the same issue.

  let orders = await Order.aggregate(
    {
      $unwind: "$candidate",
    },
    {
      $lookup: {
        from: "groups",
        localField: "candidate.groupId",
        foreignField: "_id",
        as: "groupData",
      },
    },
    {
      $lookup: {
        from: "users",
        let: {
          id: "$candidate.groupId",
        },
        pipeline: [
          { $match: { groupId: { $ne: null } } },
          {
            $match: {
              $expr: { $in: ["$$id", "$groupId"] },
            },
          },
          { $project: { name: 1, email: 1, _id: 1 } },
        ],
        as: "members",
      },
    },
    { $match: { "members._id": new ObjectId(req.userId) } },
    {
      $lookup: {
        from: "users",
        let: { ids: "$candidate.autonomousId" },
        pipeline: [
          {
            $match: {
              $expr: { $in: ["$_id", "$$ids"] },
            },
          },
          { $project: { name: 1, email: 1, _id: 1 } },
        ],
        as: "candidate",
      },
    },
    {
      $project: {
        groupData: 1,
        members: 1,
        candidate: 1,
        _id: 0,
        stillAvailable: {
          $setDifference: [
            {
              $map: {
                input: "$members",
                as: "member",
                in: "$$member._id",
              },
            },
            {
              $map: {
                input: "$candidate",
                as: "el",
                in: "$$el._id",
              },
            },
          ],
        },
      },
    },
    {
      $lookup: {
        from: "users",
        let: {
          ids: "$stillAvailable",
        },
        pipeline: [
          {
            $match: {
              $expr: { $in: ["$_id", "$$ids"] },
            },
          },
          { $project: { name: 1, email: 1, _id: 1 } },
        ],
        as: "stillAvailable",
      },
    }
  ).toArray();