MongoDB Mongoose querying a deeply nested array of subdocuments by date range

Question

I have a question that is similar to this other question but not exactly the same because my data structure is more deeply nested, and the accepted answer did not resolve the issue.

Technologies: MongoDB 3.6, Mongoose 5.5, NodeJS 12

I am trying to query a deeply nested array of objects. The query will accept a "Start Date" and an "End Date" from the user. Item Report is an array of subdocuments that contains another array of subdocuments "Work Done By". All WorkDoneBy objects that have a "CompletedDate" in the Start and End date range should be returned along with several other properties.

Desired return properties:

RecordID, RecordType, Status, ItemReport.WorkDoneBy.DateCompleted, ItemReport.WorkDoneBy.CompletedHours, ItemReport.WorkDoneBy.Person

Record schema:

let RecordsSchema = new Schema({
  RecordID: {
    type: Number,
    index: true
  },
  RecordType: {
    type: String,
    enum: ['Item', 'OSW']
  },
  Status: {
    type: String
  },
  // ItemReport array of subdocuments
  ItemReport: [ItemReportSchema],
}, {
  collection: 'records',
  selectPopulatedPaths: false
});

let ItemReportSchema = new Schema({
  // ObjectId reference
  ReportBy: {
    type: Schema.Types.ObjectId,
    ref: 'people'
  },
  ReportDate: {
    type: Date,
    required: true
  },
  WorkDoneBy: [{
    Person: {
      type: Schema.Types.ObjectId,
      ref: 'people'
    },
    CompletedHours: {
      type: Number,
      required: true
    },
    DateCompleted: {
      type: Date
    }
  }],
});

Attempt 1:

db.records.aggregate([
    {
        "$match": {
            "ItemReport.WorkDoneBy.DateCompleted": { "$gt": new Date("2017-01-01T12:00:00.000Z"), "$lt": new Date("2018-12-31T12:00:00.000Z") }
        }
    },
    {
        "$project": {
            "ItemReport.WorkDoneBy": {
                "$filter": {
                    "input": "$ItemReport.WorkDoneBy",
                    "as": "value",
                    "cond": {
                        "$and": [
                            { "$ne": [ "$$value.DateCompleted", null ] },
                            { "$gt": [ "$$value.DateCompleted", new Date("2017-01-01T12:00:00.000Z") ] },
                            { "$lt": [ "$$value.DateCompleted", new Date("2018-12-31T12:00:00.000Z") ] }
                        ]
                    }
                }
            }
        }
    }
])

Attempt 1 returns:

{ "_id" : ObjectId("5dcb6406e63830b7aa54269d"), "ItemReport" : [ { "WorkDoneBy" : [ ] } ] }
{ "_id" : ObjectId("5dcb6406e63830b7aa5426fb"), "ItemReport" : [ { "WorkDoneBy" : [ ] } ] }
{ "_id" : ObjectId("5dcb6406e63830b7aa542708"), "ItemReport" : [ { "WorkDoneBy" : [ ] } ] }
{ "_id" : ObjectId("5dcb6406e63830b7aa542712"), "ItemReport" : [ { "WorkDoneBy" : [ ] } ] }

Desired return (removed _id for brevity):

Note that objects in the WorkDoneBy array should be returned ONLY if they are within the specified date range. For example RecordID 9018 ItemReport.WorkDoneBy actually has dates in 2016 but those are not returned because they are not within the specified date range.

{ "ItemReport" : [ { "WorkDoneBy" : [ { "CompletedHours" : 11, "DateCompleted" : ISODate("2017-09-29T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fd6e") }, { "CompletedHours" : 36, "DateCompleted" : ISODate("2018-05-18T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fd6e") }, { "CompletedHours" : 32, "DateCompleted" : ISODate("2018-05-18T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fd6e") } ] } ], "RecordID" : 9018, "RecordType" : "Item", "Status" : "Done" }
{ "ItemReport" : [ { "WorkDoneBy" : [ { "CompletedHours" : 1.5, "DateCompleted" : ISODate("2017-09-01T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fe5f") } ] } ], "RecordID" : 9019, "RecordType" : "Item", "Status" : "Done" }
{ "ItemReport" : [ { "WorkDoneBy" : [ { "CompletedHours" : 2, "DateCompleted" : ISODate("2017-09-08T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fd6e") }, { "CompletedHours" : 18, "DateCompleted" : ISODate("2017-09-15T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fd6e") }, { "CompletedHours" : 7, "DateCompleted" : ISODate("2017-09-20T04:00:00Z"), "Person" : ObjectId("5dcb6409e63830b7aa54fd6e") } ] } ], "RecordID" : 9017, "RecordType" : "Item", "Status" : "Done" }

Answer 1

The problem here is that WorkDoneBy is an array nested in another array ( ItemReport ). Therefore single $filter is not enough since you need to iterate twice. You can add $map to iterate over the outer array:

db.records.aggregate([
    {
        "$project": {
            "ItemReport": {
                $map: {
                    input: "$ItemReport",
                    as: "ir",
                    in: {
                        WorkDoneBy: {
                            $filter: {
                                input: "$$ir.WorkDoneBy",
                                as: "value",
                                cond: {
                                    "$and": [
                                        { "$ne": [ "$$value.DateCompleted", null ] },
                                        { "$gt": [ "$$value.DateCompleted", new Date("2017-01-01T12:00:00.000Z") ] },
                                        { "$lt": [ "$$value.DateCompleted", new Date("2018-12-31T12:00:00.000Z") ] }
                                    ]
                                }
                            }
                        }
                    }
                }
            }
        }
    }
])

Answer 2

please check below:

db.collection.aggregate([
  {
    $project: {
      _id: 0,
      RecordID: 1,
      RecordType: 1,
      Status: 1,
      ItemReport: {
        $let: {
          vars: {
            wdb: {
              $reduce: {
                input: "$ItemReport.WorkDoneBy",
                initialValue: [],
                in: {
                  $concatArrays: [
                    "$$this",
                    "$$value"
                  ]
                }
              }
            }
          },
          in: {
            WorkDoneBy: {
              $filter: {
                input: "$$wdb",
                as: "item",
                cond: {
                  $and: [
                    {
                      $gte: [
                        "$$item.DateCompleted",
                        ISODate("2010-01-10T04:00:00Z") // start date
                      ]
                    },
                    {
                      $lte: [
                        "$$item.DateCompleted",
                        ISODate("2018-01-10T04:00:00Z") // end date
                      ]
                    },
                  ]
                }
              }
            }
          }
        }
      }
    }
  }
])

mongodb playground