How can apply a group by to a result of another group by using MongoDb?

Question

Let's suppose I have the following POCO's representing documents in Mongo:

public class A
{
    [BsonId, BsonRepresentation(BsonType.ObjectId)]
    public string Id { get; set; }
    public string Name { get; set; }
    public string LastName { get; set; }
    public bool Locked { get; set; }
}
public class B
{
    [BsonId, BsonRepresentation(BsonType.ObjectId)]
    public string Id { get; set; }
    public string Name { get; set; }
}
public class C
{
    [BsonId, BsonRepresentation(BsonType.ObjectId)]
    public string Id { get; set; }
    public string Name { get; set; }
}
public class AB
{
    [BsonId, BsonRepresentation(BsonType.ObjectId)]
    public string Id { get; set; }
    [BsonRepresentation(BsonType.ObjectId)]
    public string AId { get; set; }
    [BsonRepresentation(BsonType.ObjectId)]
    public string BId { get; set; }
}
public class AC
{
    [BsonId, BsonRepresentation(BsonType.ObjectId)]
    public string Id { get; set; }
    [BsonRepresentation(BsonType.ObjectId)]
    public string AId { get; set; }
    [BsonRepresentation(BsonType.ObjectId)]
    public string CId { get; set; }
}

The relationships are A many-to-many B and A many-to-many C . My goal is to query all A documents that aren't locked and include all B 's and C 's related to the A document. For that I've tried a lot of attempts but I'll post the most (I think) completed one:

var q = from a in A_Col.AsQueryable()
     where !a.Locked
     join ab in AB_Col.AsQueryable() on a.Id equals ab.AId
     join b in B_Col.AsQueryable() on ab.BId equals b.Id into Bs
     group new { a.Id, a.Name, a.LastName, Bs = Bs.DefaultIfEmpty() }
     by new { a.Id, a.Name, a.LastName } into ABs
     from abs in ABs.DefaultIfEmpty()
     join ac in AC_Col.AsQueryable() on abs.Id equals ac.AId
     join c in C_Col.AsQueryable() on ac.CId equals c.Id into Cs
     group new { abs.Id, abs.Name, abs.LastName, abs.Bs, Cs = Cs.DefaultIfEmpty() }
     by new { abs.Id, abs.Name, abs.LastName };

When I try to execute the query I get the following error:

The method GroupBy is not supported in the expression tree: aggregate([]).Where(a => Not(a.Locked)).Join(aggregate([]), a => a.Id, ab => ab.AId, (a, ab) => new <>f__AnonymousType0 2(a = a, ab = ab)).GroupJoin(aggregate([]), <>h__TransparentIdentifier0 => <>h__TransparentIdentifier0.ab.BId, b => b.Id, (<>h__TransparentIdentifier0, Bs) => new <>f__AnonymousType1 2(<>h__TransparentIdentifier0 = <>h__TransparentIdentifier0, Bs = Bs)).GroupBy(<>h__TransparentIdentifier1 => new <>f__AnonymousType3 3(Id = <>h__TransparentIdentifier1.<>h__TransparentIdentifier0.a.Id, Name = <>h__TransparentIdentifier1.<>h__TransparentIdentifier0.a.Name, LastName = <>h__TransparentIdentifier1.<>h__TransparentIdentifier0.a.LastName), <>h__TransparentIdentifier1 => new <>f__AnonymousType2 4(Id = <>h__TransparentIdentifier1.<>h__TransparentIdentifier0.a.Id, Name = <>h__TransparentIdentifier1.<>h__TransparentIdentifier0.a.Name, LastName = <>h__TransparentIdentifier1.<>h__TransparentIdentifier0.a.LastName, Bs = <>h_ _TransparentIdentifier1.Bs.DefaultIfEmpty()))

I also noted that the resolved GroupBy is coming from System.Linq namespace (not 100% sure) and not from MongoDB.Driver.Linq namespace. After a while modifying here and there I tried to do so in the following way:

var q = A_Col.AsQueryable().Where(a => !a.Locked)
    .Join(
       AB_Col.AsQueryable(),
       a => a.Id, 
       ab => ab.AId,
       (a, ab) => new
       {
          A = new { a.Id, a.Name, a.LastName },
          AB = new { ab.AId, ab.BId }
       })
    .Join(
       B_Col.AsQueryable(), 
       a_doc => a_doc.AB.BId,
       b => b.Id, 
       (a, b) => new { a.A.Id, a.A.Name, a.A.LastName, B = new { b.Name, b.Id } })
    .GroupBy(doc => new { doc.Id, doc.Name, doc.LastName })
    .Join(
        AC_Col.AsQueryable(),
        g => g.Key.Id,
        ac => ac.AId,
        (g, ac) => new
        {
           g.Key.Id,
           g.Key.Name,
           g.Key.LastName,
           Bs = g.Select(x => x.B),
           AC = new { ac.AId, ac.CId }
        })
    .Join(
        C_Col.AsQueryable(),
        gac => gac.AC.CId,
        c => c.Id,
        (gac, c) => new { gac.Id, gac.Name, gac.LastName, gac.Bs, C = new { c.Name, c.Id } })
    .GroupBy(doc => new { doc.Id, doc.Name, doc.LastName, doc.Bs })
    .Select(g => new { g.Key.Id, g.Key.Name, g.Key.LastName, g.Key.Bs, Cs = g.Select(x => x.C) });

And now the error is

$project or $group does not support {document}.

I've seen this one before but in the above code I can't see why it's happening. Can you help me out? It's so frustrating I can't figure this out!

The next data inserted into a test DB:

var mongoClient = new MongoClient("mongodb://localhost");
var database = mongoClient.GetDatabase("TestDb");

var a1 = new A { Name = "A1->Name", LastName = "A1->LastName", Locked = false };
var a2 = new A { Name = "A2->Name", LastName = "A2->LastName", Locked = true };

var A_Col = database.GetCollection<A>("A_COL");
A_Col.InsertMany(new[] { a1, a2 });

var b1 = new B { Name = "B1->Name" };
var b2 = new B { Name = "B2->Name" };
var b3 = new B { Name = "B3->Name" };

var B_Col = database.GetCollection<B>("B_COL");
B_Col.InsertMany(new[] { b1, b2, b3 });

var c1 = new C { Name = "C1->Name" };
var c2 = new C { Name = "C2->Name" };
var c3 = new C { Name = "C3->Name" };

var C_Col = database.GetCollection<C>("C_COL");
C_Col.InsertMany(new[] { c1, c2, c3 });

var ab1 = new AB { AId = a1.Id, BId = b1.Id };
var ab2 = new AB { AId = a1.Id, BId = b2.Id };

var AB_Col = database.GetCollection<AB>("AB_COL");
AB_Col.InsertMany(new[] { ab1, ab2 });

var ac1 = new AC { AId = a1.Id, CId = c1.Id };
var ac2 = new AC { AId = a1.Id, CId = c2.Id };
var ac3 = new AC { AId = a1.Id, CId = c3.Id };

var AC_Col = database.GetCollection<AC>("AC_COL");
AC_Col.InsertMany(new[] { ac1, ac2, ac3 });

Should give the following results:

* a1
    -Bs
      b1
      b2
    -Cs
      c1
      c2
      c3 

Thanks in advance!

Environment: .net core 2.1, MongoDb Version: 4.0.3, Driver Version: 2.7.1

Answer 1

Update:

If the below doesn't help, please post IQueryable example of what MongoDB driver is generating. You can find it before the query is executed in the debug mode.

---

I think what you're facing with is MongoDB driver projection of anonymous types. It can't $project the document itself because it should map it to some field name . So probably workaround on it would be just give naming to the properties inside the anonymous types:

var q = A_Col.AsQueryable().Where(a => !a.Locked)
.Join(
   AB_Col.AsQueryable(),
   a => a.Id, 
   ab => ab.AId,
   (a, ab) => new
   {
      A = new { Id = a.Id, Name = a.Name, LastName = a.LastName }, // change here
      AB = new { AId = ab.AId, BId = ab.BId } // change here
   })
.Join(
   B_Col.AsQueryable(), 
   a_doc => a_doc.AB.BId,
   b => b.Id, 
   (a, b) => new { Id = a.A.Id, Name = a.A.Name, LastName = a.A.LastName, B = new { Name = b.Name, Id = b.Id } }) // change here
.GroupBy(doc => new { Id = doc.Id, Name = doc.Name, LastName = doc.LastName }) // change here
.Join(
    AC_Col.AsQueryable(),
    g => g.Key.Id,
    ac => ac.AId,
    (g, ac) => new
    {
       Id = g.Key.Id, // change here
       Name = g.Key.Name, // change here
       LastName = g.Key.LastName, // change here
       Bs = g.Select(x => x.B),
       AC = new { ac.AId, ac.CId }
    })
.Join(
    C_Col.AsQueryable(),
    gac => gac.AC.CId,
    c => c.Id,
    (gac, c) => new { Id = gac.Id, Name = gac.Name, LastName = gac.LastName, Bs = gac.Bs, C = new { Name = c.Name, Id = c.Id } }) // change here
.GroupBy(doc => new { Id = doc.Id, Name = doc.Name, LastName = doc.LastName, Bs = doc.Bs }) // change here
.Select(g => new { Id = g.Key.Id, Name = g.Key.Name, LastName = g.Key.LastName, Bs = g.Key.Bs, Cs = g.Select(x => x.C) }); // change here

Answer 2

The problem weren't at the properties' names but at a missing .Select . This is the working query (and it's so much shorter):

var q = A_Col.AsQueryable().Where(a => !a.Locked)
  .Join(
      AB_Col.AsQueryable(), a => a.Id, ab => ab.AId,
      (a, ab) => new { A = new { a.Id, a.Name, a.LastName }, ab.BId })
  .Join(
      B_Col.AsQueryable(),
      a_doc => a_doc.BId,
      b => b.Id,
      (a, b) => new { a.A, B = new { b.Id, b.Name } })
  .GroupBy(doc => doc.A)
  .Select(g => new { A = g.Key, Bs = g.Select(i => i.B)}) // <----- THIS ONE HERE!!
  .Join(
      AC_Col.AsQueryable(),
      r => r.A.Id,
      ac => ac.AId,
      (r, ac) => new { r.A, r.Bs, ac.CId })
  .Join(
      C_Col.AsQueryable(),
      gac => gac.CId,
      c => c.Id,
      (gac, c) => new { gac.A, gac.Bs, C = new { c.Name, c.Id } })
  .GroupBy(doc => new { doc.A, doc.Bs })
  .Select(g => new { g.Key.A, g.Key.Bs, Cs = g.Select(i => i.C) });

I'm not sure why but it's working. Now I'm working on the LEFT JOIN. :)

EDIT: After a lot of struggling I finally have the desired query with the LEFT JOIN s included.

 var q = from a in A_Col.AsQueryable()
     where !a.Locked
     join ab in AB_Col.AsQueryable() on a.Id equals ab.AId into j1
     from abi in j1.DefaultIfEmpty()
     select new {A = new {a.Id, a.Name, a.LastName }, abi.BId } into r1
     join b in B_Col.AsQueryable() on r1.BId equals b.Id into r2
     from bi in r2.DefaultIfEmpty()
     select new { r1.A, B = new { bi.Id, bi.Name } } into r2
     group r2 by r2.A into g1
     select new { A = g1.Key, Bs = g1.Select(i => new { Id = i.B.Id ?? "", Name = i.B.Name ?? ""  }) } into r3
     join ac in AC_Col.AsQueryable() on r3.A.Id equals ac.AId into j2
     from aci in j2.DefaultIfEmpty()
     select new { r3.A, r3.Bs, aci.CId } into r4
     join c in C_Col.AsQueryable() on r4.CId equals c.Id into j2
     from ci in j2.DefaultIfEmpty()
     select new { r4.A, r4.Bs, C = new { ci.Id, ci.Name } } into r5
     group r5 by new {r5.A, r5.Bs} into g2
     select new { g2.Key.A, g2.Key.Bs, Cs = g2.Select(i => new { Id = i.C.Id ?? "", Name = i.C.Name ?? ""  }) };