EF Core navigation properties not populated (again!)

Question

The setup is very simple. Two tables, Endpoints and Deployments, are many-to-many and joined by a table called Services.

var query = from endpoint in db.Endpoints
    from deployment in endpoint.Service.Deployments
    select new Endpoint(
            deployment.Host,
            endpoint.Method,
            endpoint.Path,
            new Cost(endpoint.CostCredits, new PerRequest()));

var result = await query.ToListAsync();

When I run this, I get 0 results.

Inspecting db.Endpoints[0].Service in the debugger shows null. Coming from a slightly different angle:

var query = db.Endpoints
    .Include(endpoint => endpoint.Service)
    .ThenInclude(s => s.Deployments) // out of desperation!
    .SelectMany(endpoint => endpoint.Service.Deployments.Select(deployment =>
    new Endpoint(
        deployment.Host,
        endpoint.Method,
        endpoint.Path,
        new Cost(endpoint.CostCredits, new PerRequest()))));

This throws a NullReferenceException. I even tried adding this before the query:

await db.Endpoints.ToListAsync();
await db.Deployments.ToListAsync();
await db.Services.ToListAsync();

Same behaviour.

I've seen numerous SO questions like EF Core returns null relations until direct access but obviously they are not my case - the navigation properties are not populated even after all the data from all three tables is loaded.

How can I get the above query to work?

Here is a complete minimal repro (same as a runnable Dotnetfiddle ):

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;

namespace Repro
{
    class Program
    {
        static async Task Main()
        {
            var db = new MorpherDbContext();
            db.AddSomeData();

            var query = from endpoint in db.Endpoints
                from deployment in endpoint.Service.Deployments
                select new {
                    deployment.Host,
                    endpoint.Method,
                    endpoint.Path
                    };

            var result = await query.ToListAsync();

            Console.WriteLine(result.Count);
        }
    }

    public class Deployment
    {
        public int Id { get; set; }
        public virtual Service Service { get; set; }
        public int ServiceId { get; set; }

        public string Host { get; set; }
        public short? Port { get; set; }
        public string BasePath { get; set; }
    }
    public class Service
    {
        public int Id { get; set; }
        public string Name { get; set; }

        public string UrlSlug { get; set; }

        public virtual ICollection<Endpoint> Endpoints { get; set; }

        public virtual ICollection<Deployment> Deployments { get; set; }
    }

    public class Endpoint
    {
        public int Id { get; set; }

        public virtual Service Service { get; set; }

        public int ServiceId { get; set; }
        public string Path { get; set; } 
        public string Method { get; set; } 
        public int CostCredits { get; set; } 
        public string CostType { get; set; } 
        public string CostTypeParameter { get; set; } 
    }

    public partial class MorpherDbContext : DbContext
    {
        protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
        {
            optionsBuilder.UseInMemoryDatabase("db1");
            //optionsBuilder.UseSqlServer("Server=(localdb)\\mssqllocaldb;Database=ReproDb;Trusted_Connection=True;MultipleActiveResultSets=true");
            base.OnConfiguring(optionsBuilder);
        }

        public virtual DbSet<Endpoint> Endpoints { get; set; }
        public virtual DbSet<Deployment> Deployments { get; set; }
        public virtual DbSet<Service> Services { get; set; }

        protected override void OnModelCreating(ModelBuilder modelBuilder)
        {
            modelBuilder.HasAnnotation("Relational:DefaultSchema", "dbo");

            modelBuilder.Entity<Deployment>(entity =>
            {
                entity.Property(e => e.Host).IsRequired().HasMaxLength(256);
                entity.Property(e => e.BasePath).HasMaxLength(512);

                entity.HasOne(deployment => deployment.Service)
                    .WithMany(service => service.Deployments)
                    .HasForeignKey(d => d.ServiceId)
                    .OnDelete(DeleteBehavior.Restrict)
                    .HasConstraintName("FK_Deployments_Services");
            });

            modelBuilder.Entity<Service>(entity =>
            {
                entity.Property(e => e.Name).IsRequired().HasMaxLength(256);
                entity.Property(e => e.UrlSlug).IsRequired().HasMaxLength(256);
            });

            modelBuilder.Entity<Endpoint>(endpoint =>
            {
                endpoint.Property(e => e.Path).IsRequired();
                endpoint.Property(e => e.Method).IsRequired().HasMaxLength(6);
                endpoint.Property(e => e.CostCredits).IsRequired();
                endpoint.Property(e => e.CostType).IsRequired().HasMaxLength(50);
                endpoint.Property(e => e.CostTypeParameter).IsRequired().HasMaxLength(150);

                endpoint.HasOne(e => e.Service)
                    .WithMany(service => service.Endpoints)
                    .HasForeignKey(e => e.ServiceId)
                    .OnDelete(DeleteBehavior.Restrict)
                    .HasConstraintName("FK_Endpoints_Services");
            });

            OnModelCreatingPartial(modelBuilder);
        }

        partial void OnModelCreatingPartial(ModelBuilder modelBuilder);

        public void AddSomeData()
        {
            var ws3 = new Service { Name = "Веб-сервис «Морфер»", UrlSlug = "ws"};

            Services.Add(ws3);
            Deployments.Add(new Deployment {Service = ws3, Host = "ws3.morpher.ru"});

            Endpoints.AddRange(new []
            {
                new Endpoint {Method = "GET", Path = "russian/declension", CostCredits = 1, CostType = "PerRequest"},
                new Endpoint {Method = "POST", Path = "russian/declension", CostCredits = 1, CostType = "PerBodyLine"},
                new Endpoint {Method = "*", Path = "russian/userdict", CostCredits = 1, CostType = "PerRequest"},
            });

            ws3.Endpoints = Endpoints.ToList();

            SaveChanges();
        }
    }
}

namespace Repro.Migrations
{
    public partial class InitialCreate : Migration
    {
        protected override void Up(MigrationBuilder migrationBuilder)
        {
            migrationBuilder.EnsureSchema(
                name: "dbo");

            migrationBuilder.CreateTable(
                name: "Services",
                schema: "dbo",
                columns: table => new
                {
                    Id = table.Column<int>(nullable: false)
                        .Annotation("SqlServer:Identity", "1, 1"),
                    Name = table.Column<string>(maxLength: 256, nullable: false),
                    UrlSlug = table.Column<string>(maxLength: 256, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_Services", x => x.Id);
                });

            migrationBuilder.CreateTable(
                name: "Deployments",
                schema: "dbo",
                columns: table => new
                {
                    Id = table.Column<int>(nullable: false)
                        .Annotation("SqlServer:Identity", "1, 1"),
                    ServiceId = table.Column<int>(nullable: false),
                    Host = table.Column<string>(maxLength: 256, nullable: false),
                    Port = table.Column<short>(nullable: true),
                    BasePath = table.Column<string>(maxLength: 512, nullable: true)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_Deployments", x => x.Id);
                    table.ForeignKey(
                        name: "FK_Deployments_Services",
                        column: x => x.ServiceId,
                        principalSchema: "dbo",
                        principalTable: "Services",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Restrict);
                });

            migrationBuilder.CreateTable(
                name: "Endpoints",
                schema: "dbo",
                columns: table => new
                {
                    Id = table.Column<int>(nullable: false)
                        .Annotation("SqlServer:Identity", "1, 1"),
                    ServiceId = table.Column<int>(nullable: false),
                    Path = table.Column<string>(nullable: false),
                    Method = table.Column<string>(maxLength: 6, nullable: false),
                    CostCredits = table.Column<int>(nullable: false),
                    CostType = table.Column<string>(maxLength: 50, nullable: false),
                    CostTypeParameter = table.Column<string>(maxLength: 150, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_Endpoints", x => x.Id);
                    table.ForeignKey(
                        name: "FK_Endpoints_Services",
                        column: x => x.ServiceId,
                        principalSchema: "dbo",
                        principalTable: "Services",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Restrict);
                });

            migrationBuilder.CreateIndex(
                name: "IX_Deployments_ServiceId",
                schema: "dbo",
                table: "Deployments",
                column: "ServiceId");

            migrationBuilder.CreateIndex(
                name: "IX_Endpoints_ServiceId",
                schema: "dbo",
                table: "Endpoints",
                column: "ServiceId");
        }

        protected override void Down(MigrationBuilder migrationBuilder)
        {
            migrationBuilder.DropTable(
                name: "Deployments",
                schema: "dbo");

            migrationBuilder.DropTable(
                name: "Endpoints",
                schema: "dbo");

            migrationBuilder.DropTable(
                name: "Services",
                schema: "dbo");
        }
    }
}

namespace Repro.Migrations
{
    [DbContext(typeof(MorpherDbContext))]
    partial class MorpherDbContextModelSnapshot : ModelSnapshot
    {
        protected override void BuildModel(ModelBuilder modelBuilder)
        {
#pragma warning disable 612, 618
            modelBuilder
                .HasDefaultSchema("dbo")
                .HasAnnotation("ProductVersion", "3.1.3")
                .HasAnnotation("Relational:MaxIdentifierLength", 128)
                .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn);

            modelBuilder.Entity("Repro.Deployment", b =>
                {
                    b.Property<int>("Id")
                        .ValueGeneratedOnAdd()
                        .HasColumnType("int")
                        .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn);

                    b.Property<string>("BasePath")
                        .HasColumnType("nvarchar(512)")
                        .HasMaxLength(512);

                    b.Property<string>("Host")
                        .IsRequired()
                        .HasColumnType("nvarchar(256)")
                        .HasMaxLength(256);

                    b.Property<short?>("Port")
                        .HasColumnType("smallint");

                    b.Property<int>("ServiceId")
                        .HasColumnType("int");

                    b.HasKey("Id");

                    b.HasIndex("ServiceId");

                    b.ToTable("Deployments");
                });

            modelBuilder.Entity("Repro.Endpoint", b =>
                {
                    b.Property<int>("Id")
                        .ValueGeneratedOnAdd()
                        .HasColumnType("int")
                        .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn);

                    b.Property<int>("CostCredits")
                        .HasColumnType("int");

                    b.Property<string>("CostType")
                        .IsRequired()
                        .HasColumnType("nvarchar(50)")
                        .HasMaxLength(50);

                    b.Property<string>("CostTypeParameter")
                        .IsRequired()
                        .HasColumnType("nvarchar(150)")
                        .HasMaxLength(150);

                    b.Property<string>("Method")
                        .IsRequired()
                        .HasColumnType("nvarchar(6)")
                        .HasMaxLength(6);

                    b.Property<string>("Path")
                        .IsRequired()
                        .HasColumnType("nvarchar(max)");

                    b.Property<int>("ServiceId")
                        .HasColumnType("int");

                    b.HasKey("Id");

                    b.HasIndex("ServiceId");

                    b.ToTable("Endpoints");
                });

            modelBuilder.Entity("Repro.Service", b =>
                {
                    b.Property<int>("Id")
                        .ValueGeneratedOnAdd()
                        .HasColumnType("int")
                        .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn);

                    b.Property<string>("Name")
                        .IsRequired()
                        .HasColumnType("nvarchar(256)")
                        .HasMaxLength(256);

                    b.Property<string>("UrlSlug")
                        .IsRequired()
                        .HasColumnType("nvarchar(256)")
                        .HasMaxLength(256);

                    b.HasKey("Id");

                    b.ToTable("Services");
                });

            modelBuilder.Entity("Repro.Deployment", b =>
                {
                    b.HasOne("Repro.Service", "Service")
                        .WithMany("Deployments")
                        .HasForeignKey("ServiceId")
                        .HasConstraintName("FK_Deployments_Services")
                        .OnDelete(DeleteBehavior.Restrict)
                        .IsRequired();
                });

            modelBuilder.Entity("Repro.Endpoint", b =>
                {
                    b.HasOne("Repro.Service", "Service")
                        .WithMany("Endpoints")
                        .HasForeignKey("ServiceId")
                        .HasConstraintName("FK_Endpoints_Services")
                        .OnDelete(DeleteBehavior.Restrict)
                        .IsRequired();
                });
#pragma warning restore 612, 618
        }
    }
}

Answer 1

The problem is the following line:

ws3.Endpoints = Endpoints.ToList();

This line will mess up the navigation properties/collections which are maintained by the current context of entity framework. In fact, when you try to set Service = ws3 for the Endpoint objects in your AddSomeData() method you will get the following exception message:

The association between entities 'Service' and 'Endpoint' with the key value '{ServiceId: 1}' has been severed but the relationship is either marked as 'Required' or is implicitly required because the foreign key is not nullable. If the dependent/child entity should be deleted when a required relationship is severed, then setup the relationship to use cascade deletes.

You are adding Endpoint objects via the Add()/AddRange() methods but without any Service reference assigned (or ServiceId value set). In combination with the line above it will break (somehow).

To fix the issue you should remove either the ws3.Endpoints = Endpoints.ToList() line and set the Service navigation properties or remove the Add()/AddRange() calls for the Endpoint objects. The AddSomeData() method might look like this:

public void AddSomeData()
{
    var ws3 = new Service { Name = "Веб-сервис «Морфер»", UrlSlug = "ws"};

    Services.Add(ws3);
    Deployments.Add(new Deployment {Service = ws3, Host = "ws3.morpher.ru"});
    var endpointsToAdd = new []
    {
        new Endpoint {Method = "GET", Path = "russian/declension", CostCredits = 1, CostType = "PerRequest"},
        new Endpoint {Method = "POST", Path = "russian/declension", CostCredits = 1, CostType = "PerBodyLine"},
        new Endpoint {Method = "*", Path = "russian/userdict", CostCredits = 1, CostType = "PerRequest"},
    };

    ws3.Endpoints = endpointsToAdd.ToList();

    SaveChanges();
}

The other solution would be:

public void AddSomeData()
{
    var ws3 = new Service { Name = "Веб-сервис «Морфер»", UrlSlug = "ws"};

    Services.Add(ws3);
    Deployments.Add(new Deployment {Service = ws3, Host = "ws3.morpher.ru"});
    Endpoints.Add(new Endpoint {Service = ws3, Method = "GET", Path = "russian/declension", CostCredits = 1, CostType = "PerRequest"});
    Endpoints.Add(new Endpoint {Service = ws3, Method = "POST", Path = "russian/declension", CostCredits = 1, CostType = "PerBodyLine"});
    Endpoints.Add(new Endpoint {Service = ws3, Method = "*", Path = "russian/userdict", CostCredits = 1, CostType = "PerRequest"});

    SaveChanges();
}

Answer 2

The official documentation discourages using the InMemory provider for any non-trivial database queries:

we use the EF in-memory database when unit testing something that uses DbContext. In this case using the EF in-memory database is appropriate because the test is not dependent on database behavior. Just don't do this to test actual database queries or updates.

The InMemory provider won't run any joins for you which is what I was trying to do. The alternatives are using LocalDb or SqLite.