Entityframework core where condition, sql string that executed and its result

Question

I implemented an api and I used EF core.

I have a complex structure that its core entity is an entity that I called it Project. i should say that i used EF Core as DB First. Then I created my database at first and after that I used "Scaffold-Database" to create my Model in code.

The Project's model is:

using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations.Schema;

namespace myProj_Model.Entities
{
    public partial class Projects
    {
        public Projects()
        {
            Boreholes = new HashSet<Boreholes>();
            ProjectsWorkAmounts = new HashSet<ProjectsWorkAmount>();
            Zones = new HashSet<Zones>();
        }

        public long Id { get; set; }
        public string Number { get; set; }
        public string Name { get; set; }
        public string Description { get; set; }
        public string ClientName { get; set; }
        public int? NoOfRecord { get; set; }
        public byte[] Logo { get; set; }
        public int? LogoWidth { get; set; }
        public int? LogoHeight { get; set; }
        public string Version { get; set; }
        public byte? Revision { get; set; }
        public byte? WorkValueIsLimit { get; set; }
        public long? CreatedBy { get; set; }
        public DateTime? CreatedDate { get; set; }
        public long? ModifiedBy { get; set; }
        public DateTime? ModifiedDate { get; set; }

        public virtual Users CreatedBy_User { get; set; }
        public virtual Users ModifiedBy_User { get; set; }
        public virtual ProjectsDrillingLog ProjectsDrillingLog { get; set; }
        public virtual ProjectsDutchCone ProjectsDutchCone { get; set; }
        public virtual ProjectsGap ProjectsGap { get; set; }
        //public virtual ProjectsLogDrafting ProjectsLogDrafting { get; set; }
        public virtual ProjectsRole ProjectsRole { get; set; }
        public virtual ProjectsUnc ProjectsUnc { get; set; }
        public virtual ICollection<Boreholes> Boreholes { get; set; }
        public virtual ICollection<ProjectsWorkAmount> ProjectsWorkAmounts { get; set; }
        public virtual ICollection<Zones> Zones { get; set; }
    }
}

I mentioned again that the model was created by "Scaffold" command. CRUD operation were handled by GenericRepository:

using geotech_Tests_Model.Entities;
using geotech_Tests_Model.Interfaces;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Query.Internal;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;

namespace myProj_Model.Repositories
{
    public class GenericRepository<T> : IGenericRepository<T> where T : class
    {
        protected geotechContext _context { get; set; }
        public GenericRepository(geotechContext context)
        {
            this._context = context;
        }

        public void Add(T entity)
        {
            try
            {
                _context.Set<T>().Add(entity);
                _context.SaveChanges();
            }
            catch (Exception eXp)
            {
                string Myname = eXp.Message;
            }
        }

        public void AddRange(IEnumerable<T> entities)
        {
            try
            {
                _context.Set<T>().AddRange(entities);
                _context.SaveChanges();
            }
            catch (Exception eXp)
            {
                string Myname = eXp.Message;
            }
        }

        public void Update(T entity)
        {
            try
            {
                _context.Set<T>().Update(entity);
                _context.SaveChanges();
            }
            catch (Exception eXp)
            {
                string Myname = eXp.Message;
            }
        }

        public void UpdateRange(IEnumerable<T> entities)
        {
            try
            {
                _context.Set<T>().UpdateRange(entities);
                _context.SaveChanges();
            }
            catch (Exception eXp)
            {
                string Myname = eXp.Message;
            }
        }

        public IQueryable<T> Find()
        {
            return _context.Set<T>().AsQueryable();
        }

        public List<T> Find(FilterStruct filterstruct)
        {
            Expression<Func<T, bool>> myExpresion = Expresion(filterstruct);

            return _context.Set<T>().AsQueryable().IgnoreAutoIncludes().Where(myExpresion).ToList ();
        }

        public T GetById(long id)
        {
            return _context.Set<T>().Find(id);
        }

        public void Remove(T entity)
        {
            try
            {
                _context.Set<T>().Remove(entity);
                _context.SaveChanges();
            }
            catch (Exception eXp)
            {
                string Myname = eXp.Message;
            }
        }

        public void RemoveRange(IEnumerable<T> entities)
        {
            try
            {
                _context.Set<T>().RemoveRange(entities);
                _context.SaveChanges();
            }
            catch (Exception eXp)
            {
                string Myname = eXp.Message;
            }
        }

        public Expression< Func<T, bool>> Expresion(FilterStruct filters)
        {

            //IQueryable<T> myQry = _context.Set<T>().AsQueryable<T>();
            //IQueryable<T> myQryFilter = _context.Set<T>().AsQueryable<T>();


            List<QueryStruct> queries = filters.Queries;

            Expression predicateBody = null;

            ParameterExpression myExp = Expression.Parameter(typeof(T), typeof(T).Name);

            if (queries != null)
            {
                foreach (QueryStruct query in queries)
                {
                    Expression e1 = null;
                    Expression left = Expression.Property(myExp, typeof(T).GetProperty(query.columnName));
                    Type actualType = Type.GetType(left.Type.FullName);
                    var myValue = Convert.ChangeType(query.value, actualType);

                    Expression right = Expression.Constant(myValue);

                    e1 = ApplyOperand(left, right, query.operatorName);

                    if (predicateBody == null)
                        predicateBody = e1;
                    else
                    {
                        predicateBody = ApplyAndOr(predicateBody, e1, query.AndOr);
                    }

                }
            }


            //var p = Expression.Parameter(typeof(T), typeof(T).Name);
            //if (predicateBody == null) predicateBody = Expression.Constant(true);
            //MethodCallExpression whereCallExpression = Expression.Call(
            //typeof(Queryable),
            //"Where", new Type[] { myQryFilter.ElementType },
            //myQryFilter.Expression, Expression.Lambda<Func<T>, bool> > (predicateBody, myExp));

            var Lambda = Expression.Lambda <Func<T, bool>>(predicateBody, myExp);
            return Lambda;
        }

        public static Expression ApplyOperand(Expression Left, Expression Rigth, OperandEnum Operand)
        {
            Expression result = null;
            switch (Operand)
            {
                case (OperandEnum.Equal):
                    {
                        result = Expression.Equal(Left, Rigth);
                        break;
                    }
                case (OperandEnum.NotEqual):
                    {
                        result = Expression.NotEqual(Left, Rigth);
                        break;
                    }
                case (OperandEnum.GreaterThan):
                    {
                        result = Expression.GreaterThan(Left, Rigth);
                        break;
                    }
                case (OperandEnum.GreaterThanOrEqual):
                    {
                        result = Expression.GreaterThanOrEqual(Left, Rigth);
                        break;
                    }
                case (OperandEnum.LessThan):
                    {
                        result = Expression.LessThan(Left, Rigth);
                        break;
                    }
                case (OperandEnum.LessThanOrEqual):
                    {
                        result = Expression.LessThanOrEqual(Left, Rigth);
                        break;
                    }
            }

            return result;


        }

        public static Expression ApplyAndOr(Expression Left, Expression Rigth, AndOrEnum AndOr)
        {
            Expression result = null;
            switch (AndOr)
            {
                case (AndOrEnum.And):
                    {
                        result = Expression.And(Left, Rigth);
                        break;
                    }
                case (AndOrEnum.AndAlso):
                    {
                        result = Expression.AndAlso(Left, Rigth);
                        break;
                    }
                case (AndOrEnum.AndAssign):
                    {
                        result = Expression.AndAssign(Left, Rigth);
                        break;
                    }
                case (AndOrEnum.Or):
                    {
                        result = Expression.Or(Left, Rigth);
                        break;
                    }
                case (AndOrEnum.OrAssign):
                    {
                        result = Expression.OrAssign(Left, Rigth);
                        break;
                    }
                case (AndOrEnum.OrElse):
                    {
                        result = Expression.OrElse(Left, Rigth);
                        break;
                    }
            }

            return result;


        }





    }
}

and I have ConfigureServices like this in my Startup.cs:

    public void ConfigureServices(IServiceCollection services)
{

    services.AddCors(options =>
    {
        options.AddPolicy(_appCorsPolicy,
            builder =>
            {
                builder.WithOrigins("http://127.0.0.1:23243")
                .AllowAnyHeader();
                //.AllowAnyMethod();
            });
    });

    //****************************************************************************************
    services.AddControllers();

    services.AddControllersWithViews().AddNewtonsoftJson(options => options.SerializerSettings.ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore);

    services.Configure<IISOptions>(options =>
    {
    });
    services.AddAutoMapper(typeof(Startup));
    // Register the Swagger generator, defining 1 or more Swagger documents
    services.AddSwaggerGen(c =>
    {
        c.SwaggerDoc("v1", new OpenApiInfo { Title = "Organization, .Net Core", Version = "V 01" });

        // Set the comments path for the Swagger JSON and UI.
        var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml";
        var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);

        c.IncludeXmlComments(xmlPath);
    });

    string connectionStr = Configuration.GetConnectionString("DefaultConnection");

    services.AddDbContext<geotechContext>(options => options.UseLazyLoadingProxies().UseSqlServer(connectionStr));

    //Farzin
    services.Add(new ServiceDescriptor(typeof(IProjectsService), typeof(ProjectsService), ServiceLifetime.Scoped));
    
    //  {AddServicesHere}

}

After that I have a service class for my Project's entity. GetById is one of the several functoin's that I have in Service class.

    public EventResult GetById(long id)
{
    EventResult result = new EventResult();
    //result.Data = service.GetById (id);
    result.Data = service.Find().IgnoreAutoIncludes().Where(a => (a.Id == id)).FirstOrDefault();
    return result;
}

Now main problem is, Respond time is considerably high. I trace every line of code and even the sql command that executed. Sql profiler show's the sql command that sent to SqlServer. The command is:

    exec sp_executesql N'SELECT TOP(1) [g].[Id], [g].[gtp_ClientName], [g].[CreatedBy], [g].[CreatedDate], [g].[gtP_Description], [g].[gtp_Logo], [g].[gtp_LogoHeight], [g].[gtp_LogoWidth], [g].[ModifiedBy], [g].[ModifiedDate], [g].[gtP_Name], [g].[gtp_NoOfRecord], [g].[gtP_ProjectNumber], [g].[gtp_Revision], [g].[gtp_Version], [g].[gtp_WorkValueIsLimit]
FROM [gt_Projects] AS [g]
WHERE [g].[Id] = @__id_0',N'@__id_0 bigint',@__id_0=1

there is no any relationship and result of executing of this sql is a simple row without any additional data. But something that I get as answer in swagger is a complex record with all related data.

Result of query

and I expect that result in swagger be something like this

Expected Result

But result is something like this:

    {
  "errorNumber": 0,
  "errorMessage": "",
  "eventId": 0,
  "data": {
    "createdBy_User": {
      "projectsRoleId1": null,
      "boreholeCreatedByNavigations": [],
      "boreholeModifiedByNavigations": [],
      "boreholeTypeCreatedByNavigations": [
        {
          "boreholes": [],
          "id": 1,
          "abbriviation": "P",
          "name": "Primary",
          "description": "Primary Boreholes",
          "order": 1,
          "color": null,
          "createdBy": 1,
          "createdDate": "1400-05-27T00:00:00",
          "modifiedBy": 1,
          "modifiedDate": "1400-05-27T00:00:00"
        }
      ],
      "boreholeTypeModifiedByNavigations": [
        {
          "boreholes": [],
          "id": 1,
          "abbriviation": "P",
          "name": "Primary",
          "description": "Primary Boreholes",
          "order": 1,
          "color": null,
          "createdBy": 1,
          "createdDate": "1400-05-27T00:00:00",
          "modifiedBy": 1,
          "modifiedDate": "1400-05-27T00:00:00"
        }
      ],
      "boreholesWorkAmountCreatedByNavigations": [],
      "boreholesWorkAmountModifiedByNavigations": [],
      "dailyActivityDaCoSupervisorNavigations": [],
      "dailyActivityDaSpecialistNavigations": [],
      "dailyActivityDaSupervisorNavigations": [],
      "dailyActivityDaTechnision01Navigations": [],
      "dailyActivityDaTechnision02Navigations": [],
      "created_Projects": [
        {
          "projectsDrillingLog": null,
          "projectsDutchCone": null,
          "projectsGap": null,
          "projectsRole": null,
          "projectsUnc": null,
          "boreholes": [],
          "projectsWorkAmounts": [],
          "zones": [],
          "id": 2,
          "number": "001",
          "name": "Yes",
          "description": "ss",
          "clientName": "ss",
          "noOfRecord": 1,
          "logo": null,

.........
.........
        "id": 5,
        "workActivity": 6,
        "project": 1,
        "activityPredicted": 5,
        "activityActual": null,
        "startDatePredicted": null,
        "endDatePredicted": null,
        "startDateActual": null,
        "endDateActual": null,
        "createdBy": null,
        "createdDate": null,
        "modifiedBy": null,
        "modifiedDate": null
      }
    ],
    "zones": [],
    "id": 1,
    "number": "001",
    "name": "No",
    "description": "ss",
    "clientName": "ss",
    "noOfRecord": 1,
    "logo": null,
    "logoWidth": 11,
    "logoHeight": 11,
    "version": "1",
    "revision": 1,
    "workValueIsLimit": 1,
    "createdBy": 1,
    "createdDate": "1400-06-01T00:00:00",
    "modifiedBy": 1,
    "modifiedDate": "1400-06-01T00:00:00"
  }
}

The answer contains near 4700 line of data, and this happened when my db is almost empty. i don't know why and what I should do, to get my expected result as answer.

Answer 1

I note you have .UseLazyLoadingProxies() which means you can load some data from a DB and then trigger loading some more data just by accessing the property; for example you download just the one Project but as soon as you try and access its Boreholes collection, that access will trigger another query like SELECT * FROM Boreholes WHERE projectId = (whatever the current project id is) to fill the boreholes collection before returning it..

This means when the serializer is enumerating your starting Project 's properties, looking for data it can serialize instead of getting a "null" or "empty" when it accesses the navigation proeprty to some related data, it's the serializer that is triggering a db lookup of every related entity... and then it serializes all those related entities and enumerating their props triggers even more lookups.

What starts out as just one Project, snowballs into loading every related entity up and down the tree, maybe even the whole database, just because you've asked the serializer to turn the Project object into json..

---

Removing the lazy loading proxies feature will stop that, but then other parts of the project won't auto load data upon access; what you do about this is perhaps a wider design decision over how you should load related data..