How to query Azure Table storage by nullable fields?

Question

I have table with nullable fields :

public int? Order {get; set;}
public DateTime? StartDate {get; set;}
public DateTime? EndDate {get; set;}
public string Text {get; set;}

All this fields can be with NULL value

Problem starts when I want to query records

• where Order, StartDate, EndDate and Text aren't NULL or

• where Order, StartDate and Text aren't null but EndDate is null or

• where Order and Text aren't null but StartDate and EndDate are null

  Order.HasValue && StartDate.HasValue && EndDate.HasValue && !string.IsNullOrEmpty(Text) || Order.HasValue && StartDate.HasValue && !EndDate.HasValue && !string.IsNullOrEmpty(Text) || Order.HasValue && !StartDate.HasValue && !EndDate.HasValue && !string.IsNullOrEmpty(Text)

Using such query I get error or 400 (bad request) or unsupported operator (isnullorempty isn't supported)

According to this answer https://stackoverflow.com/a/4263091/3917754 it is impossible to query NULL values...

Answer 1

AFAIK, you could not set DateTime to null or empty. So, I suggest that you could store your datetime as string and when you want to map to view model you could convert string to DateTime.

When you insert the entity, convert DateTime to string:

    TableBatchOperation batchOperation = new TableBatchOperation();
    DateTime dts = DateTime.Now;
    DateTime dte = DateTime.UtcNow;

    // Create a customer entity and add it to the table.
    CustomerEntity customer1 = new CustomerEntity("Smith", "Jeff");
    customer1.Order = 1;
    customer1.StartDate = Convert.ToString(dts);
    customer1.EndDate = Convert.ToString(dte);
    customer1.Text = "text1";

    // Create another customer entity and add it to the table.
    CustomerEntity customer2 = new CustomerEntity("Smith", "Ben");
    customer2.Order = 2;
    customer2.StartDate = Convert.ToString(dts);
    customer2.EndDate = "";
    customer2.Text = "text2";

    CustomerEntity customer3 = new CustomerEntity("Smith", "Cai");
    customer3.Order = 3;
    customer3.StartDate = "";
    customer3.EndDate = "";
    customer3.Text = "text3";

    // Add both customer entities to the batch insert operation.
    batchOperation.Insert(customer1);
    batchOperation.Insert(customer2);
    batchOperation.Insert(customer3);

    // Execute the batch operation.
    table.ExecuteBatch(batchOperation);

The Entity is as below:

public class CustomerEntity : TableEntity
        {
            public CustomerEntity(string lastName, string firstName)
            {
                this.PartitionKey = lastName;
                this.RowKey = firstName;
            }

            public CustomerEntity() { }

            public int? Order { get; set; }
            public string StartDate { get; set; }
            public string EndDate { get; set; }
            public string Text { get; set; }

            public DateTime? ConvertTime(string dateStr)
            {
                if (string.IsNullOrEmpty(dateStr))
                    return null;
                DateTime dt;
                var convert=DateTime.TryParse(dateStr, out dt);
                return dt;
            }
        }

When you show it or map it to model, you could use ConvertTime method to judge if the column is null with ConvertTime(entity.StartDate) . If it is null, it will show null and if it has value, it will convert string to DateTime.

            string orderhasvalue = TableQuery.GenerateFilterCondition("Order", QueryComparisons.NotEqual, null);
            string startdatehasvalue = TableQuery.GenerateFilterCondition("StartDate", QueryComparisons.NotEqual, null);
            string enddatehasvalue = TableQuery.GenerateFilterCondition("EndDate", QueryComparisons.NotEqual, null);
            string texthasvalue = TableQuery.GenerateFilterCondition("Text", QueryComparisons.NotEqual, null);
            string startdatenothasvalue = TableQuery.GenerateFilterCondition("StartDate", QueryComparisons.Equal, null);
            string enddatenothasvalue = TableQuery.GenerateFilterCondition("EndDate", QueryComparisons.Equal, null);

            TableQuery<CustomerEntity> query1 = new TableQuery<CustomerEntity>().Where(
                TableQuery.CombineFilters(
                    TableQuery.CombineFilters(
                        TableQuery.CombineFilters(
                            orderhasvalue, TableOperators.And, startdatehasvalue),
                    TableOperators.And, enddatehasvalue),
                TableOperators.And, texthasvalue)
            );
            TableQuery<CustomerEntity> query2 = new TableQuery<CustomerEntity>().Where(
                TableQuery.CombineFilters(
                    TableQuery.CombineFilters(
                        TableQuery.CombineFilters(
                            orderhasvalue, TableOperators.And, startdatehasvalue),
                    TableOperators.And, enddatenothasvalue),
                TableOperators.And, texthasvalue)
            );
            TableQuery<CustomerEntity> query3 = new TableQuery<CustomerEntity>().Where(
                TableQuery.CombineFilters(
                    TableQuery.CombineFilters(
                        TableQuery.CombineFilters(
                            orderhasvalue, TableOperators.And, startdatenothasvalue),
                    TableOperators.And, enddatenothasvalue),
                TableOperators.And, texthasvalue)
            );

            // Print the fields for each customer.
            foreach (CustomerEntity entity in table.ExecuteQuery(query2))
            {

                Console.WriteLine("{0}, {1}\t{2}\t{3}\t{4}\t{5}", entity.PartitionKey, entity.RowKey,
                    entity.Order, entity.ConvertTime(entity.StartDate), entity.ConvertTime(entity.EndDate), entity.Text);
            }

Answer 2

You cannot query a null value in Azure Table Storage, string is no exception to that. The property with null value does not exist in tabular form when written to azure table, so queries referring to that property will always return an unexpected result. What you can do as a workaround is to provide default non-null values and query for those instead. In case of string assigning the value "" to a string , allows you to query for string.Empty (not null because "" is not null ). For DateTime? type, again same workaround but instead of "" , you can assign an obscure default value ie. DateTime.MinValue (or MaxValue ) if actual value of the property is null, else you can convert it to string and assign empty string as a default value but you need to pay the price there for conversion back and forth so I personally prefer to avoid that if possible.

Answer 3

You can query the null filter on azure table storage for string data type field.

OData Query = not (Id ne '')

Using above Odata query you can able to filter the null data only on string data type field.

You can provide above query at azure table storage Text Editor or use Below is C# Code to fetch the data

TableQuery partitionKeysQuery = new TableQuery() .Where("not (Id ne '')");

List contentKeyEntities = new List(); TableQuerySegment partitionKeySegment = null; while (partitionKeySegment == null || partitionKeySegment.ContinuationToken != null) { Task<TableQuerySegment> t1 = Task.Run(() => O365ReportingTable.ExecuteQuerySegmentedAsync(partitionKeysQuery, partitionKeySegment?.ContinuationToken)); t1.Wait(); partitionKeySegment = t1.Result; contentKeyEntities.AddRange(partitionKeySegment.Results); Console.WriteLine(contentKeyEntities.Count); } return contentKeyEntities;

public class AuditLogEntity : TableEntity { #region Constructor Code

    /// <summary>
    /// default constructor
    /// </summary>
    public AuditLogEntity() { }

    /// <summary>
    /// constructor for creating an AuditLogEntity
    /// </summary>
    /// <param name="partitionKey">partition key of the data</param>
    /// <param name="rowKey">row key for the row</param>
    public AuditLogEntity(string partitionKey, string rowKey, JObject content)
    {
        PartitionKey = partitionKey;
        RowKey = rowKey;
        Properties = ConvertToEntityProperty(content);
    }

    #endregion Constructor Code

    #region Public Properties

    /// <summary>
    /// additional properties for the entity
    /// </summary>
    public IDictionary<string, EntityProperty> Properties { get; set; }

    #endregion Public Properties

    #region Private Methods

    /// <summary>
    /// converts JObjects keys into properties
    /// </summary>
    /// <param name="content">JObject to convert</param>
    /// <returns>Dictionary with key value pairs of JOjbect keys</returns>
    private IDictionary<string, EntityProperty> ConvertToEntityProperty(JObject content)
    {
        IDictionary<string, EntityProperty> properties = new Dictionary<string, EntityProperty>();

        if (content != null)
        {
            foreach (JProperty prop in content.Properties())
            {
                properties.Add(prop.Name, new EntityProperty(prop.Value.ToString()));
            }
        }

        return properties;
    }

    /// <summary>
    /// overrides the base WriteEntry to dynamically write properties to table storage
    /// </summary>
    /// <param name="operationContext">operation being performed</param>
    /// <returns>dictionary of set properties</returns>
    public override IDictionary<string, EntityProperty> WriteEntity(OperationContext operationContext)
    {
        IDictionary<string, EntityProperty> results = base.WriteEntity(operationContext);

        foreach (string key in Properties.Keys)
        {
            results.Add(key, Properties[key]);
        }

        return results;
    }

    /// <summary>
    /// overridden base ReadEntry method to convert propeties coming from table storage into the properties from this object
    /// </summary>
    /// <param name="properties">properties read in from table storage</param>
    /// <param name="operationContext">operation being performed</param>
    public override void ReadEntity(IDictionary<string, EntityProperty> properties, OperationContext operationContext)
    {
        base.ReadEntity(properties, operationContext);
        Properties = properties;
    }

    #endregion Private Methods
}