Optimize algorithm: Update a collection by other collection with order C#

Question

Problem: I have a collection need to be update by other collection. The length of them are not equal and each item are not same structure. But both of them are have identity field to detect same.

I write a generic algorithm to use everywhere but the complexity are O(m*n*3). The main problem is: Are there any better optimization algorithm? The input must should be generic as IList<T> and delegate to reusable.

Current approach:

• Source : Data collection is used to display on GUI.

• Target : Data collection which is got from remote server.

  1. Remove old items from Source which doesn't existed from Target via delegate comparer.
  2. Add new items from Target to Source .
  3. Reorder Source by Target .

Scenario Usage: If you have an application display about 1~2k row on list view. And you have interval timer update this list item (about 1 seconds). The main point here application should be smooth, doesn't flick , keep state of object: Selected, change data... etc.

Here is source code:

using System;
using System.Collections.Generic;
using System.Linq;

namespace UpdateCollection
{
    /// <summary>
    /// Person DTO (from other 3rd assembly) to get data from their remote server.
    /// </summary>
    public class PersonDto
    {
        public int Id { get; set; }
        public string Name { get; set; }
        public DateTime Birthday { get; set; }
    }

    /// <summary>
    /// Person model is used to display on GUI (our application).
    /// </summary>
    public class PersonModel
    {
        public string Identity { get; set; }
        public string DisplayName { get; set; }
        public int? Age { get; set; }
        public bool Selected { get; set; }
        public override string ToString()
        {
            return string.Format("{0} {1} {2} {3}", Identity, DisplayName, Age, Selected);
        }
    }

    static class Program
    {
        /// <summary>
        /// Encapsulates a method that has two parameters and does not return a value.
        /// </summary>
        /// <typeparam name="T1">The type of the first parameter of the method that this delegate encapsulates.This     type parameter is contravariant. That is, you can use either the type you specified or any type that is less derived. For more information about covariance and contravariance, see Covariance and Contravariance in Generics.</typeparam>
        /// <typeparam name="T2">The type of the second parameter of the method that this delegate encapsulates.</typeparam>
        /// <param name="arg1"></param>
        /// <param name="arg2"></param>
        public delegate void RefAction<T1, in T2>(ref T1 arg1, T2 arg2);

        /// TODO: The complexity of algorithm is: O(m*n*3) Need to be optimization. For example: m*log2(n)+ m + n
        /// <summary>
        /// Update source by target.
        /// </summary>
        /// <typeparam name="TSourceType"></typeparam>
        /// <typeparam name="TTargetType"></typeparam>
        /// <param name="source">Source collection.</param>
        /// <param name="target">Target collection.</param>
        /// <param name="compare">Comparing method between source and target.</param>
        /// <param name="convert">Convert method</param>
        /// <param name="update">Update method</param>
        /// <param name="remove">Remove method</param>
        public static void UpdateBy<TSourceType, TTargetType>(
            this IList<TSourceType> source,
            IList<TTargetType> target,
            Func<TSourceType, TTargetType, bool> compare,
            Func<TTargetType, TSourceType> convert,
            RefAction<TSourceType, TTargetType> update,
            Func<TSourceType, bool> remove = null)
        {
            if (source == null || target == null)
                return;

            if (convert == null)
                throw new AggregateException("convert");
            if (compare == null)
                throw new ArgumentNullException("compare");

            // Remove item
            for (var index = 0; index < source.Count; ++index)
            {
                if (target.Any(c => compare(source[index], c))) continue;
                var temp = source[index];
                if (remove == null)
                    source.RemoveAt(index--);
                else if (remove(temp))
                    source.RemoveAt(index--);
            }
            // Add new item
            foreach (var t in target.Where(t => !source.Any(c => compare(c, t))))
            {
                source.Add(convert(t));
            }
            // Sort by target
            for (var index = 0; index < target.Count; ++index)
            {
                for (var pos = 0; pos < source.Count; ++pos)
                {
                    if (!compare(source[pos], target[index])) continue;

                    var temp = source[pos];
                    if (update != null)
                        update(ref temp, target[index]);
                    source[pos] = temp;

                    if (pos == index) continue;

                    temp = source[pos];
                    source[pos] = source[index];
                    source[index] = temp;
                }
            }
        }

        public static IList<PersonModel> GetFromUserInterface()
        {
            return new List<PersonModel>
            {
                new PersonModel {Identity = "1", DisplayName = "a",},
                new PersonModel {Identity = "2", DisplayName = "b", Selected = true},
                new PersonModel {Identity = "3", DisplayName = "c", Selected = true},
                new PersonModel {Identity = "4", DisplayName = "D"}
            };
        }

        public static IList<PersonDto> GetFromRemoteServer()
        {
            return new List<PersonDto>
            {
                new PersonDto {Id = 6, Name = "F", Birthday = DateTime.Parse("1984-01-02")},
                new PersonDto {Id = 4, Name = "D", Birthday = DateTime.Parse("1986-01-12")},
                new PersonDto {Id = 3, Name = "C", Birthday = DateTime.Parse("1982-03-05")},
                new PersonDto {Id = 5, Name = "E", Birthday = DateTime.Parse("1984-05-22")},
                new PersonDto {Id = 1, Name = "A", Birthday = DateTime.Parse("1986-02-14")}
            };
        }

        public static bool Compare(PersonModel source, PersonDto target)
        {
            return source.Identity == target.Id.ToString();
        }

        public static PersonModel Convert(PersonDto target)
        {
            return new PersonModel
            {
                Identity = target.Id.ToString(),
                Age = target.Birthday.Year,
                DisplayName = target.Name,
            };
        }

        public static void Update(ref PersonModel source, PersonDto target)
        {
            source.Age = target.Birthday.Year;
            source.DisplayName = target.Name;
        }

        static void Main(string[] args)
        {

            var source = GetFromUserInterface();
            var target = GetFromRemoteServer();

            Console.WriteLine("==> Before Update:\r\n");
            foreach (var item in source)
                Console.Write("{0}\r\n\r\n", item);

            // TODO: How to optimize UpdateBy algorithm to better?
            source.UpdateBy(target, Compare, Convert, Update);

            Console.WriteLine("==> After Update:\r\n");

            foreach (var item in source)
                Console.Write("{0}\r\n\r\n", item);

            Console.ReadLine();
        }
    }
}

Answer 1

You are storing your data in a list; switching to a hash table structure would give you (roughly) constant time access to the members by key. If the ordering of the data is important, many languages have some kind of ordered hash construct you can use.

Answer 2

Finally, I found the best algorithm for this issue. Base on QuickSort algorithm. The procedure are:

1. Make lookup index base on Target collection.
2. Sort Source base on Target index with QuickSort algorithm. Keep track index of item from Source which doesn't exist in Target
3. Remove item from Source .
4. Add/update item from Target to Source

The complexity or algorithm is: O(m * 2 + n + n * log2(n))

Here is source code:

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;

namespace UpdateCollection
{

    static class CollectionUpdater
    {
        /// <summary>
        /// Encapsulates a method that has two parameters and does not return a value.
        /// </summary>
        /// <typeparam name="T1">The type of the first parameter of the method that this delegate encapsulates.This     type parameter is contravariant. That is, you can use either the type you specified or any type that is less derived. For more information about covariance and contravariance, see Covariance and Contravariance in Generics.</typeparam>
        /// <typeparam name="T2">The type of the second parameter of the method that this delegate encapsulates.</typeparam>
        /// <param name="arg1"></param>
        /// <param name="arg2"></param>
        public delegate void RefAction<T1, in T2>(ref T1 arg1, T2 arg2);

        /// <summary>
        /// Update source collection by target collection.
        /// </summary>
        /// <remarks>The complexity of algorithm is: O(m * 2 + n + n * log2(n))</remarks>
        /// <typeparam name="TSourceType">Source data type.</typeparam>
        /// <typeparam name="TTargetType">Target data type.</typeparam>
        /// <typeparam name="TIdentity"></typeparam>
        /// <param name="source">The source collection.</param>
        /// <param name="target">The target collection.</param>
        /// <param name="targetIdentity">Convert target to identity.</param>
        /// <param name="sourceIdentity">Convert source to identity.</param>
        /// <param name="targetToSourceConverter">Convert target to source.</param>
        /// <param name="sourceUpdater">Update source from target.</param>
        /// <param name="sourceRemover">Remove source item.</param>
        [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")]
        [SuppressMessage("ReSharper", "PossibleNullReferenceException")]
        [SuppressMessage("ReSharper", "AccessToModifiedClosure")]
        public static void UpdateBy<TSourceType, TTargetType, TIdentity>(
            this IList<TSourceType> source,
            IEnumerable<TTargetType> target,
            Func<TTargetType, TIdentity> targetIdentity,
            Func<TSourceType, TIdentity> sourceIdentity,
            Func<TTargetType, TSourceType> targetToSourceConverter,
            RefAction<TSourceType, TTargetType> sourceUpdater,
            Func<TSourceType, bool> sourceRemover = null) where TIdentity : IComparable<TIdentity>
        {
            // Step 1: Make index of target. O(m)
            // Index, target, order.
            var targetOrderLookup = new Dictionary<TIdentity, Tuple<TTargetType, int>>();

            do
            {
                var counterIndex = 0;
                foreach (var item in target)
                {
                    targetOrderLookup.Add(targetIdentity(item), Tuple.Create(item, counterIndex));
                    ++counterIndex;
                }
            } while (false);
            var skipRemoveIdentity = new HashSet<TIdentity>();


            if (source.Count != 0)
            {
                // Step 2: Re-implement quick-sort.

                Action<IList<TSourceType>, int, int, Comparison<Tuple<TSourceType, int>>> quickSort = null;

                var removeIdentityLockup = new Dictionary<TIdentity, int>();
                quickSort = (elements, left, right, comparer) =>
                {
                    var i = left;
                    var j = right;

                    var pivotIndex = (left + right) / 2;
                    var pivot = elements[pivotIndex];

                    while (i <= j)
                    {
                        while (comparer(Tuple.Create(elements[i], i), Tuple.Create(pivot, pivotIndex)) < 0)
                            i++;

                        while (comparer(Tuple.Create(elements[j], j), Tuple.Create(pivot, pivotIndex)) > 0)
                            j--;

                        if (i <= j)
                        {
                            var leftId = sourceIdentity(elements[i]);
                            var rightId = sourceIdentity(elements[j]);


                            var leftValue = targetOrderLookup.ContainsKey(leftId);
                            var rightValue = targetOrderLookup.ContainsKey(rightId);

                            if (!leftValue)
                                removeIdentityLockup[leftId] = j;

                            if (!rightValue)
                                removeIdentityLockup[rightId] = i;

                            // Swap
                            var tmp = elements[i];
                            elements[i] = elements[j];
                            elements[j] = tmp;


                            i++;
                            j--;
                        }
                    }

                    // Recursive calls
                    if (left < j)
                        quickSort(elements, left, j, comparer);

                    if (i < right)
                        quickSort(elements, i, right, comparer);
                };

                // Step 2: Sort source. O(log2(n))


                quickSort(source, 0, source.Count - 1, (c, d) =>
                {
                    var leftId = sourceIdentity(c.Item1);
                    var rightId = sourceIdentity(d.Item1);

                    Tuple<TTargetType, int> leftValue;
                    if (!targetOrderLookup.TryGetValue(leftId, out leftValue))
                        removeIdentityLockup[leftId] = c.Item2;

                    Tuple<TTargetType, int> rightValue;
                    if (!targetOrderLookup.TryGetValue(rightId, out rightValue))
                        removeIdentityLockup[rightId] = d.Item2;

                    if (leftValue == null && rightValue == null)
                        return 0;
                    if (leftValue == null)
                        return -1;
                    if (rightValue == null)
                        return 1;

                    return leftValue.Item2.CompareTo(rightValue.Item2);
                });

                // Remove item
                foreach (KeyValuePair<TIdentity, int> item in removeIdentityLockup.OrderByDescending(v => v.Value))
                {
                    if (sourceRemover == null)
                    {
                        if (source.IsReadOnly)
                            skipRemoveIdentity.Add(item.Key);
                        else
                        {
                            source.RemoveAt(item.Value);
                        }
                    }
                    else
                    {
                        if (sourceRemover(source[item.Value]))
                        {
                            if (source.IsReadOnly)
                                skipRemoveIdentity.Add(item.Key);
                            else
                                source.RemoveAt(item.Value);
                        }
                        else // Keep can remove, avoid update
                            skipRemoveIdentity.Add(item.Key);
                    }
                }
            }


            // Add new item
            var sourceIndex = 0;
            foreach (var item in target)
            {
                var targetItem = item;
                if (sourceIndex < source.Count)
                {
                    var sourceItem = source[sourceIndex];
                    var sourceId = sourceIdentity(sourceItem);
                    var targetId = targetIdentity(targetItem);

                    while (skipRemoveIdentity.Contains(sourceId) && sourceIndex < source.Count)
                    {
                        ++sourceIndex;
                        sourceItem = source[sourceIndex];
                        sourceId = sourceIdentity(sourceItem);
                    }

                    if (sourceIndex < source.Count)
                    {
                        if (sourceId.CompareTo(targetId) == 0) // Update source
                        {
                            sourceUpdater(ref sourceItem, targetItem);
                            source[sourceIndex] = sourceItem;
                            ++sourceIndex;
                        }
                        else // Insert new
                        {
                            if (source.IsReadOnly) continue;
                            source.Insert(sourceIndex, targetToSourceConverter(targetItem));
                            ++sourceIndex;
                        }
                    }
                    else
                    {
                        if (source.IsReadOnly) continue;
                        source.Add(targetToSourceConverter(targetItem));
                    }
                }
                else
                {
                    if (source.IsReadOnly) continue;
                    source.Add(targetToSourceConverter(targetItem));
                    ++sourceIndex;
                }
            }
        }

        /// <summary>
        /// Person DTO (from other 3rd assembly) to get data from their remote server.
        /// </summary>
        public class PersonDto
        {
            public int Id { get; set; }
            public string Name { get; set; }
            public DateTime Birthday { get; set; }
        }

        /// <summary>
        /// Person model is used to display on GUI (our application).
        /// </summary>
        public class PersonModel
        {
            public string Identity { get; set; }
            public string DisplayName { get; set; }
            public int? Age { get; set; }
            public bool Selected { get; set; }
            public override string ToString()
            {
                return string.Format("\"{0}\" {1} {2} {3}", Identity, DisplayName, Age, Selected ? "selected" : string.Empty);
            }
        }

        /// <summary>
        /// Get from user interface, it work for both fix & non-fixed collection.
        /// </summary>
        public static IList<PersonModel> GetFromUserInterface()
        {
            //return new List<PersonModel>  // For non-fixed collection. Add/remove/update support
            return new[]     // For fix collection. Just update support
            {
                new PersonModel {Identity = "4", DisplayName = "D"},
                new PersonModel {Identity = "13", DisplayName = "", Selected = true}, // Must remove.
                new PersonModel {Identity = "1", DisplayName = "a",},
                new PersonModel {Identity = "10", DisplayName = "", Selected = true}, // Must remove.
                new PersonModel {Identity = "3", DisplayName = "c", Selected = true},
                new PersonModel {Identity = "9", DisplayName = "", Selected = true}, // Must remove.
                new PersonModel {Identity = "2", DisplayName = "", Selected = true} // Must remove.
            };
        }

        /// <summary>
        /// Get from remote service.
        /// </summary>
        public static IEnumerable<PersonDto> GetFromRemoteServer()
        {
            return new List<PersonDto>
            {
                new PersonDto {Id = 6, Name = "F", Birthday = DateTime.Parse("1984-01-02")}, // Must add
                new PersonDto {Id = 4, Name = "D", Birthday = DateTime.Parse("1986-01-12")},
                new PersonDto {Id = 3, Name = "C", Birthday = DateTime.Parse("1982-03-05")},
                new PersonDto {Id = 5, Name = "E", Birthday = DateTime.Parse("1984-05-22")}, // Must Add
                new PersonDto {Id = 1, Name = "A", Birthday = DateTime.Parse("1986-02-14")}
            };
        }

        /// <summary>
        /// Convert target to source.
        /// </summary>
        public static PersonModel Convert(PersonDto target)
        {
            return new PersonModel
            {
                Identity = target.Id.ToString(),
                Age = DateTime.Now.Year - target.Birthday.Year,
                DisplayName = target.Name,
            };
        }

        /// <summary>
        /// Update target from source.
        /// </summary>
        public static void Update(ref PersonModel source, PersonDto target)
        {
            source.Age = DateTime.Now.Year - target.Birthday.Year;
            source.DisplayName = target.Name;
        }

        /// <summary>
        /// Get identity.
        /// </summary>
        public static string Identity(PersonModel arg)
        {
            return arg.Identity;
        }

        /// <summary>
        /// Get identity.
        /// </summary>
        public static string Identity(PersonDto arg)
        {
            return arg.Id.ToString();
        }

        static void Main()
        {
            var source = GetFromUserInterface();
            var target = GetFromRemoteServer();

            Console.WriteLine("==> Before Update:\r\n");
            foreach (var item in source)
                Console.Write("{0}\r\n\r\n", item);

            // TODO: Update source collection by target.
            source.UpdateBy(target, Identity, Identity, Convert, Update);

            Console.WriteLine("==> After Update:\r\n");

            foreach (var item in source)
                Console.Write("{0}\r\n\r\n", item);

            Console.ReadLine();
        }
    }
}