C# 使用foreach循环批量复制数据到SQL服务器

Question

I have a C# console app which reads million source data from a CSV file and inserts them into SQL Server in batches.

I group the data by 1000 count and use foreach to loop the groups. Each loop creates a new SqlConnection and a new SqlBulkCopy objects and disposes them at the end of the loop.

for (int index = 0; index < dts.Count; index++)
{
    DataTable _dt = dts[index];
    try
    {
        using (SqlConnection connection = new SqlConnection(conn))
        {
            await connection.OpenAsync();
            //using (SqlTransaction trans = connection.BeginTransaction())
            using (CancellationTokenSource cts = new CancellationTokenSource())
            using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
            {
                bulkCopy.DestinationTableName = "[dbo].[XXXX]";
                bulkCopy.BatchSize = 1000;
                bulkCopy.WriteToServer(_dt);
            }
        }
    }
}                

The first 10 groups work fast in 1.x seconds, but after that it takes 4x secs to 60secs every group.

Answer 1

If you use other cores of the processor, you will use a little more speed. splitting data by number of cores using threads saves time.

 Thread t1 = new Thread(new ThreadStart(Thread1));
  
 Thread t2 = new Thread(new ThreadStart(Thread2));

    t1.Start();
    t2.Start();
              
   int piece = dts.Count / 2;

   public static void Thread1()
    {
        for (int index = 0; index < piece; index++)
        {
            DataTable _dt = dts[index];
            try
            {
                using (SqlConnection connection = new SqlConnection(conn))
                {
                    await connection.OpenAsync();
                    //using (SqlTransaction trans = connection.BeginTransaction())
                    using (CancellationTokenSource cts = new CancellationTokenSource())
                    using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
                    {
                        bulkCopy.DestinationTableName = "[dbo].[XXXX]";
                        bulkCopy.BatchSize = 1000;
                        bulkCopy.WriteToServer(_dt);
                    }
                }
            }
            }
    }
    public static void Thread2()
    {
        for (int index = piece; index < dts.Count; index++)
        {
            DataTable _dt = dts[index];
            try
            {
                using (SqlConnection connection = new SqlConnection(conn))
                {
                    await connection.OpenAsync();
                    //using (SqlTransaction trans = connection.BeginTransaction())
                    using (CancellationTokenSource cts = new CancellationTokenSource())
                    using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
                    {
                        bulkCopy.DestinationTableName = "[dbo].[XXXX]";
                        bulkCopy.BatchSize = 1000;
                        bulkCopy.WriteToServer(_dt);
                    }
                }
            }
            }
    }

if you just want foreach

 foreach(DataTable _dt in dts){
         try
                {
                    using (SqlConnection connection = new SqlConnection(conn))
                    {
                        await connection.OpenAsync();
                        //using (SqlTransaction trans = connection.BeginTransaction())
                        using (CancellationTokenSource cts = new CancellationTokenSource())
                        using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
                        {
                            bulkCopy.DestinationTableName = "[dbo].[XXXX]";
                            bulkCopy.BatchSize = 1000;
                            bulkCopy.WriteToServer(_dt);
                      }
                  }
             }            
        }