Insert more than 1000 records in SQL Server 2008 in one go

Question

I am creating a WPF application where clients can insert multiple records into a database.

Minimum limit of records to insert is 1000.

I used below code to insert the same in c#.

var response = SearchEngine.Search(objSearchRequest);

if (response.ResponseStatus == SearchResponse.SearchResponseStatus.Success)
{
    ImageClippingQueue objImageClippingQueue;

    if (response.SearchResults.Count > 0)
    {
       foreach (var item in response.SearchResults)
       {
          objImageClippingQueue = new ImageClippingQueue();
          objImageClippingQueue.ImageID = item.Value.ImageId;
          objImageClippingQueue.pubid = item.Value.PubId;

          new CommonMethods().InsertImageData(objImageClippingQueue);
       }

       UpdateClippingSearchKeyword(keywordId, response.TotalAvailableResults, (int)totalRecordsImported);
    }
}

This method InsertImageData inserts data into database.

Is there any fastest way to go so that database hit should be one and not hitting again and again.

Thanks

Answer 1

You can make use of the SqlBulkCopy class to insert the data. The following example was taken from http://msdn.microsoft.com/en-us/library/system.data.sqlclient.sqlbulkcopy(v=vs.110).aspx

            using (SqlBulkCopy bulkCopy =
                       new SqlBulkCopy(destinationConnection))
            {
                bulkCopy.DestinationTableName =
                    "dbo.BulkCopyDemoMatchingColumns";

                try
                {
                    // Write from the source to the destination.
                    bulkCopy.WriteToServer(reader);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.Message);
                }
                finally
                {
                    // Close the SqlDataReader. The SqlBulkCopy 
                    // object is automatically closed at the end 
                    // of the using block.
                    reader.Close();
                }
            }

Answer 2

I've done something similar to the following:

public static void insert(List<myRecord> records)
{
    string sql = "insert into myTable (col1, col2) values ";

    int i = 0;
    foreach (record in records)
    {
        if (0 < i) sql += ",";

        sql += "(@col1_" + i + ",@col2_" + i + ")";
        ++i;
    }

    SqlConnection conn = new SqlConnection(myConnStr);
    SqlCommand cmd = new SqlCommand(sql, conn);

    i = 0;
    foreach (record in records)
    {
        cmd.Parameters.Add("@col1_" + i, System.Data.SqlDbType.Int).Value = record.Val1;
        cmd.Parameters.Add("@col2_" + i, System.Data.SqlDbType.Int).Value = record.Val2;
        ++i;
    }

    // and so on...
}

I've also had to limit it to only inserting say 500 records per insert but that is hidden from the client. Using a StringBuilder would also be an improvement.

Answer 3

I used a store procedure to complete this task.

Link: http://programcall.com/11/dotnet/how-to-read-data-from-xml-string--and-insert-to-table-sql-server.aspx

-- To import multiple records into database
Create PROCEDURE SP_Importimagedata (
 @xmlData XML 
)

AS
BEGIN

    INSERT INTO  [imagedata](
    ImageID,
    ImageClippingStatusID,
    [Priority],
    UserID,
    pubid,
    ClipTypeID
    )

    SELECT
    COALESCE([Table].[Column].value('ImageID[1]', 'int'),0) as 'ImageID',
    [Table].[Column].value('ImageClippingStatusID[1]', 'int') as 'ImageClippingStatusID',
    [Table].[Column].value(' Priority[1]', 'int') as 'Priority',
    [Table].[Column].value(' UserID[1]', 'int') as 'UserID',
    [Table].[Column].value(' pubid[1]', 'int') as 'pubid',
    [Table].[Column].value(' ClipTypeID[1]', 'int') as 'ClipTypeID'
    FROM @xmlData.nodes('/Images/Image') as [Table]([Column])
END


 var response = SearchEngine.Search(objSearchRequest);

            if (response.ResponseStatus == SearchResponse.SearchResponseStatus.Success)
            {
                if (response.SearchResults.Count > 0)
                {
                    sbCreateDataXml = new StringBuilder();
                    sbCreateDataXml.Append("<Images>");
                    foreach (var item in response.SearchResults)
                    {
                        sbCreateDataXml.Append("<Image>");
                        sbCreateDataXml.Append("<ImageID>" + item.Value.ImageId + "</ImageID>");
                        sbCreateDataXml.Append("<ImageClippingStatusID>" + imageClippingStatusID + "</ImageClippingStatusID>");
                        sbCreateDataXml.Append("<Priority>" + priority + "</Priority>");
                        sbCreateDataXml.Append("<UserID>" + userID + "</UserID>");
                        sbCreateDataXml.Append("<pubid>" + item.Value.PubId + "</pubid>");
                        sbCreateDataXml.Append("<ClipTypeID>" + clipTypeID + "</ClipTypeID>");
                        sbCreateDataXml.Append("</Image>");
                        counter++;
                    }
                    sbCreateDataXml.Append("</Images>");
                    new CommonMethods().InsertImageClippingQueue(sbCreateDataXml.ToString());
                    totalRecordsImported = (int)totalRecordsImported + counter;
                    UpdateClippingSearchKeyword(keywordId, response.TotalAvailableResults, (int)totalRecordsImported);
                }
            }

Its works better.

Please let me know if we can do it better than this.

Thank you.

Answer 4

You can use sql bulk insert :

DataTable table = new DataTable("States");
// construct DataTable
table.Columns.Add(new DataColumn("id_state", typeof(int))); 
table.Columns.Add(new DataColumn("state_name", typeof(string)));

// note: if "id_state" is defined as an identity column in your DB,
// row values for that column will be ignored during the bulk copy
table.Rows.Add("1", "Atlanta");
table.Rows.Add("2", "Chicago");
table.Rows.Add("3", "Springfield");

using(SqlBulkCopy bulkCopy = new SqlBulkCopy(connectionString))
{
  bulkCopy.BulkCopyTimeout = 600; // in seconds
  bulkCopy.DestinationTableName = "state";
  bulkCopy.WriteToServer(table);
}