Insert chunk using linq to sql?

Question

in ac# application, I need to insert into sql server 2005 table a lot or records. I split datatable in 2000 records chunk, using linq to sql server but dont work good becasue dont stop ever! I've 900.000 recors and insert a lot more thant this. What I'm doing wrong? This is my code:

int jump = 0;
while (ds.Tables[0].Rows.Count < ds.Tables[0].Rows.Count + 1)
{
    String xmlData = ConvertDataTableToXML(ds.Tables[0].AsEnumerable().Skip(jump).Take(2000 + jump).CopyToDataTable());
    jump = jump + 2001;

    SqlConnection conn = new SqlConnection
        ("Data Source=SERVER;Initial Catalog=DATABASE;Persist Security Info=True;User ID=USER;Password=PASS;");
    conn.Open();
    SqlCommand insert = new SqlCommand
        ("sp_InsertData'" + xmlData + "'", conn);
    insert.ExecuteNonQuery();
    conn.Close();
}

Answer 1

if you understand Temp Tables in SQL Server as well as how to use the OPENXML command you can try the following. it works even if your DBA disables BULK INSERTS

substitute my field names with your own and the field mappings in this portion of the code should match your tables schema / field definitions / data types FROM OPENXML (@xmlHandle, '/NewDataSet/XMLDataTable',1) WITH (

ALTER PROCEDURE [dbo].[sp_InsertData]
(@xmlString VARCHAR(MAX))
AS
BEGIN
    /* Initialize a handle for the XmlDocument */ 
    DECLARE @xmlHandle INT 

    /* 
        Created by @MethodMan you first want to create / declare a TEMP TABLE which 
        Mimic's the structure of the Target table that you are inserting into
    */ 

    DECLARE @someTargetTable TABLE 
    ( 
        [EN_INTFC_ID]           varchar(25), 
        [EN_INTFC_LINE_NUM]     varchar(5), 
        [EN_BILL_SOURCE_ID]     varchar(10),
        [EN_BUSINESS_UNIT]      varchar(12),
        [EN_ASSET_NAME]         varchar(4),
        [EN_POSTING_DATE]       DateTime,
        [EN_FISCAL_YEAR]        varchar(4),
        [EN_FISCAL_PERIOD]      varchar(3),
        [EN_CUSTOMER_ID]        varchar(50),
        [EN_DOC_TYPE]           varchar(4),
        [EN_TARGET_INVOICE]     varchar(16),
        [EN_INVOICE_DT]         DateTime,
        [EN_REVNUE_TYPE]        varchar(15),
        [EN_QTY]                decimal(15,0),
        [EN_GROSS_EXT_AMT]      decimal(25,2),
        [EN_DESCR]              varchar(50),
        [EN_CONTRACT]           varchar(20),
        [EN_PRODUCT_TYPE]       varchar(15),
        [EN_UNIT_OF_MEASURE]    varchar(3)
    ) 

    /* 
    Create the XmlDocument using the handle above and the Xml 
    string as parameters. If your stored procedure has an varchar input 
    parameter named @xmlString, it would look like this instead: 
    EXEC sp_xml_preparedocument @xmlHandle output,@xmlString 
    */ 
    EXEC sp_xml_preparedocument @xmlHandle output, @xmlString 


    /* 
    Use the OPENXML method to query the XmlDocument starting at 
    /NewDataSet/SampleDataTable node.
    */ 
    INSERT INTO @someTargetTable 
    SELECT  [EN_INTFC_ID],  
            [EN_INTFC_LINE_NUM],  
            [EN_BILL_SOURCE_ID],     
            [EN_BUSINESS_UNIT],     
            [EN_ASSET_NAME],    
            [EN_POSTING_DATE], 
            [EN_FISCAL_YEAR],   
            [EN_FISCAL_PERIOD], 
            [EN_CUSTOMER_ID],     
            [EN_DOC_TYPE],    
            [EN_TARGET_INVOICE],   
            [EN_INVOICE_DT],    
            [EN_REVNUE_TYPE],
            [EN_QTY],   
            [EN_GROSS_EXT_AMT],  
            [EN_DESCR],
            [EN_CONTRACT],
            [EN_PRODUCT_TYPE],
            [EN_UNIT_OF_MEASURE] 
    FROM OPENXML (@xmlHandle, '/NewDataSet/XMLDataTable',1) 
        WITH (
            [EN_INTFC_ID]         varchar(25) '@EN_INTFC_ID', 
            [EN_INTFC_LINE_NUM]   varchar(5) '@EN_INTFC_LINE_NUM', 
            [EN_BILL_SOURCE_ID]   varchar(10) '@EN_BILL_SOURCE_ID',
            [EN_BUSINESS_UNIT]    varchar(12) '@EN_BUSINESS_UNIT',
            [EN_ASSET_NAME]       varchar(4) '@EN_ASSET_NAME',
            [EN_POSTING_DATE]     DateTime '@EN_POSTING_DATE',
            [EN_FISCAL_YEAR]      varchar(4) '@EN_FISCAL_YEAR',
            [EN_FISCAL_PERIOD]    varchar(3) '@EN_FISCAL_PERIOD',
            [EN_CUSTOMER_ID]      varchar(50) '@EN_CUSTOMER_ID',
            [EN_DOC_TYPE]         varchar(4) '@EN_DOC_TYPE',
            [EN_TARGET_INVOICE]   varchar(16) '@EN_TARGET_INVOICE',
            [EN_INVOICE_DT]       DateTime '@EN_INVOICE_DT',
            [EN_REVNUE_TYPE]      varchar(15) '@EN_REVNUE_TYPE',
            [EN_QTY]              decimal(15,0) '@EN_QTY',
            [EN_GROSS_EXT_AMT]    decimal(25,2) '@EN_GROSS_EXT_AMT',
            [EN_DESCR]            varchar(50) '@EN_DESCR',
            [EN_CONTRACT]         varchar(20) '@EN_CONTRACT',
            [EN_PRODUCT_TYPE]     varchar(15)   '@EN_PRODUCT_TYPE',
            [EN_UNIT_OF_MEASURE] varchar(3)  '@EN_UNIT_OF_MEASURE'
             )

    /*Insert the records into the table variable  */ 

    INSERT INTO Your_Actual_Table_Name (
           [EN_INTFC_ID],  
           [EN_INTFC_LINE_NUM],  
           [EN_BILL_SOURCE_ID],     
           [EN_BUSINESS_UNIT],     
           [EN_ASSET_NAME],    
           [EN_POSTING_DATE], 
           [EN_FISCAL_YEAR],   
           [EN_FISCAL_PERIOD], 
           [EN_CUSTOMER_ID],     
           [EN_DOC_TYPE],    
           [EN_TARGET_INVOICE],   
           [EN_INVOICE_DT],    
           [EN_REVNUE_TYPE],
           [EN_QTY],   
           [EN_GROSS_EXT_AMT],  
           [EN_DESCR],
           [EN_CONTRACT],
           [EN_PRODUCT_TYPE],
           [EN_UNIT_OF_MEASURE] ) 
    (SELECT [EN_INTFC_ID],  
           [EN_INTFC_LINE_NUM],  
           [EN_BILL_SOURCE_ID],     
           [EN_BUSINESS_UNIT],     
           [EN_ASSET_NAME],    
           [EN_POSTING_DATE], 
           [EN_FISCAL_YEAR],   
           [EN_FISCAL_PERIOD], 
           [EN_CUSTOMER_ID],     
           [EN_DOC_TYPE],    
           [EN_TARGET_INVOICE],   
           [EN_INVOICE_DT],    
           [EN_REVNUE_TYPE],
           [EN_QTY],   
           [EN_GROSS_EXT_AMT],  
           [EN_DESCR],
           [EN_CONTRACT],
           [EN_PRODUCT_TYPE],
           [EN_UNIT_OF_MEASURE] 
    FROM @someTargetTable)
    /* Remove the document from memory */ 
    EXEC sp_xml_removedocument @xmlHandle
END

// your sql command below.
SqlCommand insert = new SqlCommand ("sp_InsertData '" + xmlData + "'", conn); 
insert.CommandTimeout = 5000; 
insert.ExecuteNonQuery(); 
conn.Close()

// my code and how you can utilize the using(){} statement along with code to convert a DataTable to XML then pass that xml to the stored procedure which I have depicted above

private bool ProcessSomeDataTableToXML(DataTable dataTable)
{
    String xmlData = ConvertDataTableToXML(dataTable);
    var ConnString = System.Configuration.ConfigurationManager.ConnectionStrings["yourdatabase"].ConnectionString;
    using (SqlConnection connection = new SqlConnection(ConnString))
    {
        using (SqlCommand command = new SqlCommand("sp_InsertData '" + xmlData + "'", connection))
        {
            connection.Open();
            try
            {
                command.ExecuteNonQuery();
                fileInserted = true;
            }
            catch (SqlException sqlEx)
            {
                fileInserted = false;

                Console.WriteLine(sqlEx.Message);
            }
        }
    }
    return fileInserted;
}

private static string ConvertDataTableToXML(DataTable dtData)
{
    DataSet dsData = new DataSet();
    StringBuilder sbSQL;
    StringWriter swSQL;
    string XMLformat;
    try
    {
        sbSQL = new StringBuilder();
        swSQL = new StringWriter(sbSQL);
        dsData.Merge(dtData, true, MissingSchemaAction.AddWithKey);
        dsData.Tables[0].TableName = "XMLDataTable";
        foreach (DataColumn col in dsData.Tables[0].Columns)
        {
            col.ColumnMapping = MappingType.Attribute;
        }
        dsData.WriteXml(swSQL, XmlWriteMode.WriteSchema);
        XMLformat = sbSQL.ToString();
        sbSQL = null;
        swSQL = null;
        return XMLformat;
    }
    catch (Exception sysException)
    {
        throw sysException;
    }
}

Answer 2

An obvious problem is in your while condition:

ds.Tables[0].Rows.Count < ds.Tables[0].Rows.Count + 1

ie count < count + 1

ie true

Your loop is designed never to stop. You might change it to

while (jump < ds.Tables[0].Rows.Count)

Answer 3

1) You have an infite loop (ds.Tables[0].Rows.Count < ds.Tables[0].Rows.Count + 1)

2) Connection management: You don't need to open an close the connection on each iteration. Use a using block for disposing the connection once you are done with it.