[英]import from text file to SQL Server Database, is ADO.NET too slow?
我的程序現在仍在運行,以將數據從日志文件導入遠程SQL Server數據庫。 日志文件大小約為80MB,包含大約470000行,包含大約25000行數據。 我的程序只能導入300行/秒,這真的很糟糕。 :(
public static int ImportData(string strPath)
{
//NameValueCollection collection = ConfigurationManager.AppSettings;
using (TextReader sr = new StreamReader(strPath))
{
sr.ReadLine(); //ignore three first lines of log file
sr.ReadLine();
sr.ReadLine();
string strLine;
var cn = new SqlConnection(ConnectionString);
cn.Open();
while ((strLine = sr.ReadLine()) != null)
{
{
if (strLine.Trim() != "") //if not a blank line, then import into database
{
InsertData(strLine, cn);
_count++;
}
}
}
cn.Close();
sr.Close();
return _count;
}
}
InsertData只是使用ADO.NET的普通插入方法。 它使用解析方法:
public Data(string strLine)
{
string[] list = strLine.Split(new[] {'\t'});
try
{
Senttime = DateTime.Parse(list[0] + " " + list[1]);
}
catch (Exception)
{
}
Clientip = list[2];
Clienthostname = list[3];
Partnername = list[4];
Serverhostname = list[5];
Serverip = list[6];
Recipientaddress = list[7];
Eventid = Convert.ToInt16(list[8]);
Msgid = list[9];
Priority = Convert.ToInt16(list[10]);
Recipientreportstatus = Convert.ToByte(list[11]);
Totalbytes = Convert.ToInt32(list[12]);
Numberrecipient = Convert.ToInt16(list[13]);
DateTime temp;
if (DateTime.TryParse(list[14], out temp))
{
OriginationTime = temp;
}
else
{
OriginationTime = null;
}
Encryption = list[15];
ServiceVersion = list[16];
LinkedMsgid = list[17];
MessageSubject = list[18];
SenderAddress = list[19];
}
InsertData方法:
private static void InsertData(string strLine, SqlConnection cn)
{
var dt = new Data(strLine); //parse the log line into proper fields
const string cnnStr =
"INSERT INTO LOGDATA ([SentTime]," + "[client-ip]," +
"[Client-hostname]," + "[Partner-Name]," + "[Server-hostname]," +
"[server-IP]," + "[Recipient-Address]," + "[Event-ID]," + "[MSGID]," +
"[Priority]," + "[Recipient-Report-Status]," + "[total-bytes]," +
"[Number-Recipients]," + "[Origination-Time]," + "[Encryption]," +
"[service-Version]," + "[Linked-MSGID]," + "[Message-Subject]," +
"[Sender-Address]) " + " VALUES ( " + "@Senttime," + "@Clientip," +
"@Clienthostname," + "@Partnername," + "@Serverhostname," + "@Serverip," +
"@Recipientaddress," + "@Eventid," + "@Msgid," + "@Priority," +
"@Recipientreportstatus," + "@Totalbytes," + "@Numberrecipient," +
"@OriginationTime," + "@Encryption," + "@ServiceVersion," +
"@LinkedMsgid," + "@MessageSubject," + "@SenderAddress)";
var cmd = new SqlCommand(cnnStr, cn) {CommandType = CommandType.Text};
cmd.Parameters.AddWithValue("@Senttime", dt.Senttime);
cmd.Parameters.AddWithValue("@Clientip", dt.Clientip);
cmd.Parameters.AddWithValue("@Clienthostname", dt.Clienthostname);
cmd.Parameters.AddWithValue("@Partnername", dt.Partnername);
cmd.Parameters.AddWithValue("@Serverhostname", dt.Serverhostname);
cmd.Parameters.AddWithValue("@Serverip", dt.Serverip);
cmd.Parameters.AddWithValue("@Recipientaddress", dt.Recipientaddress);
cmd.Parameters.AddWithValue("@Eventid", dt.Eventid);
cmd.Parameters.AddWithValue("@Msgid", dt.Msgid);
cmd.Parameters.AddWithValue("@Priority", dt.Priority);
cmd.Parameters.AddWithValue("@Recipientreportstatus", dt.Recipientreportstatus);
cmd.Parameters.AddWithValue("@Totalbytes", dt.Totalbytes);
cmd.Parameters.AddWithValue("@Numberrecipient", dt.Numberrecipient);
if (dt.OriginationTime != null)
cmd.Parameters.AddWithValue("@OriginationTime", dt.OriginationTime);
else
cmd.Parameters.AddWithValue("@OriginationTime", DBNull.Value);
//if OriginationTime was null, then insert with null value to this column
cmd.Parameters.AddWithValue("@Encryption", dt.Encryption);
cmd.Parameters.AddWithValue("@ServiceVersion", dt.ServiceVersion);
cmd.Parameters.AddWithValue("@LinkedMsgid", dt.LinkedMsgid);
cmd.Parameters.AddWithValue("@MessageSubject", dt.MessageSubject);
cmd.Parameters.AddWithValue("@SenderAddress", dt.SenderAddress);
cmd.ExecuteNonQuery();
}
我的程序如何運行得更快? 非常感謝!
使用SqlBulkCopy 。
編輯:我創建了一個IDataReader
的最小實現並創建了一個Batch
類型,以便我可以使用SqlBulkCopy
插入任意內存數據。 這是重要的一點:
IDataReader dr = batch.GetDataReader();
using (SqlTransaction tx = _connection.BeginTransaction())
{
try
{
using (SqlBulkCopy sqlBulkCopy =
new SqlBulkCopy(_connection, SqlBulkCopyOptions.Default, tx))
{
sqlBulkCopy.DestinationTableName = TableName;
SetColumnMappings(sqlBulkCopy.ColumnMappings);
sqlBulkCopy.WriteToServer(dr);
tx.Commit();
}
}
catch
{
tx.Rollback();
throw;
}
}
其余的實現留給讀者:)
提示:您需要實現的IDataReader
的唯一位是Read
, GetValue
和FieldCount
。
嗯,讓我們稍微分解一下。
在偽代碼中你所做的是ff:
現在這樣做的根本問題是:
InsertData
方法正在執行的操作之前,我們不會知道 執行此操作的最佳方法是將文件整體解析,然后批量插入它們。 您可以使用SqlBulkCopy (由Matt Howells建議)或SQL Server Integration Services執行此操作。
如果你想堅持使用ADO.NET,你可以將INSERT語句匯集在一起,然后將它們傳遞給一個大的SQLCommand,而不是這樣做,例如,為每個insert語句設置一個SQLCommand對象。
您為每行數據創建SqlCommand對象。 因此,最簡單的改進就是創造一個
private static SqlCommand cmdInsert
並使用Parameters.Add()方法聲明參數。 然后,對於每個數據行,使用設置參數值
cmdInsert.Parameters["@paramXXX"].Value = valueXXX;
第二個性能改進可能是跳過為每一行創建Data對象,並直接從list []數組中分配Parameter值。
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.