[英]Increase performance of SSIS Package dumping data from Excel to SQL Server table
我創建了一個SSIS包,該包將數據從Excel文件轉儲到SQL Server表。 該軟件包僅包含2個任務:腳本組件任務和OLE DB目標
請在下面找到我在腳本任務中添加的代碼:
/* Microsoft SQL Server Integration Services Script Component
* Write scripts using Microsoft Visual C# 2008.
* ScriptMain is the entry point class of the script.*/
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Pipeline.Wrapper;
using Microsoft.SqlServer.Dts.Runtime.Wrapper;
using System.IO;
using System.Data.OleDb;
using System.Xml;
[Microsoft.SqlServer.Dts.Pipeline.SSISScriptComponentEntryPointAttribute]
public class ScriptMain : UserComponent
{
private OleDbDataReader excelReader;
private OleDbConnection excelConnection;
private OleDbCommand excelCommand;
DataTable tablesInFile;
string currentTable;
public override void PreExecute()
{
base.PreExecute();
GetDataFromExcelToReader(Variables.FilePath);
}
public override void PostExecute()
{
base.PostExecute();
excelReader.Close();
excelConnection.Close();
}
//private void GetDataFromExcelToReader(string p_strFileName, string p_strTabName)
private void GetDataFromExcelToReader(string p_strFileName)
{
string l_strConnectionString;
if (File.Exists(p_strFileName))
{
l_strConnectionString = "Provider=Microsoft.ACE.OLEDB.12.0;" +
"Data Source=" + p_strFileName + ";Extended Properties=\"Excel 12.0;HDR=YES;IMEX=1\";";
excelConnection = new OleDbConnection(l_strConnectionString);
excelConnection.Open();
tablesInFile = excelConnection.GetSchema("Tables");
foreach (DataRow tableInFile in tablesInFile.Rows)
{
currentTable = tableInFile["TABLE_NAME"].ToString();
break;
}
excelCommand = excelConnection.CreateCommand();
//excelCommand.CommandText = "SELECT * FROM [" + p_strTabName + "A1:AX1048576]";
excelCommand.CommandText = "SELECT * FROM [" + currentTable + "]";
excelCommand.CommandType = CommandType.Text;
excelReader = excelCommand.ExecuteReader();
}
}
public override void CreateNewOutputRows()
{
int counter = 0;
while (excelReader.Read())
{
Output0Buffer.AddRow();
Output0Buffer.Source = Variables.Source.ToString();
Output0Buffer.Column = excelReader.FieldCount > 0 ? excelReader[0].ToString() : null;
Output0Buffer.Column1 = excelReader.FieldCount > 1 ? excelReader[1].ToString() : null;
Output0Buffer.Column2 = excelReader.FieldCount > 2 ? excelReader[2].ToString() : null;
Output0Buffer.Column3 = excelReader.FieldCount > 3 ? excelReader[3].ToString() : null;
Output0Buffer.Column4 = excelReader.FieldCount > 4 ? excelReader[4].ToString() : null;
Output0Buffer.Column5 = excelReader.FieldCount > 5 ? excelReader[5].ToString() : null;
Output0Buffer.Column6 = excelReader.FieldCount > 6 ? excelReader[6].ToString() : null;
Output0Buffer.Column7 = excelReader.FieldCount > 7 ? excelReader[7].ToString() : null;
Output0Buffer.Column8 = excelReader.FieldCount > 8 ? excelReader[8].ToString() : null;
Output0Buffer.Column9 = excelReader.FieldCount > 9 ? excelReader[9].ToString() : null;
Output0Buffer.Column10 = excelReader.FieldCount > 10 ? excelReader[10].ToString() : null;
Output0Buffer.Column11 = excelReader.FieldCount > 11 ? excelReader[11].ToString() : null;
Output0Buffer.Column12 = excelReader.FieldCount > 12 ? excelReader[12].ToString() : null;
Output0Buffer.Column13 = excelReader.FieldCount > 13 ? excelReader[13].ToString() : null;
Output0Buffer.Column14 = excelReader.FieldCount > 14 ? excelReader[14].ToString() : null;
Output0Buffer.Column15 = excelReader.FieldCount > 15 ? excelReader[15].ToString() : null;
Output0Buffer.Column16 = excelReader.FieldCount > 16 ? excelReader[16].ToString() : null;
Output0Buffer.Column17 = excelReader.FieldCount > 17 ? excelReader[17].ToString() : null;
Output0Buffer.Column18 = excelReader.FieldCount > 18 ? excelReader[18].ToString() : null;
Output0Buffer.Column19 = excelReader.FieldCount > 19 ? excelReader[19].ToString() : null;
Output0Buffer.Column20 = excelReader.FieldCount > 20 ? excelReader[20].ToString() : null;
Output0Buffer.Column21 = excelReader.FieldCount > 21 ? excelReader[21].ToString() : null;
Output0Buffer.Column22 = excelReader.FieldCount > 22 ? excelReader[22].ToString() : null;
Output0Buffer.Column23 = excelReader.FieldCount > 23 ? excelReader[23].ToString() : null;
Output0Buffer.Column24 = excelReader.FieldCount > 24 ? excelReader[24].ToString() : null;
Output0Buffer.Column25 = excelReader.FieldCount > 25 ? excelReader[25].ToString() : null;
Output0Buffer.Column26 = excelReader.FieldCount > 26 ? excelReader[26].ToString() : null;
Output0Buffer.Column27 = excelReader.FieldCount > 27 ? excelReader[27].ToString() : null;
Output0Buffer.Column28 = excelReader.FieldCount > 28 ? excelReader[28].ToString() : null;
Output0Buffer.Column29 = excelReader.FieldCount > 29 ? excelReader[29].ToString() : null;
Output0Buffer.Column30 = excelReader.FieldCount > 30 ? excelReader[30].ToString() : null;
Output0Buffer.Column31 = excelReader.FieldCount > 31 ? excelReader[31].ToString() : null;
Output0Buffer.Column32 = excelReader.FieldCount > 32 ? excelReader[32].ToString() : null;
Output0Buffer.Column33 = excelReader.FieldCount > 33 ? excelReader[33].ToString() : null;
Output0Buffer.Column34 = excelReader.FieldCount > 34 ? excelReader[34].ToString() : null;
Output0Buffer.Column35 = excelReader.FieldCount > 35 ? excelReader[35].ToString() : null;
Output0Buffer.Column36 = excelReader.FieldCount > 36 ? excelReader[36].ToString() : null;
Output0Buffer.Column37 = excelReader.FieldCount > 37 ? excelReader[37].ToString() : null;
Output0Buffer.Column38 = excelReader.FieldCount > 38 ? excelReader[38].ToString() : null;
Output0Buffer.Column39 = excelReader.FieldCount > 39 ? excelReader[39].ToString() : null;
Output0Buffer.Column40 = excelReader.FieldCount > 40 ? excelReader[40].ToString() : null;
Output0Buffer.Column41 = excelReader.FieldCount > 41 ? excelReader[41].ToString() : null;
Output0Buffer.Column42 = excelReader.FieldCount > 42 ? excelReader[42].ToString() : null;
Output0Buffer.Column43 = excelReader.FieldCount > 43 ? excelReader[43].ToString() : null;
Output0Buffer.Column44 = excelReader.FieldCount > 44 ? excelReader[44].ToString() : null;
Output0Buffer.Column45 = excelReader.FieldCount > 45 ? excelReader[45].ToString() : null;
Output0Buffer.Column46 = excelReader.FieldCount > 46 ? excelReader[46].ToString() : null;
Output0Buffer.Column47 = excelReader.FieldCount > 47 ? excelReader[47].ToString() : null;
Output0Buffer.Column48 = excelReader.FieldCount > 48 ? excelReader[48].ToString() : null;
Output0Buffer.Column49 = excelReader.FieldCount > 49 ? excelReader[49].ToString() : null;
}
}
}
請在下面找到在OLE DB目標任務中完成的配置快照:
當前,如果我運行包含約99000條記錄的xlsx文件包,則需要4分鍾的時間將數據轉儲到相當大的數據庫中。 期望在幾秒鍾內完成該過程。 如何在此程序包的第二部分中實現此目標。 我的Excel文件是動態文件,可以包含最少1條記錄,最多1.5-2條lacs記錄。
請幫忙。 提前致謝!!
在OLE DB目標任務中進行更改可以達到我的目的,並且可以極大地提高性能。
我通過調整幾個屬性來使用“ 快速加載”選項。 現在,我可以在大約15秒內轉儲2laks記錄。
如果您正在尋找性能的觀點,那么我建議您使用以下方法:
1)在數據庫中創建一個完全相同的臨時表,該表將在該過程完成后被刪除或截斷。
2)將記錄從excel源插入TEMP TABLE。
3)下一步,從TEMP TABLE中獲取記錄,並將其插入到目標表中。
注意:第一步,您可以拖動“ Execute SQL Task
要么在存儲過程中執行,要么僅執行簡單查詢
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.