[英]Bulk insert from C# list into SQL Server into multiple tables with foreign key constraints
[英]Bulk Insert of Generic List C# into SQL Server
如何將 C# 中的通用列表批量插入 SQL Server,而不是遍歷列表並一次插入單個項目?
我目前有這個;
private void AddSnapshotData()
{
var password = Cryptography.DecryptString("vhx7Hv7hYD2bF9N4XhN5pkQm8MRfxi+kogALYqwqSuo=");
var figDb = "ZEUS";
var connString =
String.Format(
"Data Source=1xx.x.xx.xxx;Initial Catalog={0};;User ID=appuser;Password={1};MultipleActiveResultSets=True",
figDb, password);
var myConnection = new SqlConnection(connString);
myConnection.Open();
foreach (var holding in _dHoldList)
{
lbStatus.Text = "Adding information to SQL for client: " + holding.ClientNo;
_reports.AddZeusData("tblAllBrooksHoldingsSnapshot", "CliNo, SEDOL, ISIN, QtyHeld, DateOfSnapshot",
"'" + holding.ClientNo + "','" + holding.Sedol + "','" + holding.ISIN + "','" +
holding.QuantityHeld + "','" + DateTime.Today.ToString("yyyyMMdd") + "'", false, myConnection);
}
myConnection.Close();
lbStatus.Visible = false;
}
其中dHoldList
是DHOLDS
的列表;
public class DHOLDS : ExcelReport
{
public String ClientNo { get; set; }
public String Sedol { get; set; }
public Double QuantityHeld { get; set; }
public Double ValueOfStock { get; set; }
public String Depot { get; set; }
public String ValC4 { get; set; }
public String StockR1 { get; set; }
public String StockR2 { get; set; }
public Double BookCost { get; set; }
public String ISIN { get; set; }
}
您可以將列表映射到數據表,然后使用SqlBulkCopy一次插入所有行。
4年后,這是我的貢獻。 我有同樣的問題,我想批量插入,但傳遞了一些不會在數據庫中的字段,特別是 EF 導航屬性,所以我寫了這個通用類:
/// <summary>
/// This class is intended to perform a bulk insert of a list of elements into a table in a Database.
/// This class also allows you to use the same domain classes that you were already using because you
/// can include not mapped properties into the field excludedPropertyNames.
/// </summary>
/// <typeparam name="T">The class that is going to be mapped.</typeparam>
public class BulkInsert<T> where T : class
{
#region Fields
private readonly LoggingService _logger = new LoggingService(typeof(BulkInsert<T>));
private string _connectionString;
private string _tableName;
private IEnumerable<string> _excludedPropertyNames;
private int _batchSize;
private IEnumerable<T> _data;
private DataTable _dataTable;
#endregion
#region Constructor
/// <summary>
/// Initializes a new instance of the <see cref="BulkInsert{T}"/> class.
/// </summary>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">Name of the table.</param>
/// <param name="data">The data.</param>
/// <param name="excludedPropertyNames">The excluded property names.</param>
/// <param name="batchSize">Size of the batch.</param>
public BulkInsert(
string connectionString,
string tableName,
IEnumerable<T> data,
IEnumerable<string> excludedPropertyNames,
int batchSize = 1000)
{
if (string.IsNullOrEmpty(connectionString)) throw new ArgumentNullException(nameof(connectionString));
if (string.IsNullOrEmpty(tableName)) throw new ArgumentNullException(nameof(tableName));
if (data == null) throw new ArgumentNullException(nameof(data));
if (batchSize <= 0) throw new ArgumentOutOfRangeException(nameof(batchSize));
_connectionString = connectionString;
_tableName = tableName;
_batchSize = batchSize;
_data = data;
_excludedPropertyNames = excludedPropertyNames == null ? new List<string>() : excludedPropertyNames;
_dataTable = CreateCustomDataTable();
}
#endregion
#region Public Methods
/// <summary>
/// Inserts the data with a bulk copy inside a transaction.
/// </summary>
public void Insert()
{
using (var connection = new SqlConnection(_connectionString))
{
connection.Open();
SqlTransaction transaction = connection.BeginTransaction();
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default | SqlBulkCopyOptions.KeepIdentity, transaction))
{
bulkCopy.BatchSize = _batchSize;
bulkCopy.DestinationTableName = _tableName;
// Let's fix tons of mapping issues by
// Setting the column mapping in SqlBulkCopy instance:
foreach (DataColumn dataColumn in _dataTable.Columns)
{
bulkCopy.ColumnMappings.Add(dataColumn.ColumnName, dataColumn.ColumnName);
}
try
{
bulkCopy.WriteToServer(_dataTable);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
transaction.Rollback();
connection.Close();
}
}
transaction.Commit();
}
}
#endregion
#region Private Helper Methods
/// <summary>
/// Creates the custom data table.
/// </summary>
private DataTable CreateCustomDataTable()
{
PropertyDescriptorCollection properties = TypeDescriptor.GetProperties(typeof(T));
var table = new DataTable();
foreach (PropertyDescriptor prop in properties)
{
// Just include the not excluded columns
if (_excludedPropertyNames.All(epn => epn != prop.Name))
{
if (prop.PropertyType.Name == "DbGeography")
{
var type = typeof(SqlGeography);
table.Columns.Add(prop.Name, type);
}
else
{
table.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
}
}
}
foreach (T item in _data)
{
DataRow row = table.NewRow();
foreach (PropertyDescriptor prop in properties)
{
// Just include the values in not excluded properties
if (_excludedPropertyNames.All(epn => epn != prop.Name))
{
if (prop.PropertyType.Name == "DbGeography")
{
row[prop.Name] = SqlGeography.Parse(((DbGeography)prop.GetValue(item)).AsText()).MakeValid();
}
else
{
row[prop.Name] = prop.GetValue(item) ?? DBNull.Value;
}
}
}
table.Rows.Add(row);
}
return table;
}
#endregion
}
它的用法如下:
//1st.- You would have a colection of entities:
var myEntities = new List<MyEntity>();
// [...] With thousands or millions of items
// 2nd.- You would create the BulkInsert:
myEntityTypeBulk = new BulkInsert<MyEntity>(_connectionString, "MyEntitiesTableName", myEntities, new[] { "ObjectState", "SkippedEntityProperty1", "SkippedEntityProperty2" });
// 3rd.- You would execute it:
myEntityTypeBulk.Insert();
獲得的性能和此類的可重用性值得此消息。 我希望它有幫助:
胡安
或者,您也可以將列表轉換為 XML,如本博文所述: http://charleskong.com/blog/2009/09/insert-aspnet-objects-to-sql-server/但 SqlBulkCopy 方法似乎更好。
另一句話:如果您想通過迭代代碼中的元素來解決它,那么如果您在一個事務中完成所有插入,則可能會提高性能。
Juan 的解決方案非常適合我,所以我做了一些更改,現在支持異步和一些重構
public class BulkInsert<T> where T : class
{
private readonly string _connectionString;
private readonly string _tableName;
private readonly IEnumerable<string>? _excludedPropertyNames;
private readonly int _batchSize;
private readonly IEnumerable<T> _data;
private readonly DataTable _dataTable;
/// <summary>
/// Initializes a new instance of the <see cref="BulkInsert{T}"/> class.
/// </summary>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">Name of the table.</param>
/// <param name="data">The data.</param>
/// <param name="excludedPropertyNames">The excluded property names.</param>
/// <param name="batchSize">Size of the batch.</param>
public BulkInsert(
string connectionString,
string tableName,
IEnumerable<T> data,
IEnumerable<string>? excludedPropertyNames,
int batchSize = 1000)
{
if (string.IsNullOrEmpty(connectionString)) throw new ArgumentNullException(nameof(connectionString));
if (string.IsNullOrEmpty(tableName)) throw new ArgumentNullException(nameof(tableName));
if (data == null) throw new ArgumentNullException(nameof(data));
if (batchSize <= 0) throw new ArgumentOutOfRangeException(nameof(batchSize));
_connectionString = connectionString;
_tableName = tableName;
_batchSize = batchSize;
_data = data;
_excludedPropertyNames = excludedPropertyNames ?? new List<string>();
_dataTable = CreateCustomDataTable();
}
/// <summary>
/// Inserts the data with a bulk copy inside a transaction.
/// </summary>
public async Task InsertAsync()
{
await using var connection = new SqlConnection(_connectionString);
connection.Open();
var transaction = connection.BeginTransaction();
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default | SqlBulkCopyOptions.KeepIdentity, transaction))
{
bulkCopy.BatchSize = _batchSize;
bulkCopy.DestinationTableName = _tableName;
// Let's fix tons of mapping issues by
// Setting the column mapping in SqlBulkCopy instance:
foreach (DataColumn dataColumn in _dataTable.Columns)
{
bulkCopy.ColumnMappings.Add(dataColumn.ColumnName, dataColumn.ColumnName);
}
try
{
await bulkCopy.WriteToServerAsync(_dataTable);
}
catch (Exception)
{
await transaction.RollbackAsync();
await connection.CloseAsync();
}
}
transaction.Commit();
}
/// <summary>
/// Creates the custom data table.
/// </summary>
private DataTable CreateCustomDataTable()
{
var properties = TypeDescriptor.GetProperties(typeof(T));
var table = new DataTable();
foreach (PropertyDescriptor prop in properties)
{
// Just include the not excluded columns
if (_excludedPropertyNames != null && _excludedPropertyNames.All(epn => epn != prop.Name))
{
table.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
}
}
foreach (T item in _data)
{
var row = table.NewRow();
foreach (PropertyDescriptor prop in properties)
{
// Just include the values in not excluded properties
if (_excludedPropertyNames != null && _excludedPropertyNames.All(epn => epn != prop.Name))
{
row[prop.Name] = prop.GetValue(item) ?? DBNull.Value;
}
}
table.Rows.Add(row);
}
return table;
}
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.