[英]Bulk Insert of Generic List C# into SQL Server
How can I bulk insert a generic list in c# into SQL Server, rather than looping through the list and inserting individual items one at a time?如何将 C# 中的通用列表批量插入 SQL Server,而不是遍历列表并一次插入单个项目?
I currently have this;我目前有这个;
private void AddSnapshotData()
{
var password = Cryptography.DecryptString("vhx7Hv7hYD2bF9N4XhN5pkQm8MRfxi+kogALYqwqSuo=");
var figDb = "ZEUS";
var connString =
String.Format(
"Data Source=1xx.x.xx.xxx;Initial Catalog={0};;User ID=appuser;Password={1};MultipleActiveResultSets=True",
figDb, password);
var myConnection = new SqlConnection(connString);
myConnection.Open();
foreach (var holding in _dHoldList)
{
lbStatus.Text = "Adding information to SQL for client: " + holding.ClientNo;
_reports.AddZeusData("tblAllBrooksHoldingsSnapshot", "CliNo, SEDOL, ISIN, QtyHeld, DateOfSnapshot",
"'" + holding.ClientNo + "','" + holding.Sedol + "','" + holding.ISIN + "','" +
holding.QuantityHeld + "','" + DateTime.Today.ToString("yyyyMMdd") + "'", false, myConnection);
}
myConnection.Close();
lbStatus.Visible = false;
}
Where dHoldList
is a list of DHOLDS
;其中
dHoldList
是DHOLDS
的列表;
public class DHOLDS : ExcelReport
{
public String ClientNo { get; set; }
public String Sedol { get; set; }
public Double QuantityHeld { get; set; }
public Double ValueOfStock { get; set; }
public String Depot { get; set; }
public String ValC4 { get; set; }
public String StockR1 { get; set; }
public String StockR2 { get; set; }
public Double BookCost { get; set; }
public String ISIN { get; set; }
}
您可以将列表映射到数据表,然后使用SqlBulkCopy一次插入所有行。
4 years later this is my contribution. 4年后,这是我的贡献。 I had the same problem, I wanted to to bulk insert but passing over some fields that weren't going to be in the database, specifically EF navigation properties, so I wrote this generic class:
我有同样的问题,我想批量插入,但传递了一些不会在数据库中的字段,特别是 EF 导航属性,所以我写了这个通用类:
/// <summary>
/// This class is intended to perform a bulk insert of a list of elements into a table in a Database.
/// This class also allows you to use the same domain classes that you were already using because you
/// can include not mapped properties into the field excludedPropertyNames.
/// </summary>
/// <typeparam name="T">The class that is going to be mapped.</typeparam>
public class BulkInsert<T> where T : class
{
#region Fields
private readonly LoggingService _logger = new LoggingService(typeof(BulkInsert<T>));
private string _connectionString;
private string _tableName;
private IEnumerable<string> _excludedPropertyNames;
private int _batchSize;
private IEnumerable<T> _data;
private DataTable _dataTable;
#endregion
#region Constructor
/// <summary>
/// Initializes a new instance of the <see cref="BulkInsert{T}"/> class.
/// </summary>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">Name of the table.</param>
/// <param name="data">The data.</param>
/// <param name="excludedPropertyNames">The excluded property names.</param>
/// <param name="batchSize">Size of the batch.</param>
public BulkInsert(
string connectionString,
string tableName,
IEnumerable<T> data,
IEnumerable<string> excludedPropertyNames,
int batchSize = 1000)
{
if (string.IsNullOrEmpty(connectionString)) throw new ArgumentNullException(nameof(connectionString));
if (string.IsNullOrEmpty(tableName)) throw new ArgumentNullException(nameof(tableName));
if (data == null) throw new ArgumentNullException(nameof(data));
if (batchSize <= 0) throw new ArgumentOutOfRangeException(nameof(batchSize));
_connectionString = connectionString;
_tableName = tableName;
_batchSize = batchSize;
_data = data;
_excludedPropertyNames = excludedPropertyNames == null ? new List<string>() : excludedPropertyNames;
_dataTable = CreateCustomDataTable();
}
#endregion
#region Public Methods
/// <summary>
/// Inserts the data with a bulk copy inside a transaction.
/// </summary>
public void Insert()
{
using (var connection = new SqlConnection(_connectionString))
{
connection.Open();
SqlTransaction transaction = connection.BeginTransaction();
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default | SqlBulkCopyOptions.KeepIdentity, transaction))
{
bulkCopy.BatchSize = _batchSize;
bulkCopy.DestinationTableName = _tableName;
// Let's fix tons of mapping issues by
// Setting the column mapping in SqlBulkCopy instance:
foreach (DataColumn dataColumn in _dataTable.Columns)
{
bulkCopy.ColumnMappings.Add(dataColumn.ColumnName, dataColumn.ColumnName);
}
try
{
bulkCopy.WriteToServer(_dataTable);
}
catch (Exception ex)
{
_logger.LogError(ex.Message);
transaction.Rollback();
connection.Close();
}
}
transaction.Commit();
}
}
#endregion
#region Private Helper Methods
/// <summary>
/// Creates the custom data table.
/// </summary>
private DataTable CreateCustomDataTable()
{
PropertyDescriptorCollection properties = TypeDescriptor.GetProperties(typeof(T));
var table = new DataTable();
foreach (PropertyDescriptor prop in properties)
{
// Just include the not excluded columns
if (_excludedPropertyNames.All(epn => epn != prop.Name))
{
if (prop.PropertyType.Name == "DbGeography")
{
var type = typeof(SqlGeography);
table.Columns.Add(prop.Name, type);
}
else
{
table.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
}
}
}
foreach (T item in _data)
{
DataRow row = table.NewRow();
foreach (PropertyDescriptor prop in properties)
{
// Just include the values in not excluded properties
if (_excludedPropertyNames.All(epn => epn != prop.Name))
{
if (prop.PropertyType.Name == "DbGeography")
{
row[prop.Name] = SqlGeography.Parse(((DbGeography)prop.GetValue(item)).AsText()).MakeValid();
}
else
{
row[prop.Name] = prop.GetValue(item) ?? DBNull.Value;
}
}
}
table.Rows.Add(row);
}
return table;
}
#endregion
}
It's usage would be like:它的用法如下:
//1st.- You would have a colection of entities:
var myEntities = new List<MyEntity>();
// [...] With thousands or millions of items
// 2nd.- You would create the BulkInsert:
myEntityTypeBulk = new BulkInsert<MyEntity>(_connectionString, "MyEntitiesTableName", myEntities, new[] { "ObjectState", "SkippedEntityProperty1", "SkippedEntityProperty2" });
// 3rd.- You would execute it:
myEntityTypeBulk.Insert();
The performance gained and the reusability of this class were worth this message.获得的性能和此类的可重用性值得此消息。 I hope it helps:
我希望它有帮助:
Juan胡安
Alternatively you could convert your list to XML as well, as described in this blogpost: http://charleskong.com/blog/2009/09/insert-aspnet-objects-to-sql-server/ But the SqlBulkCopy approach seems better.或者,您也可以将列表转换为 XML,如本博文所述: http://charleskong.com/blog/2009/09/insert-aspnet-objects-to-sql-server/但 SqlBulkCopy 方法似乎更好。
Another remark: if you want to solve it through iterating over the elements in code, than it could possible improve performance if you do all the inserts in one single transaction.另一句话:如果您想通过迭代代码中的元素来解决它,那么如果您在一个事务中完成所有插入,则可能会提高性能。
Juan's solution worked perfect for me, so I've made a few more changes and now supports async and a few refactoring Juan 的解决方案非常适合我,所以我做了一些更改,现在支持异步和一些重构
public class BulkInsert<T> where T : class
{
private readonly string _connectionString;
private readonly string _tableName;
private readonly IEnumerable<string>? _excludedPropertyNames;
private readonly int _batchSize;
private readonly IEnumerable<T> _data;
private readonly DataTable _dataTable;
/// <summary>
/// Initializes a new instance of the <see cref="BulkInsert{T}"/> class.
/// </summary>
/// <param name="connectionString">The connection string.</param>
/// <param name="tableName">Name of the table.</param>
/// <param name="data">The data.</param>
/// <param name="excludedPropertyNames">The excluded property names.</param>
/// <param name="batchSize">Size of the batch.</param>
public BulkInsert(
string connectionString,
string tableName,
IEnumerable<T> data,
IEnumerable<string>? excludedPropertyNames,
int batchSize = 1000)
{
if (string.IsNullOrEmpty(connectionString)) throw new ArgumentNullException(nameof(connectionString));
if (string.IsNullOrEmpty(tableName)) throw new ArgumentNullException(nameof(tableName));
if (data == null) throw new ArgumentNullException(nameof(data));
if (batchSize <= 0) throw new ArgumentOutOfRangeException(nameof(batchSize));
_connectionString = connectionString;
_tableName = tableName;
_batchSize = batchSize;
_data = data;
_excludedPropertyNames = excludedPropertyNames ?? new List<string>();
_dataTable = CreateCustomDataTable();
}
/// <summary>
/// Inserts the data with a bulk copy inside a transaction.
/// </summary>
public async Task InsertAsync()
{
await using var connection = new SqlConnection(_connectionString);
connection.Open();
var transaction = connection.BeginTransaction();
using (var bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.Default | SqlBulkCopyOptions.KeepIdentity, transaction))
{
bulkCopy.BatchSize = _batchSize;
bulkCopy.DestinationTableName = _tableName;
// Let's fix tons of mapping issues by
// Setting the column mapping in SqlBulkCopy instance:
foreach (DataColumn dataColumn in _dataTable.Columns)
{
bulkCopy.ColumnMappings.Add(dataColumn.ColumnName, dataColumn.ColumnName);
}
try
{
await bulkCopy.WriteToServerAsync(_dataTable);
}
catch (Exception)
{
await transaction.RollbackAsync();
await connection.CloseAsync();
}
}
transaction.Commit();
}
/// <summary>
/// Creates the custom data table.
/// </summary>
private DataTable CreateCustomDataTable()
{
var properties = TypeDescriptor.GetProperties(typeof(T));
var table = new DataTable();
foreach (PropertyDescriptor prop in properties)
{
// Just include the not excluded columns
if (_excludedPropertyNames != null && _excludedPropertyNames.All(epn => epn != prop.Name))
{
table.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
}
}
foreach (T item in _data)
{
var row = table.NewRow();
foreach (PropertyDescriptor prop in properties)
{
// Just include the values in not excluded properties
if (_excludedPropertyNames != null && _excludedPropertyNames.All(epn => epn != prop.Name))
{
row[prop.Name] = prop.GetValue(item) ?? DBNull.Value;
}
}
table.Rows.Add(row);
}
return table;
}
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.