sql server sqlbulkcopy 批量数据插入数据库使用的是System.Data.SqlClient中的 SqlBulkCopy批量数据插入数据库
sql server 使用SqlBulkCopy批量插入数据库
SqlBulkCopy位于位于命名空间System.Data.SqlClient下，主要功能是把其他数据源(数据行DataRow，数据表DataTable，数据读取器IDataReader等)的数据有效批量的导入到SQL Server表中的功能。类似与 Microsoft SQL Server 包中名为 bcp 的命令行应用程序。但是使用 SqlBulkCopy 类可以编写托管代码解决方案，性能上优于bcp命令行应用程序，更优于如Insert方式向SQL Server数据库插入大量数据。因此SqlBulkCopy在应用到大批量数据的插入时很方便。
下面时具体案例(包括Entityframwork Code First 生成数据库，批量测试一万条数据插入数据库，读取excel文件导入数据库)
1、新建项目BatchImportStaffEmployee
2、主窗体form1改成FrmMain，
3、NuGet添加EntityFramework，NPOI。EF用于生成数据库，NPOI用于创建excel表格和读取。
4、添加User类和UserDBContext类
User类如下：
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace BatchImportStaffEmployee
{
public class User
{
public int Id { get; set; }
public string Name { get; set; }
public string PassWord { get; set; }
public string Address { get; set; }
public string Telephone { get; set; }
}
}
UserDBContext类如下：
using System;
using System.Collections.Generic;
using System.Data.Entity;
using System.Linq;
using System.Text;
namespace BatchImportStaffEmployee
{
public class UserDBContext : DbContext
{
//默认数据库
//public UserDBContext()
//{
// //如果是第一次使用EF Code First，没有进行任何数据库配置，那么默认是保存到“.SQLEXPRESS”数据库实例，如果未安装“.SQLEXPRESS”则默认使用LocalDb
//}
//指定数据库
public UserDBContext(string connectString) : base(connectString)
{
//如果需要指定的服务器上创建数据库，管理数据库的话，如本机的数据库实例，MyStudent数据库，那么这时需要配置App.Config中配置一个数据库连接串，然后在我们的数据库上下文中指定这个连接名称。
//< connectionStrings >
//< add name = "MyUserDB" connectionString = "Data Source=127.0.0.1;Database=MyUserDB;User ID=sa;PWD=123;" providerName = "System.Data.SqlClient" ></ add >
//</ connectionStrings >
}
public DbSet<User> Users { get; set; }
}
}
5、添加文本标签和按钮，如下
6、配置App.config
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
</configSections>
<connectionStrings>
<!--配置数据库-->
<add name="MyUserDB" connectionString="Data Source=127.0.0.1;Database=MyUserDB;User ID=sa;PWD=123;" providerName="System.Data.SqlClient"></add>
</connectionStrings>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.LocalDbConnectionFactory, EntityFramework">
<parameters>
<parameter value="mssqllocaldb" />
</parameters>
</defaultConnectionFactory>
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
</providers>
</entityFramework>
</configuration>
7、添加主窗体FrmMain对应事件以及方法，代码如下
using NPOI.HSSF.UserModel;
using NPOI.SS.UserModel;
using NPOI.XSSF.UserModel;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Configuration;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Windows.Forms;
namespace BatchImportStaffEmployee
{
public partial class FrmMain : Form
{
public FrmMain()
{
InitializeComponent();
}
#region Method
/// <summary>
/// 生成一万条测试数据
/// </summary>
/// <returns></returns>
public DataTable GetDataTableData()
{
List<User> userList = new List<User>();
for (int i = 0; i < 10000; i++)
{
User user = new User();
user.Name = "张三" + i;
user.PassWord = i.ToString();
user.Address = $"上海路{i}号";
user.Telephone = "18765443310";
userList.Add(user);
}
using (DataTable dt = new DataTable())
{
dt.Columns.Add("Name", typeof(string));
dt.Columns.Add("PassWord", typeof(string));
dt.Columns.Add("Address", typeof(string));
dt.Columns.Add("Telephone", typeof(string));
foreach (User user in userList)
{
DataRow dr = dt.NewRow();
dr["Name"] = user.Name;
dr["PassWord"] = user.PassWord;
dr["Address"] = user.Address;
dr["Telephone"] = user.Telephone;
dt.Rows.Add(dr);
}
return dt;
}
}
/// <summary>
/// 从文件中读取数据，并生成数据表DataTable
/// </summary>
/// <param name="fileName">文件路径</param>
/// <returns>数据表DataTable</returns>
public DataTable FillDataTable(string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
return null;
}
try
{
DataTable table = new DataTable();
table.Columns.Add("UserID");//不为空
table.Columns.Add("PlaceID");//不为空
table.Columns.Add("UserNO");//不为空
table.Columns.Add("UserName");
table.Columns.Add("Password");//不为空
table.Columns.Add("UserType");
table.Columns.Add("Level");
table.Columns.Add("LevelCalcTime");
table.Columns.Add("UpdateTime");
table.Columns.Add("ManualLevel");
table.Columns.Add("EnableManualLevel");
table.Columns.Add("UserPic");
table.Columns.Add("UserMotto");
table.Columns.Add("UserCard");
table.Columns.Add("UserCardLocation");
table.Columns.Add("Position");
table.Columns.Add("WindowNo");
table.Columns.Add("WindowName");
using (StreamReader sr = new StreamReader(fileName, Encoding.Default))
{
while (!sr.EndOfStream)
{
DataRow dr = table.NewRow();
string readStr = sr.ReadLine();
if (readStr.StartsWith("1"))
{
string[] strs = readStr.Split(new char[] { ' ', '"' }, StringSplitOptions.RemoveEmptyEntries);
string startNum = strs[0];
string numArea = strs[1];
string numType = strs[2];
//往对应的 行中添加数据
dr["UserID"] = numType;//不为空
dr["PlaceID"] = numType;//不为空
dr["UserNO"] = numType;//不为空
dr["UserName"] = numType;
dr["Password"] = numType;//不为空
dr["UserType"] = numType;
dr["Level"] = numType;
dr["LevelCalcTime"] = numType;
dr["UpdateTime"] = numType;//不为空
dr["ManualLevel"] = numType;//不为空
dr["EnableManualLevel"] = numType;//不为空
dr["UserPic"] = numType;
dr["UserMotto"] = numType;
dr["UserCard"] = numType;
dr["UserCardLocation"] = numType;
dr["Position"] = numType;
dr["WindowNo"] = numType;
dr["WindowName"] = numType;
table.Rows.Add(dr);//将创建的数据行添加到table中
}
}
}
return table;
}
catch (Exception ex)
{
MessageBox.Show($"FillDataTable失败，原因：{ex.ToString()}", "提示：");
throw;
}
}
/// <summary>
/// 将数据源导入数据库中
/// </summary>
/// <param name="dataTable">数据源</param>
/// <returns>是否导入成功</returns>
public bool ImportDB(DataTable dataTable)
{
bool isOK = false;
string conStr = ConfigurationManager.ConnectionStrings["MyUserDB"].ConnectionString;
SqlConnection connection = new SqlConnection(conStr);
try
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conStr, SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.UseInternalTransaction))
{
bulkCopy.DestinationTableName = "[MyUserDB].[dbo].[Users]";//目标表，即要将数据插入到哪张表中去
bulkCopy.ColumnMappings.Add("Name", "Name");//数据源中的列名与目标表的属性的映射关系
bulkCopy.ColumnMappings.Add("PassWord", "PassWord");
bulkCopy.ColumnMappings.Add("Address", "Address");
bulkCopy.ColumnMappings.Add("Telephone", "Telephone");
//DataTable dt = GetDataTableData();//数据源数据
DataTable dt = dataTable;
Stopwatch stopwatch = new Stopwatch();//秒表进行时间的统计
stopwatch.Start();
bulkCopy.WriteToServer(dt);//将数据源数据写入到数据库中
this.rtb.Text = "插入数据所用时间:" + stopwatch.Elapsed.ToString();
isOK = true;
}
}
catch (Exception ex)
{
this.rtb.Text = $"插入数据所用时间:{ex.Message}";
isOK = false;
}
return isOK;
}
/// <summary>
/// 将数据源导入数据库中
/// </summary>
/// <param name="dataTable">数据源</param>
/// <returns>是否导入成功</returns>
public bool ImportDatabase(DataTable dataTable)
{
bool isOK = false;
string conStr = ConfigurationManager.ConnectionStrings["MyUserDB"].ConnectionString;
try
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conStr, SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.UseInternalTransaction))
{
//目标表，即要将数据插入到哪张表中去
bulkCopy.DestinationTableName = "[MyUserDB].[dbo].[UserInfo]";
#region 数据源中的列名与目标表的属性的映射关系
bulkCopy.ColumnMappings.Add("UserID", "UserID");
bulkCopy.ColumnMappings.Add("PlaceID", "PlaceID");
bulkCopy.ColumnMappings.Add("UserNO", "UserNO");
bulkCopy.ColumnMappings.Add("UserName", "UserName");
bulkCopy.ColumnMappings.Add("Password", "Password");
bulkCopy.ColumnMappings.Add("UserType", "UserType");
bulkCopy.ColumnMappings.Add("Level", "Level");
bulkCopy.ColumnMappings.Add("LevelCalcTime", "LevelCalcTime");
bulkCopy.ColumnMappings.Add("UpdateTime", "UpdateTime");
bulkCopy.ColumnMappings.Add("ManualLevel", "ManualLevel");
bulkCopy.ColumnMappings.Add("EnableManualLevel", "EnableManualLevel");
bulkCopy.ColumnMappings.Add("UserPic", "UserPic");
bulkCopy.ColumnMappings.Add("UserMotto", "UserMotto");
bulkCopy.ColumnMappings.Add("UserCard", "UserCard");
bulkCopy.ColumnMappings.Add("UserCardLocation", "UserCardLocation");
bulkCopy.ColumnMappings.Add("Position", "Position");
bulkCopy.ColumnMappings.Add("WindowNo", "WindowNo");
bulkCopy.ColumnMappings.Add("WindowName", "WindowName");
#endregion
//数据源数据
//DataTable dt = GetDataTableData();
DataTable dt = dataTable;
//秒表进行时间的统计
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//将数据源数据写入到数据库中
bulkCopy.WriteToServer(dt);
this.rtb.Text = "将数据源导入数据库中成功，插入数据所用时间:" + stopwatch.Elapsed.ToString();
isOK = true;
}
}
catch (Exception ex)
{
this.rtb.Text = $"将数据源导入数据库中失败，原因:{ex.Message}";
isOK = false;
}
return isOK;
}
/// <summary>
/// 将数据源导入数据库中
/// </summary>
/// <param name="dataTableReader">数据源</param>
/// <returns>是否导入成功</returns>
public bool ImportDatabae(DataTableReader dataTableReader)
{
bool isOK = false;
string conStr = ConfigurationManager.ConnectionStrings["MyUserDB"].ConnectionString;
try
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conStr, SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.UseInternalTransaction))
{
//目标表，即要将数据插入到哪张表中去
bulkCopy.DestinationTableName = "[MyUserDB].[dbo].[EvMS]";
#region 数据源中的列名与目标表的属性的映射关系
bulkCopy.ColumnMappings.Add("UserID", "UserID");
bulkCopy.ColumnMappings.Add("PlaceID", "PlaceID");
bulkCopy.ColumnMappings.Add("UserNO", "UserNO");
bulkCopy.ColumnMappings.Add("UserName", "UserName");
bulkCopy.ColumnMappings.Add("Password", "Password");
bulkCopy.ColumnMappings.Add("UserType", "UserType");
bulkCopy.ColumnMappings.Add("Level", "Level");
bulkCopy.ColumnMappings.Add("LevelCalcTime", "LevelCalcTime");
bulkCopy.ColumnMappings.Add("UpdateTime", "UpdateTime");
bulkCopy.ColumnMappings.Add("ManualLevel", "ManualLevel");
bulkCopy.ColumnMappings.Add("EnableManualLevel", "EnableManualLevel");
bulkCopy.ColumnMappings.Add("UserPic", "UserPic");
bulkCopy.ColumnMappings.Add("UserMotto", "UserMotto");
bulkCopy.ColumnMappings.Add("UserCard", "UserCard");
bulkCopy.ColumnMappings.Add("UserCardLocation", "UserCardLocation");
bulkCopy.ColumnMappings.Add("Position", "Position");
bulkCopy.ColumnMappings.Add("WindowNo", "WindowNo");
bulkCopy.ColumnMappings.Add("WindowName", "WindowName");
#endregion
//数据源数据
//DataTable dt = GetDataTableData();
//秒表进行时间的统计
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//将数据源数据写入到数据库中
bulkCopy.WriteToServer(dataTableReader);
this.rtb.Text = "插入数据所用时间:" + stopwatch.Elapsed.ToString();
isOK = true;
}
}
catch (Exception ex)
{
this.rtb.Text = $"插入数据所用时间:{ex.Message}";
isOK = false;
}
return isOK;
}
/// <summary>
/// 将DataTable数据导入到excel中
/// </summary>
/// <param name="data">数据源</param>
/// <param name="fileName">保存excel文件路径，包括名称</param>
/// <returns></returns>
public bool DataTableToExcel(DataTable data, string fileName)
{
if (string.IsNullOrEmpty(fileName))
{
return false;
}
var fs = new FileStream(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
// .xlsx 是2007;.xls是 97-2003
IWorkbook workbook = new HSSFWorkbook();
try
{
ISheet sheet = workbook.CreateSheet(data.TableName);
int rowNO = 0;
//第一行创建 DataTable的列名
IRow row = sheet.CreateRow(0);
for (int i = 0; i < data.Columns.Count; ++i)
{
row.CreateCell(i).SetCellValue(data.Columns[i].ColumnName);
}
rowNO = 1;
for (int j = 0; j < data.Rows.Count; ++j)
{
IRow creatrow = sheet.CreateRow(rowNO);
for (int k = 0; k < data.Columns.Count; ++k)
{
creatrow.CreateCell(k).SetCellValue(data.Rows[j][k].ToString());
}
rowNO += 1; ;
}
workbook.Write(fs);
return true;
}
catch (Exception)
{
return false;
}
}
/// <summary>
/// 将excel中的数据导入到DataTable中
/// </summary>
/// <param name="sheetName">excel工作薄sheet的名称</param>
/// <param name="firstRowIsColumn">第一行是否是DataTable的列名</param>
/// <returns>返回的DataTable</returns>
public DataTable ExcelToDataTable(string excelFileName, bool firstRowIsColumn)
{
DataTable data = new DataTable();
int startRow = 0;
try
{
var fs = new FileStream(excelFileName, FileMode.Open, FileAccess.Read);
IWorkbook workbook = new HSSFWorkbook(fs);
ISheet sheet = workbook.GetSheetAt(0);
if (sheet != null)
{
IRow firstRow = sheet.GetRow(0);
int cellCount = firstRow.LastCellNum;
//第一行是否是DataTable的列名
if (firstRowIsColumn)
{
for (int i = firstRow.FirstCellNum; i < cellCount; ++i)
{
ICell cell = firstRow.GetCell(i);
if (cell != null)
{
string cellValue = cell.StringCellValue;
if (cellValue != null)
{
DataColumn column = new DataColumn(cellValue);
data.Columns.Add(column);
}
}
}
startRow = sheet.FirstRowNum + 1;
}
else
{
startRow = sheet.FirstRowNum;
}
//最后一列的标号
int rowCount = sheet.LastRowNum;
for (int i = startRow; i <= rowCount; ++i)
{
IRow row = sheet.GetRow(i);
if (row == null) continue; //没有数据的行默认是null
DataRow dataRow = data.NewRow();
for (int j = row.FirstCellNum; j < cellCount; ++j)
{
if (row.GetCell(j) != null)
{
dataRow[j] = row.GetCell(j).ToString();
}
else
{
dataRow[j] = null;
}
}
data.Rows.Add(dataRow);
}
}
return data;
}
catch (Exception ex)
{
MessageBox.Show($"Exception:{ex.Message}", "提示：");
return null;
}
}
#endregion
#region Event
private void BtnSelectFile_Click(object sender, EventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.Filter = "表格文档|*.xls|文本文档|*.txt|所有文件|*.*";
if (ofd.ShowDialog() != DialogResult.OK)
{
return;
}
this.tbxSelectFile.Text = ofd.FileName;
}
//从其他数据源读取数据导入到数据库
private void BtnStartImport_Click(object sender, EventArgs e)
{
if (string.IsNullOrEmpty(this.tbxSelectFile.Text.Trim()))
{
MessageBox.Show("数据源文件选择为空，请重新选择数据源文件。", "提示：");
this.tbxSelectFile.Focus();
return;
}
try
{
//DataTable table = FillDataTable(this.tbxSelectFile.Text);
DataTable table = ExcelToDataTable(this.tbxSelectFile.Text, true);
if (table != null)
{
bool isok = ImportDatabase(table);
if (isok)
{
MessageBox.Show($"批量导入{table.Rows.Count}条数据，导入成功！", "提示：");
}
}
else
{
MessageBox.Show($"批量导入数据失败，原因FillDataTable创建的表为空。", "提示：");
}
}
catch (Exception ex)
{
MessageBox.Show($"批量导入数据失败，原因：{ex.ToString()}。", "提示：");
}
#region MyRegion
//string conStr = ConfigurationManager.ConnectionStrings["MyUserDB"].ConnectionString;
//SqlConnection connection = new SqlConnection(conStr);
//try
//{
// using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conStr, SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.UseInternalTransaction))
// {
// bulkCopy.DestinationTableName = "[MyUserDB].[dbo].[Users]";//目标表，就是说您将要将数据插入到哪个表中去
// bulkCopy.ColumnMappings.Add("Name", "Name");//数据源中的列名与目标表的属性的映射关系
// bulkCopy.ColumnMappings.Add("PassWord", "PassWord");
// bulkCopy.ColumnMappings.Add("Address", "Address");
// bulkCopy.ColumnMappings.Add("Telephone", "Telephone");
// DataTable dt = GetDataTableData();//数据源数据
// //bulkCopy.BatchSize = 3;
// Stopwatch stopwatch = new Stopwatch();//秒表，该类可以进行时间的统计
// stopwatch.Start();//秒表开始
// bulkCopy.WriteToServer(dt);//将数据源数据写入到目标表中
// this.rtb.Text = "插入数据所用时间:" + stopwatch.Elapsed.ToString();
// }
//}
//catch (Exception ex)
//{
// this.rtb.Text = $"插入数据所用时间:{ex.Message}";
//}
////this.rtb.Text = "插入数据插入成功";
#endregion
}
//测试生成数据库
private void BtnCodeFirst_Click(object sender, EventArgs e)
{
try
{
User user = new User() { Name = "123", PassWord = "123678", Address = "长白山62号", Telephone = "52112314" };
UserDBContext userDBContext = new UserDBContext("MyUserDB");
userDBContext.Users.Add(user);
userDBContext.SaveChanges();
StringBuilder stringBuilder = new StringBuilder();
foreach (var item in userDBContext.Users)
{
stringBuilder.AppendLine($"用户姓名：{item.Name}密码：{item.PassWord}，地址：{item.Address}，电话：{item.Telephone}");
}
this.rtb.Text = stringBuilder.ToString();
MessageBox.Show(this.rtb.Text, "提示：");
}
catch (Exception ex)
{
MessageBox.Show($"错误信息：{ex}", "提示：");
}
}
//测试一万条数据导入数据库
private void BtnTestImportDatabase_Click(object sender, EventArgs e)
{
string conStr = ConfigurationManager.ConnectionStrings["MyUserDB"].ConnectionString;
SqlConnection connection = new SqlConnection(conStr);
try
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conStr, SqlBulkCopyOptions.KeepIdentity | SqlBulkCopyOptions.UseInternalTransaction))
{
bulkCopy.DestinationTableName = "dbo.[Users]";//目标表，就是说您将要将数据插入到哪个表中去
bulkCopy.ColumnMappings.Add("Name", "Name");//数据源中的列名与目标表的属性的映射关系
bulkCopy.ColumnMappings.Add("PassWord", "PassWord");
bulkCopy.ColumnMappings.Add("Address", "Address");
bulkCopy.ColumnMappings.Add("Telephone", "Telephone");
DataTable dt = GetDataTableData();//数据源数据 测试一万条数据导入数据库
//bulkCopy.BatchSize = 3;
Stopwatch stopwatch = new Stopwatch();//秒表，该类可以进行时间的统计
stopwatch.Start();//秒表开始
bulkCopy.WriteToServer(dt);//将数据源数据写入到目标表中
string outString = "插入数据插入成功,插入数据所用时间:" + stopwatch.Elapsed;
this.rtb.Text = outString;
MessageBox.Show(outString, "提示：");
}
}
catch (Exception ex)
{
this.rtb.Text = $"插入数据所用时间:{ex.Message}";
MessageBox.Show(this.rtb.Text, "提示：");
}
}
#endregion
}
}
8、运行项目
9、注意项
a、excel表格文件的内容如下
b、sqlbulkcopy中添加映射的顺序必须和数据库中的列顺序一致，也就时说，sqlbulkcopy的数据源的列顺序，sqlbulkcopy映射顺序和数据库的列顺序必须一致
c、App.config文件中数据连接字符串的配置
