今天做了个试验, 试验使用DataTable对象的过滤性能与手工比较性能.
背景: 一个表中有80W条记录, 有两个字段field1, field2, 其中distinct field1有30W条记录,相当于第个field1对应有3条field2记录, 程序的目的是field1相同时合并field2, 即最终输出为:
field10: field20;field21;field22...
...
即最终输出30多W行, 第一列是field1,第二列是field1对应的不同field2值的合并串. 使用了下面两种方法,结果大为吃惊:
>ConsoleApplication1.exe
loop1:2008-12-18 17:46:08
begin:2008-12-18 17:46:44
00:00:00.4720000
Filter:2008-12-18 17:46:44
begin:2008-12-18 17:47:33
00:00:43.7490000
程序均未算上从DB取数据的时间,只算了处理数据的时间, 发现DataTable的filter性能差得实在太远,估计有60倍的差距. 当然上面的数据中field1的记录数也太多,过滤次数太多, 这也可能是个原因.
不过以后还是使用下面的方法, 简单易理解, 性能也好, 少用select()过滤了,除非数据量非常少.
Code
1class Program
2 {
3 static OdbcConnection con = new OdbcConnection("dsn=odbc_xxx;uid=XXX;pwd=XXX");
4
5 static void Main(string[] args)
6 {
7 con.Open();
8
9
10 Loop1();
11
12 Filter();
13
14 con.Close();
15 }
16
17 static void Loop1()
18 {
19
20 Console.WriteLine("loop1:" + DateTime.Now.ToString());
21
22 string sql = "select field1,field2 from dm_pmart.xdy_coach_eachone group by 1,2 order by 1";
23 OdbcCommand cmd = new OdbcCommand(sql, con);
24 OdbcDataAdapter da = new OdbcDataAdapter(cmd);
25 DataSet dsLoop = new DataSet();
26 da.Fill(dsLoop);
27
28 Console.WriteLine("begin:" + DateTime.Now.ToString());
29 DateTime dt = DateTime.Now;
30
31 using (StreamWriter sw = File.CreateText("loop1.txt"))
32 {
33 string str = "";
34 string lastCoach = "";
35 foreach (DataRow row in dsLoop.Tables[0].Rows)
36 {
37 if (row[0].ToString().Equals(lastCoach))
38 str += ";" + row[1].ToString();
39 else
40 {
41 if (str != "")
42 sw.WriteLine(lastCoach + ":" + str);
43
44 lastCoach = row[0].ToString();
45 str = row[1].ToString();
46 }
47 }
48 }
49
50 Console.WriteLine(DateTime.Now.Subtract(dt).ToString());
51 }
52
53 static void Filter()
54 {
55 Console.WriteLine("Filter:" + DateTime.Now.ToString());
56
57 string sql = "select field1 from dm_pmart.xdy_coach_eachone group by 1";
58 OdbcCommand cmd = new OdbcCommand(sql, con);
59 OdbcDataAdapter da = new OdbcDataAdapter(cmd);
60 DataSet dsCoach = new DataSet();
61 da.Fill(dsCoach);
62
63 sql = "select field1,field2 from dm_pmart.xdy_coach_eachone group by 1,2";
64 da.SelectCommand.CommandText = sql;
65 DataSet dsAll = new DataSet();
66 da.Fill(dsAll);
67
68 Console.WriteLine("begin:" + DateTime.Now.ToString());
69 DateTime dt = DateTime.Now;
70 using (StreamWriter sw = File.CreateText("filter.txt"))
71 {
72
73 foreach (DataRow row in dsCoach.Tables[0].Rows)
74 {
75 //DataView dv = new DataView(dsAll.Tables[0], "field1='" + row[0].ToString() + "'", "", DataViewRowState.CurrentRows);
76 string str = "";
77 foreach (DataRow dr in dsAll.Tables[0].Select("field1='" + row[0].ToString() + "'"))
78 {
79 str += ";" + dr[1].ToString();
80 }
81 sw.WriteLine(row[0].ToString() + ":" + str);
82 }
83 }
84
85 Console.WriteLine(DateTime.Now.Subtract(dt).ToString());
86
87 }
88 }