oracle11g程序调优纪实

一个控制台程序，主要业务就是读取某目录的zip压缩包文件，解压后遍历每个xml文件，解析文件内容然后存储到oracle11g(互联网主机)上。

程序第一版：

1)实现方式：使用动软的代码生成器里常用的DbHelperOra类改成了使用ODP.NET的DbHelper。

DbHelper类是个静态类，每个方法都单独new一个OracleConnection，每次Open了再马上Close并释放掉OracleConnection。

ps:使用这种类ado.net的手工写法是为了使用OracleBulkCopy实现批量插入数据库。解析一个xml后需要批量插入数据库的记录多则上万，少则数千，所以使用ORM估计效率低下，故选用此方案。

2)效果：

解析一个文件需要2-3分钟。插入数据库时经常出错： ORA-00054、ORA-00604等。多次调优，发现没有在使用OracleBulkCopy时使用事务。

/// <summary>
        /// 执行OracleBulkCopy(批量数据插入)
        /// </summary>
        /// <param name="tableName">Oracle里的表名</param>
        /// <param name="dt">临时表（包含数据)表结构要和数据库表一致</param>
        public void ExecuteOracleBulkCopy(string tableName, DataTable dt)
        {
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            using (OracleBulkCopy obc = new OracleBulkCopy(dbConn))
            {
                 try
                {
                    obc.BatchSize = dt.Rows.Count;
                    obc.BulkCopyTimeout = 1800;
                    obc.DestinationTableName = tableName;
                    obc.WriteToServer(dt);
                 }
                catch (OracleException ex)
                {
                    throw new Exception(ex.Message);
                }
                finally
                {
                    obc.Close();
                    dbConn.Close();
                }
            }
        }

程序第二版：

1)调优过程：

a、DbHelper类调优，一个应用程序只建立一个OracleConnection，实现IDisposable。

程序在使用多线程时出现较多的异常，所以考虑先写单线程业务实现了再调优。这里没有使用单例模式。

using System;
using System.Collections;
using System.Collections.Specialized;
using System.Data;
using System.Configuration;
using Oracle.DataAccess;
using Oracle.DataAccess.Client;

namespace ConsoleFacDataTrans
{
    /// <summary>
    /// Copyright (C) DataCool
    /// 数据访问基础类(基于Oracle ODP.NET)
    /// </summary>
    public class DbHelperOra : IDisposable
    {
        private OracleConnection dbConn = null;

        public DbHelperOra(string connStr)
        {
            dbConn = new OracleConnection(connStr);
        }

        #region 公用方法

        public int GetMaxID(string FieldName, string TableName)
        {
            string strsql = "select max(" + FieldName + ")+1 from " + TableName;
            object obj = GetSingle(strsql);
            if (obj == null)
            {
                return 1;
            }
            else
            {
                return int.Parse(obj.ToString());
            }
        }

        public bool Exists(string strSql)
        {
            object obj = GetSingle(strSql);
            int cmdresult;
            if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value)))
            {
                cmdresult = 0;
            }
            else
            {
                cmdresult = int.Parse(obj.ToString());
            }
            if (cmdresult == 0)
            {
                return false;
            }
            else
            {
                return true;
            }
        }

        public bool Exists(string strSql, params OracleParameter[] cmdParms)
        {
            object obj = GetSingle(strSql, cmdParms);
            int cmdresult;
            if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value)))
            {
                cmdresult = 0;
            }
            else
            {
                cmdresult = int.Parse(obj.ToString());
            }
            if (cmdresult == 0)
            {
                return false;
            }
            else
            {
                return true;
            }
        }

        #endregion

        #region  执行简单SQL语句

        /// <summary>
        /// 执行SQL语句，返回影响的记录数
        /// </summary>
        /// <param name="SQLString">SQL语句</param>
        /// <returns>影响的记录数</returns>
        public int ExecuteSql(string SQLString)
        {
            using (OracleCommand cmd = new OracleCommand(SQLString, dbConn))
            {
                try
                {
                    if (dbConn.State != ConnectionState.Open)
                        dbConn.Open();
                    int rows = cmd.ExecuteNonQuery();
                    return rows;
                }
                catch (OracleException ex)
                {
                    dbConn.Close();
                    throw new Exception(ex.Message);
                }
            }
        }

        /// <summary>
        /// 执行多条SQL语句，实现数据库事务。
        /// </summary>
        /// <param name="SQLStringList">多条SQL语句</param>        
        public void ExecuteSqlTran(ArrayList SQLStringList)
        {
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            OracleCommand cmd = dbConn.CreateCommand();
            OracleTransaction tx = dbConn.BeginTransaction();
            cmd.Transaction = tx;
            try
            {
                for (int n = 0; n < SQLStringList.Count; n++)
                {
                    string strsql = SQLStringList[n].ToString();
                    if (strsql.Trim().Length > 1)
                    {
                        cmd.CommandText = strsql;
                        cmd.ExecuteNonQuery();
                    }
                }
                tx.Commit();
            }
            catch (OracleException E)
            {
                tx.Rollback();
                throw new Exception(E.Message);
            }
        }

        /// <summary>
        /// 执行一条计算查询结果语句，返回查询结果（object）。
        /// </summary>
        /// <param name="SQLString">计算查询结果语句</param>
        /// <returns>查询结果（object）</returns>
        public object GetSingle(string SQLString)
        {
            using (OracleCommand cmd = new OracleCommand(SQLString, dbConn))
            {
                try
                {
                    if (dbConn.State != ConnectionState.Open)
                        dbConn.Open();
                    object obj = cmd.ExecuteScalar();
                    if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value)))
                    {
                        return null;
                    }
                    else
                    {
                        return obj;
                    }
                }
                catch (OracleException e)
                {
                    dbConn.Close();
                    throw new Exception(e.Message);
                }
            }
        }
        /// <summary>
        /// 执行查询语句，返回OracleDataReader ( 注意：调用该方法后，一定要对SqlDataReader进行Close )
        /// </summary>
        /// <param name="strSQL">查询语句</param>
        /// <returns>OracleDataReader</returns>
        public OracleDataReader ExecuteReader(string strSQL)
        {
            OracleCommand cmd = new OracleCommand(strSQL, dbConn);
            try
            {
                if (dbConn.State != ConnectionState.Open)
                    dbConn.Open();
                OracleDataReader myReader = cmd.ExecuteReader(CommandBehavior.CloseConnection);
                return myReader;
            }
            catch (OracleException e)
            {
                throw new Exception(e.Message);
            }
        }
        /// <summary>
        /// 执行查询语句，返回DataSet
        /// </summary>
        /// <param name="SQLString">查询语句</param>
        /// <returns>DataSet</returns>
        public DataSet Query(string SQLString)
        {
            DataSet ds = new DataSet();
            try
            {
                if (dbConn.State != ConnectionState.Open)
                    dbConn.Open();
                OracleDataAdapter command = new OracleDataAdapter(SQLString, dbConn);
                command.Fill(ds, "ds");
            }
            catch (OracleException ex)
            {
                throw new Exception(ex.Message);
            }
            return ds;
        }

        #endregion

        #region 执行带参数的SQL语句

        /// <summary>
        /// 执行SQL语句，返回影响的记录数
        /// </summary>
        /// <param name="SQLString">SQL语句</param>
        /// <returns>影响的记录数</returns>
        public int ExecuteSql(string SQLString, params OracleParameter[] cmdParms)
        {
            using (OracleCommand cmd = new OracleCommand())
            {
                try
                {
                    PrepareCommand(cmd, dbConn, null, SQLString, cmdParms);
                    int rows = cmd.ExecuteNonQuery();
                    cmd.Parameters.Clear();
                    return rows;
                }
                catch (OracleException E)
                {
                    throw new Exception(E.Message);
                }
            }
        }


        /// <summary>
        /// 执行多条SQL语句，实现数据库事务。
        /// </summary>
        /// <param name="SQLStringList">SQL语句的哈希表（key为sql语句，value是该语句的OracleParameter[]）</param>
        public void ExecuteSqlTran(Hashtable SQLStringList)
        {
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            using (OracleTransaction trans = dbConn.BeginTransaction())
            {
                OracleCommand cmd = new OracleCommand();
                try
                {
                    //循环
                    foreach (DictionaryEntry myDE in SQLStringList)
                    {
                        string cmdText = myDE.Key.ToString();
                        OracleParameter[] cmdParms = (OracleParameter[])myDE.Value;
                        PrepareCommand(cmd, dbConn, trans, cmdText, cmdParms);
                        int val = cmd.ExecuteNonQuery();
                        cmd.Parameters.Clear();
                        trans.Commit();
                    }
                }
                catch
                {
                    trans.Rollback();
                    throw;
                }
            }
        }


        /// <summary>
        /// 执行一条计算查询结果语句，返回查询结果（object）。
        /// </summary>
        /// <param name="SQLString">计算查询结果语句</param>
        /// <returns>查询结果（object）</returns>
        public object GetSingle(string SQLString, params OracleParameter[] cmdParms)
        {
            using (OracleCommand cmd = new OracleCommand())
            {
                try
                {
                    PrepareCommand(cmd, dbConn, null, SQLString, cmdParms);
                    object obj = cmd.ExecuteScalar();
                    cmd.Parameters.Clear();
                    if ((Object.Equals(obj, null)) || (Object.Equals(obj, System.DBNull.Value)))
                    {
                        return null;
                    }
                    else
                    {
                        return obj;
                    }
                }
                catch (OracleException e)
                {
                    throw new Exception(e.Message);
                }
            }
        }

        /// <summary>
        /// 执行查询语句，返回OracleDataReader ( 注意：调用该方法后，一定要对SqlDataReader进行Close )
        /// </summary>
        /// <param name="strSQL">查询语句</param>
        /// <returns>OracleDataReader</returns>
        public OracleDataReader ExecuteReader(string SQLString, params OracleParameter[] cmdParms)
        {
            OracleCommand cmd = new OracleCommand();
            try
            {
                PrepareCommand(cmd, dbConn, null, SQLString, cmdParms);
                OracleDataReader myReader = cmd.ExecuteReader(CommandBehavior.CloseConnection);
                cmd.Parameters.Clear();
                return myReader;
            }
            catch (OracleException e)
            {
                throw new Exception(e.Message);
            }
        }

        /// <summary>
        /// 执行查询语句，返回DataSet
        /// </summary>
        /// <param name="SQLString">查询语句</param>
        /// <returns>DataSet</returns>
        public DataSet Query(string SQLString, params OracleParameter[] cmdParms)
        {
            OracleCommand cmd = new OracleCommand();
            PrepareCommand(cmd, dbConn, null, SQLString, cmdParms);
            using (OracleDataAdapter da = new OracleDataAdapter(cmd))
            {
                DataSet ds = new DataSet();
                try
                {
                    da.Fill(ds, "ds");
                    cmd.Parameters.Clear();
                }
                catch (OracleException ex)
                {
                    throw new Exception(ex.Message);
                }
                return ds;
            }
        }

        private void PrepareCommand(OracleCommand cmd, OracleConnection conn, OracleTransaction trans, string cmdText, OracleParameter[] cmdParms)
        {
            if (conn.State != ConnectionState.Open)
                conn.Open();
            cmd.Connection = conn;
            cmd.CommandText = cmdText;
            if (trans != null)
                cmd.Transaction = trans;
            cmd.CommandType = CommandType.Text;//cmdType;
            if (cmdParms != null)
            {
                foreach (OracleParameter parm in cmdParms)
                    cmd.Parameters.Add(parm);
            }
        }

        #endregion

        #region 存储过程操作

        /// <summary>
        /// 执行存储过程 返回SqlDataReader ( 注意：调用该方法后，一定要对SqlDataReader进行Close )
        /// </summary>
        /// <param name="storedProcName">存储过程名</param>
        /// <param name="parameters">存储过程参数</param>
        /// <returns>OracleDataReader</returns>
        public OracleDataReader RunProcedure(string storedProcName, IDataParameter[] parameters)
        {
            OracleDataReader returnReader;
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            OracleCommand command = BuildQueryCommand(dbConn, storedProcName, parameters);
            command.CommandType = CommandType.StoredProcedure;
            returnReader = command.ExecuteReader(CommandBehavior.CloseConnection);
            return returnReader;
        }


        /// <summary>
        /// 执行存储过程
        /// </summary>
        /// <param name="storedProcName">存储过程名</param>
        /// <param name="parameters">存储过程参数</param>
        /// <param name="tableName">DataSet结果中的表名</param>
        /// <returns>DataSet</returns>
        public DataSet RunProcedure(string storedProcName, IDataParameter[] parameters, string tableName)
        {
            DataSet dataSet = new DataSet();
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            OracleDataAdapter sqlDA = new OracleDataAdapter();
            sqlDA.SelectCommand = BuildQueryCommand(dbConn, storedProcName, parameters);
            sqlDA.Fill(dataSet, tableName);
            dbConn.Close();
            return dataSet;
        }


        /// <summary>
        /// 构建 OracleCommand 对象(用来返回一个结果集，而不是一个整数值)
        /// </summary>
        /// <param name="connection">数据库连接</param>
        /// <param name="storedProcName">存储过程名</param>
        /// <param name="parameters">存储过程参数</param>
        /// <returns>OracleCommand</returns>
        private OracleCommand BuildQueryCommand(OracleConnection connection, string storedProcName, IDataParameter[] parameters)
        {
            OracleCommand command = new OracleCommand(storedProcName, connection);
            command.CommandType = CommandType.StoredProcedure;
            foreach (OracleParameter parameter in parameters)
            {
                command.Parameters.Add(parameter);
            }
            return command;
        }

        /// <summary>
        /// 执行存储过程，返回影响的行数        
        /// </summary>
        /// <param name="storedProcName">存储过程名</param>
        /// <param name="parameters">存储过程参数</param>
        /// <param name="rowsAffected">影响的行数</param>
        /// <returns></returns>
        public int RunProcedure(string storedProcName, IDataParameter[] parameters, out int rowsAffected)
        {
            int result;
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            OracleCommand command = BuildIntCommand(dbConn, storedProcName, parameters);
            rowsAffected = command.ExecuteNonQuery();
            result = (int)command.Parameters["ReturnValue"].Value;
            return result;
        }

        /// <summary>
        /// 创建 OracleCommand 对象实例(用来返回一个整数值)    
        /// </summary>
        /// <param name="storedProcName">存储过程名</param>
        /// <param name="parameters">存储过程参数</param>
        /// <returns>OracleCommand 对象实例</returns>
        private OracleCommand BuildIntCommand(OracleConnection connection, string storedProcName, IDataParameter[] parameters)
        {
            OracleCommand command = BuildQueryCommand(connection, storedProcName, parameters);
            command.Parameters.Add(new OracleParameter("ReturnValue",
                OracleDbType.Int32, 4, ParameterDirection.ReturnValue,
                false, 0, 0, string.Empty, DataRowVersion.Default, null));
            return command;
        }
        #endregion

        /// <summary>
        /// 执行OracleBulkCopy(批量数据插入)
        /// </summary>
        /// <param name="tableName">Oracle里的表名</param>
        /// <param name="dt">临时表（包含数据)表结构要和数据库表一致</param>
        public void ExecuteOracleBulkCopy(string tableName, DataTable dt)
        {
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            using (OracleBulkCopy obc = new OracleBulkCopy(dbConn))
            {
                OracleTransaction transaction = dbConn.BeginTransaction();
                try
                {
                    obc.BatchSize = dt.Rows.Count;
                    obc.BulkCopyTimeout = 1800;
                    obc.DestinationTableName = tableName;
                    obc.WriteToServer(dt);
                    transaction.Commit();        
                }
                catch (OracleException ex)
                {
                    transaction.Rollback();
                    throw new Exception(ex.Message);
                }
                finally
                {
                    obc.Close();
                    transaction.Dispose();
                    dbConn.Close();
                }
            }
        }

        /// <summary>
        /// 执行无参数的存储过程
        /// </summary>
        /// <param name="spName"></param>
        public void ExecuteStoreProcedure(string spName)
        {
            using (OracleCommand cmd = new OracleCommand(spName, dbConn))
            {
                cmd.CommandType = CommandType.StoredProcedure;
                try
                {
                    if (dbConn.State != ConnectionState.Open)
                        dbConn.Open();
                    cmd.ExecuteNonQuery();
                }
                catch (OracleException ex)
                {
                    dbConn.Close();
                    throw new Exception(ex.Message);
                }
            }
        }

        public void Dispose()
        {
            if (dbConn != null)
            {
                dbConn.Close();
                dbConn.Dispose();
            }
        }
    }
}

主要原因是每次调用一个方法都重新创建一个OracleConnection对象，每次都要Open、Close等效果当然不好了。这不是B/S程序，没必要。

b)在使用OracleBulkCopy时，使用事务！这是最关键的优化！

 /// <summary>
        /// 执行OracleBulkCopy(批量数据插入)
        /// </summary>
        /// <param name="tableName">Oracle里的表名</param>
        /// <param name="dt">临时表（包含数据)表结构要和数据库表一致</param>
        public void ExecuteOracleBulkCopy(string tableName, DataTable dt)
        {
            if (dbConn.State != ConnectionState.Open)
                dbConn.Open();
            using (OracleBulkCopy obc = new OracleBulkCopy(dbConn))
            {
                OracleTransaction transaction = dbConn.BeginTransaction();
                try
                {
                    obc.BatchSize = dt.Rows.Count;
                    obc.BulkCopyTimeout = 1800;
                    obc.DestinationTableName = tableName;
                    obc.WriteToServer(dt);
                    transaction.Commit();        
                }
                catch (OracleException ex)
                {
                    transaction.Rollback();
                    throw new Exception(ex.Message);
                }
                finally
                {
                    obc.Close();
                    transaction.Dispose();
                    dbConn.Close();
                }
            }
        }

 

2)程序效果：

解析一个xml文件并成功导入到数据库耗时仅需要1-10秒！秒哦，100倍的效率提升！

搞定收工，记录下来。以后备忘！