zoukankan      html  css  js  c++  java
  • MetaData ETL 之配置

     MetaData ETL

    最近整合ETL MetaData

    大致分为4步

    1 config 整合能根据配置信息来访问不同源和目标。方便维护时迁移和临时调整

    2  业务逻辑整合所有的业务指标和逻辑在动态配置表里面实现。

    3 column mapping 的实现 ETL 时column mapping是个头疼的事情。有新增加字段时每次都要人工去刷新mapping。能动态的根据源头来实现mapping

    4 具有一定排错,对于常见错误能有一定的排错。比如snapshot延时或者创建失败时能根据错误信息做出相对措施。保证ETL的稳定。

    1 先把配置自动化掉 用script 组件实现

    代码
    /*
    Microsoft SQL Server Integration Services Script Task
    Write scripts using Microsoft Visual C# 2008.
    The ScriptMain is the entry point class of the script.
    */

    using System;
    using System.Data;
    using Microsoft.SqlServer.Dts.Runtime;
    using System.Windows.Forms;
    using System.Data.SqlClient;


    namespace ST_f2fae4c9cab446c588a765c3131803fe.csproj
    {
    [System.AddIn.AddIn(
    "ScriptMain", Version = "1.0", Publisher = "", Description = "")]
    public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
    {

    #region VSTA generated code
    enum ScriptResults
    {
    Success
    = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
    Failure
    = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
    };
    #endregion

    /*
    The execution engine calls this method when the task executes.
    To access the object model, use the Dts property. Connections, variables, events,
    and logging features are available as members of the Dts property as shown in the following examples.

    To reference a variable, call Dts.Variables["MyCaseSensitiveVariableName"].Value;
    To post a log entry, call Dts.Log("This is my log text", 999, null);
    To fire an event, call Dts.Events.FireInformation(99, "test", "hit the help message", "", 0, true);

    To use the connections collection use something like the following:
    ConnectionManager cm = Dts.Connections.Add("OLEDB");
    cm.ConnectionString = "Data Source=localhost;Initial Catalog=AdventureWorks;Provider=SQLNCLI10;Integrated Security=SSPI;Auto Translate=False;";

    Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.

    To open Help, press F1.
    */

    public void Main()
    {
    // TODO: Add your code here
    #region MyRegion
    //Dim sqlReader As SqlDataReader

    //Dim sqlConn As New SqlClient.SqlConnection("Data Source=192.168.1.214;Initial Catalog=ETLMetaData;User ID=ETLBI_User;Password=123!@#abc")

    //Dim cmd As New SqlCommand("SELECT Status,Db from dbo.etl_configure where id=143 ", sqlConn)

    //sqlConn.Open()

    //sqlReader = cmd.ExecuteReader()


    //Do While sqlReader.Read
    // Dts.Variables("Status").Value = sqlReader.Item("Status")
    // Dts.Variables("DB").Value = sqlReader.Item("DB")
    //Loop

    //sqlConn.Close()

    //MsgBox(Dts.Variables("Status").Value)

    //MsgBox(Dts.Variables("DB").Value)
    #endregion

    // system packagename variabales
    string PackageName = Dts.Variables["PackageName"].Value.ToString();
    //
    string OnlineServerIP="";
    string OnlineMainDB="";
    string BackServerIp="";
    string BackDB="";
    short Isenable;


    //MessageBox.Show(Dts.Variables["PackageName"].Value.ToString());
    //try
    //{
    System.Data.SqlClient.SqlConnection conn
    = new System.Data.SqlClient.SqlConnection
    (
    "Data Source=192.168.1.214;Initial Catalog=ETLMetaData;User ID=ETLBI_User;Password=123!@#abc");

    System.Data.SqlClient.SqlCommand cmd
    = new System.Data.SqlClient.SqlCommand(
    (
    "SELECT Status,OnlineServerIP,OnlineMainDB,Tabname,BackServerIp,BackDB,Condition,Isenable,TargetServerIp,TargetDB,TargetTabname from dbo.ETL_MetaData where PackageName='"
    + PackageName + "'" ), conn);

    conn.Open();

    System.Data.SqlClient.SqlDataReader reader
    = cmd.ExecuteReader();


    while (reader.Read())
    {

    // get variables
    Dts.Variables["Status"].Value = reader.GetInt32(0);
    OnlineServerIP
    = reader.GetString(1);
    OnlineMainDB
    = reader.GetString(2);
    Dts.Variables[
    "Tabname"].Value = reader.GetString(3);
    BackServerIp
    = reader.GetString(4);
    BackDB
    = reader.GetString(5);
    Dts.Variables[
    "Condition"].Value = reader.GetString(6);
    Isenable
    = reader.GetInt16(7);
    Dts.Variables[
    "TargetServerIp"].Value = reader.GetString(8);
    Dts.Variables[
    "TargetDB"].Value = reader.GetString(9);
    Dts.Variables[
    "TargetTabname"].Value = reader.GetString(10);

    }

    conn.Close();
    cmd.Dispose();
    // status =2
    if (int.Parse(Dts.Variables["Status"].Value.ToString()) == 2)
    {
    Dts.Variables[
    "ServerIp"].Value = OnlineServerIP;
    Dts.Variables[
    "DB"].Value = OnlineMainDB;
    }

    // status =1
    if (int.Parse(Dts.Variables["Status"].Value.ToString()) == 1)
    {
    Dts.Variables[
    "ServerIp"].Value = BackServerIp;
    Dts.Variables[
    "DB"].Value = BackDB;
    }

    //MessageBox.Show(Dts.Variables["Status"].Value.ToString());
    //MessageBox.Show(Dts.Variables["DB"].Value.ToString());


    //}
    //catch (Exception ex)
    //{
    // Dts.TaskResult = (int)ScriptResults.Failure;

    // throw;
    //}

    Dts.TaskResult
    = (int)ScriptResults.Success;
    }
    }
    }
    练一技,修百艺,而成于自然.
  • 相关阅读:
    [C/C++] 显示各种C/C++编译器的预定义宏(C11标准、C++11标准、VC、BCB、Intel、GCC)
    [使用心得] 利用按键精灵批量删除pdf中的水印 V2:用于页面内对象数量不定时删除最后一个对象
    [VBScript] allfiles.vbs: 显示子目录下的所有文件的修改时间、大小、全限定名等信息
    [C] wchar_t的格式控制字符(VC、BCB、GCC、C99标准)
    Instructions函数对照表:01 mmintrin.h与MMX指令集
    [C/C++] VC2012编译的程序在WinXP下报告“指定的可执行文件不是有效的 Win32 应用程序”错误
    [C++] cout、wcout无法正常输出中文字符问题的深入调查(2):VC2005的crt源码分析
    [C/C++] ccpuid:CPUID信息模块 V1.02版,支持Mac OS X,支持纯C,增加CPUF常数
    [C] zintrin.h: 智能引入intrinsic函数 V1.01版。改进对Mac OS X的支持,增加INTRIN_WORDSIZE宏
    IDE常用快捷键
  • 原文地址:https://www.cnblogs.com/xwj1985/p/1904738.html
Copyright © 2011-2022 走看看