1.22-1.24 Oozie企业使用案例

一、将hive的表数据用sqoop抽取到mysql

1、编写oozie workflow和Coordinator

##
[root@hadoop-senior oozie-apps]# pwd
/opt/cdh-5.3.6/oozie-4.0.0-cdh5.3.6/oozie-apps

[root@hadoop-senior oozie-apps]# ls wf-user-select/
coordinator.xml  hive-site.xml  job.properties  lib  select-user.sql  workflow.xml

[root@hadoop-senior oozie-apps]# ls wf-user-select/lib/
mysql-connector-java-5.1.27-bin.jar


##job.properties    //start:2019-05-17   end:2019-05-18 ，此任务执行周期为1天
nameNode=hdfs://hadoop-senior.ibeifeng.com:8020
jobTracker=hadoop-senior.ibeifeng.com:8032
queueName=default
oozieAppsRoot=user/root/oozie-apps
oozieDataRoot=user/root/oozie/datas

oozie.use.system.libpath=true


oozie.coord.application.path=${nameNode}/${oozieAppsRoot}/wf-user-select
start=2019-05-17T13:45+0800
end=2019-05-18T14:00+0800
workflowAppUri=${nameNode}/${oozieAppsRoot}/wf-user-select

outputDir=wf-user-select/output


##workflow.xml
<workflow-app xmlns="uri:oozie:workflow:0.5" name="wf-user-select">
    <start to="hive-node"/>

    <action name="hive-node">
        <hive xmlns="uri:oozie:hive-action:0.5">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/${oozieDataRoot}/${outputDir}"/>
            </prepare>
            <job-xml>${nameNode}/${oozieAppsRoot}/hive-select/hive-site.xml</job-xml>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
            <script>select-user.sql</script>
            <param>OUTPUT=${nameNode}/${oozieDataRoot}/${outputDir}</param>
        </hive>
        <ok to="sqoop-node"/>
        <error to="fail"/>
    </action>

    <action name="sqoop-node">
        <sqoop xmlns="uri:oozie:sqoop-action:0.3">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
            <command>export --connect jdbc:mysql://hadoop-senior.ibeifeng.com:3306/test --username root --password 123456 --table my_user --num-mappers 1 --fields-terminated-by "	" --export-dir /user/root/oozie/datas/wf-user-select/output</command>
        </sqoop>
        <ok to="end"/>
        <error to="fail"/>
    </action>


##select-user.sql
--drop table tmp_user
drop table if exists default.tmp_user ;

--create table tmp_user
create table default.tmp_user like user_hive location '${OUTPUT}' ;

--load data to tmp_user
insert into table tmp_user
select
    id, account, password
from 
    user_hive 
where 
        id >= 15 
    and 
        account like 'beif%' ;


##coordinator.xml        //每天1点30开始执行，
<coordinator-app name="cron-coord" frequency="30 1 * * *"
                start="${start}" end="${end}" timezone="GMT+0800"
                xmlns="uri:oozie:coordinator:0.4">
        <action>
        <workflow>
            <app-path>${workflowAppUri}</app-path>
            <configuration>
                <property>
                    <name>jobTracker</name>
                    <value>${jobTracker}</value>
                </property>
                <property>
                    <name>nameNode</name>
                    <value>${nameNode}</value>
                </property>
                <property>
                    <name>queueName</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
        </workflow>
    </action>
</coordinator-app>


##hive-site.xml

2、执行

[root@hadoop-senior oozie-apps]# /opt/cdh-5.3.6/hadoop-2.5.0-cdh5.3.6/bin/hdfs dfs -put wf-user-select/ oozie-apps/


[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozie job -config oozie-apps/wf-user-select/job.properties -run