1.18-1.21 Oozie Coordinator调度

一、时区问题

1、修改系统时区

##
[root@hadoop-senior hadoop-2.5.0-cdh5.3.6]# rm -rf /etc/localtime 

[root@hadoop-senior hadoop-2.5.0-cdh5.3.6]# ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime


##修改系统时间

2、oozie时区

oozie默认使用UTC(GMT)时区，而服务器上可能是CST，建议统一使用GMT+0800


在oozie-site.xml中添加：
    <property>
        <name>oozie.processing.timezone</name>
        <value>GMT+0800</value>
    </property>


##restart oozie
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozied.sh stop

##清缓存
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# cd oozie-server/

[root@hadoop-senior oozie-server]# rm -rf work/Catalina/
   
[root@hadoop-senior oozie-server]# rm -rf conf/Catalina/

[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozied.sh start



##
可以用oozie info--timezones来查看支持的时区；

使用GMT+0800后，时间不可以再使用形如2014-01-24T13：402的格式，要使用对应的形如2014-01-24T13：40+0860的格式；

还有一点比较重要，即oozie web console的TimeZone设置要和上述一数，否则你在web console中看到的时间在感官上都是不正确的；

$OOZIE_HOME/oozie-server/webapps/oozie/oozie-console.js    //修改此文件，大概在170多行，如下：
function getTimeZone() {
    Ext.state.Manager.setProvider(new Ext.state.CookieProvider());
    return Ext.state.Manager.get("TimezoneId","GMT+0800");
}

改完以后，清一下浏览器的我缓存；

二、Coordinator案例1

1、准备文件

##
[root@hadoop-senior oozie-apps]# pwd
/opt/cdh-5.3.6/oozie-4.0.0-cdh5.3.6/oozie-apps

[root@hadoop-senior oozie-apps]# mkdir cron-schedule

[root@hadoop-senior oozie-apps]# ls cron-schedule/
coordinator.xml  job.properties  workflow.xml



##job.properties
nameNode=hdfs://hadoop-senior.ibeifeng.com:8020
jobTracker=hadoop-senior.ibeifeng.com:8032
queueName=default
oozieAppsRoot=user/root/oozie-apps
oozieDataRoot=user/root/oozie/datas


oozie.coord.application.path=${nameNode}/${oozieAppsRoot}/cron-schedule
start=2019-05-15T11:42+0800
end=2019-05-16T11:46+0800
workflowAppUri=${nameNode}/${oozieAppsRoot}/cron-schedule


start：开始时间
end：结束时间


##workflow.xml
<workflow-app xmlns="uri:oozie:workflow:0.5" name="no-op-wf">
    <start to="end"/>
    <end name="end"/>
</workflow-app>


##coordinator.xml
<coordinator-app name="cron-coord" frequency="${coord:minutes(1)}"
                start="${start}" end="${end}" timezone="GMT+0800"
                xmlns="uri:oozie:coordinator:0.4">
        <action>
        <workflow>
            <app-path>${workflowAppUri}</app-path>
            <configuration>
                <property>
                    <name>jobTracker</name>
                    <value>${jobTracker}</value>
                </property>
                <property>
                    <name>nameNode</name>
                    <value>${nameNode}</value>
                </property>
                <property>
                    <name>queueName</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
        </workflow>
    </action>
</coordinator-app>


frequency="${coord:minutes(1)}"    //每分钟一次

2、运行

##
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# /opt/cdh-5.3.6/hadoop-2.5.0-cdh5.3.6/bin/hdfs dfs -put oozie-apps/cron-schedule/ oozie-apps/


##
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozie job -config oozie-apps/cron-schedule/job.properties -run

三、Coordinator案例2

1、准备文件

##
[root@hadoop-senior oozie-apps]# pwd
/opt/cdh-5.3.6/oozie-4.0.0-cdh5.3.6/oozie-apps


[root@hadoop-senior oozie-apps]# mkdir cron

[root@hadoop-senior oozie-apps]# ls cron
coordinator.xml  job.properties  lib  workflow.xml

[root@hadoop-senior oozie-apps]# ls cron/lib/
mr-wordcount.jar


##
nameNode=hdfs://hadoop-senior.ibeifeng.com:8020
jobTracker=hadoop-senior.ibeifeng.com:8032
queueName=default
oozieAppsRoot=user/root/oozie-apps
oozieDataRoot=user/root/oozie/datas

oozie.coord.application.path=${nameNode}/${oozieAppsRoot}/cron
start=2019-05-16T14:16+0800
end=2019-05-16T14:20+0800
workflowAppUri=${nameNode}/${oozieAppsRoot}/cron

inputDir=mr-wordcount-wf/input
outputDir=mr-wordcount-wf/output


##
<workflow-app xmlns="uri:oozie:workflow:0.5" name="mr-wordcount-wf">
    <start to="mr-node-wordcount"/>
    <action name="mr-node-wordcount">
        <map-reduce>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/${oozieDataRoot}/${outputDir}"/>
            </prepare>
            <configuration>
                <property>
                    <name>mapred.mapper.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapred.reducer.new-api</name>
                    <value>true</value>
                </property>
                <property>
                    <name>mapreduce.job.queuename</name>
                    <value>${queueName}</value>
                </property>
                <property>
                    <name>mapreduce.job.map.class</name>
                    <value>com.ibeifeng.hadoop.senior.mapreduce.WordCount$WordCountMapper</value>
                </property>
                <property>
                    <name>mapreduce.job.reduce.class</name>
                    <value>com.ibeifeng.hadoop.senior.mapreduce.WordCount$WordCountReducer</value>
                </property>
                
                <property>
                    <name>mapreduce.map.output.key.class</name>
                    <value>org.apache.hadoop.io.Text</value>
                </property>    
                <property>
                    <name>mapreduce.map.output.value.class</name>
                    <value>org.apache.hadoop.io.IntWritable</value>
                </property>    
                <property>
                    <name>mapreduce.job.output.key.class</name>
                    <value>org.apache.hadoop.io.Text</value>
                </property>
                <property>
                    <name>mapreduce.job.output.value.class</name>
                    <value>org.apache.hadoop.io.IntWritable</value>
                </property>
                <property>
                    <name>mapreduce.input.fileinputformat.inputdir</name>
                    <value>${nameNode}/${oozieDataRoot}/${inputDir}</value>
                </property>
                <property>
                    <name>mapreduce.output.fileoutputformat.outputdir</name>
                    <value>${nameNode}/${oozieDataRoot}/${outputDir}</value>
                </property>
            </configuration>
        </map-reduce>
        <ok to="end"/>
        <error to="fail"/>
    </action>
    <kill name="fail">
        <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>


##coordinator.xml
<coordinator-app name="cron-coord-mr" frequency="0/2 * * * *" start="${start}" 
                end="${end}" timezone="GMT+0800"
                 xmlns="uri:oozie:coordinator:0.4">
        <action>
        <workflow>
            <app-path>${workflowAppUri}</app-path>
            <configuration>
                <property>
                    <name>jobTracker</name>
                    <value>${jobTracker}</value>
                </property>
                <property>
                    <name>nameNode</name>
                    <value>${nameNode}</value>
                </property>
                <property>
                    <name>queueName</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
        </workflow>
    </action>
</coordinator-app>

2、运行

##
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# /opt/cdh-5.3.6/hadoop-2.5.0-cdh5.3.6/bin/hdfs dfs -put oozie-apps/cron/ oozie-apps/


##
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozie job -config oozie-apps/cron/job.properties -run