Hive+Sqoop+Mysql整合
在本文中，LZ随意想到了一个场景：
车，道路，监控，摄像头
即当一辆车在道路上面行驶的时候，道路上面的监控点里面的摄像头就会对车进行数据采集。
我们对采集的数据进行分析，处理，最后把结果保存到mysql数据库中供Web UI显示监控点/摄像头状态。
A:监控点/摄像头状态
工作流程如下：
1.数据格式
/** * 产生测试数据： * 数据format： * 记录时间 车牌号码 车速 道路编号 监控地点 摄像头编号 * date_time vehicle_plate vehicle_speed road_id monitor_id camera_id * * 中间使用' '隔开 * 16/01/2019 10:20:30 SCN89000J 124 10002 20004 40007 * * 具体说明： * 道路编号 * 10001 - 10100 * * 监控地点 - 在一条道路上面有2个监控点 * 20001 - 20200 * * 摄像头编号 - 在一个监控点上面2个摄像头 * 40001 - 40400 * * 道路： 10001 10002 * 监控： 20001-20002 20003-20004 * 摄像头： 40001-40002-40003-40004 40005-40006-40007-40008 * * 车速： 1-300。 如果大于260，则为超速行驶 * * 车牌： SCN89000J * * 记录时间： 16/01/2019 10:20:30 * */
2.生成测试数据
--编译运行java code cd /root/vehicle_dir/ vi DataGenerate.java
import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Serializable; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** * 产生测试数据： * 数据format： * 记录时间 车牌号码 车速 道路编号 监控地点 摄像头编号 * date_time vehicle_plate vehicle_speed road_id monitor_id camera_id * * 中间使用' '隔开 * 16/01/2019 10:20:30 SCN89000J 124 10002 20004 40007 * * 具体说明： * 道路编号 * 10001 - 10100 * * 监控地点 - 在一条道路上面有2个监控点 * 20001 - 20200 * * 摄像头编号 - 在一个监控点上面2个摄像头 * 40001 - 40400 * * 道路： 10001 10002 * 监控： 20001-20002 20003-20004 * 摄像头： 40001-40002-40003-40004 40005-40006-40007-40008 * * 车速： 1-300。 如果大于260，则为超速行驶 * * 车牌： SCN89000J * * 记录时间： 16/01/2019 10:20:30 * * @author Hongten */ public class DataGenerate { Integer[] roadIdArray = new Integer[Common.ROAD_NUM]; public static void main(String[] args) { long begin = System.currentTimeMillis(); DataGenerate dataGenerate = new DataGenerate(); dataGenerate.init(); dataGenerate.generateData(); long end = System.currentTimeMillis(); System.out.println("Total: " + (end - begin) + " ms"); } public void init() { // create files FileUtils.createFile(Common.VEHICLE_LOG); FileUtils.createFile(Common.ROAD_MONITOR_CAMERA_RELATIONSHIP); generateRoadIds(); } /** * 道路： 10001 10002 监控： 20001-20002 20003-20004 摄像头： 40001-40002-40003-40004 * 40005-40006-40007-40008 */ public void generateRoadIds() { StringBuilder readMonitorCameraRelationship = new StringBuilder(); for (int i = 0; i < Common.ROAD_NUM; i++) { int roadId = 10000 + (i + 1);// 10001 roadIdArray[i] = roadId; int monitorB = roadId * 2;// 20002 int monitorA = monitorB - 1;// 20001 int cameraAB = monitorA * 2;// 40002 int cameraAA = cameraAB - 1;// 40001 int cameraBB = monitorB * 2;// 40004 int cameraBA = cameraBB - 1;// 40003 // monitorA // 10001 20001 40001 // 10001 20001 40002 readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorA).append(Common.SEPARATOR).append(cameraAA).append(Common.LINE_BREAK); readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorA).append(Common.SEPARATOR).append(cameraAB).append(Common.LINE_BREAK); // monitorB // 10001 20002 40003 // 10001 20002 40004 readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorB).append(Common.SEPARATOR).append(cameraBA).append(Common.LINE_BREAK); readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorB).append(Common.SEPARATOR).append(cameraBB).append(Common.LINE_BREAK); } saveData(Common.ROAD_MONITOR_CAMERA_RELATIONSHIP, readMonitorCameraRelationship.toString()); } public void saveData(String pathFileName, String newContent) { //remove the last ' ' newContent = newContent.substring(0, newContent.length() - 1); FileUtils.saveFile(pathFileName, newContent); } public void generateData() { //StringBuffer可以保证线程安全 StringBuffer contentSb = new StringBuffer(); SimpleDateFormat simpleDateFormat_ddMMyyyy = new SimpleDateFormat(Common.DATE_FORMAT_YYYYMMDD); Date today = new Date(); String date = simpleDateFormat_ddMMyyyy.format(today); Random random = new Random(); //异常道路 List<Integer> errorRoadIdList = new ArrayList<Integer>(); generateErrorRoadIdList(random, errorRoadIdList); long begin = System.currentTimeMillis(); //使用多线程 ExecutorService exec = Executors.newCachedThreadPool(); for (int i = 0; i < Common.VEHICLE_NUMBER; i++) { String vehiclePlate = VehiclePlateGenerateSG.generatePlate(); //使用Future和Callable组合，可以获取到返回值 Future<String> result = exec.submit(new GenerateVehicleLog(date, random, errorRoadIdList, vehiclePlate, roadIdArray)); try { contentSb.append(result.get()); if(i % 100 == 0){ System.out.println(i); } if(i != 0 && i % 900 == 0){ long end = System.currentTimeMillis(); System.out.println(i + " sleeping 1 seconds." + " " + (end - begin)/1000 + " s"); //waiting the pre-task to finish. TimeUnit.SECONDS.sleep(1); } } catch (InterruptedException e) { e.printStackTrace(); } catch (ExecutionException e) { e.printStackTrace(); } //System.out.println(contentSb.toString()); } exec.shutdown(); saveData(Common.VEHICLE_LOG, contentSb.toString()); } private void generateErrorRoadIdList(Random random, List<Integer> errorRoadIdList) { for (int x = 0; x < Common.ROAD_ERROR_NUM; x++) { if (errorRoadIdList.contains(roadIdArray[random.nextInt(roadIdArray.length)])) { generateErrorRoadIdList(random, errorRoadIdList); } else { errorRoadIdList.add(roadIdArray[random.nextInt(roadIdArray.length)]); } } } } class GenerateVehicleLog implements Callable<String>{ StringBuffer contentSb; String date; Random random; List<Integer> errorRoadIdList; String vehiclePlate; Integer[] roadIdArray; public GenerateVehicleLog( String date, Random random, List<Integer> errorRoadIdList, String vehiclePlate, Integer[] roadIdArray){ this.contentSb = new StringBuffer(); this.date = date; this.random = random; this.errorRoadIdList = errorRoadIdList; this.vehiclePlate = vehiclePlate; this.roadIdArray = roadIdArray; } @Override public String call() throws Exception { return getVehicleLog(contentSb, date, random, errorRoadIdList, vehiclePlate, roadIdArray); } private String getVehicleLog(StringBuffer contentSb, String date, Random random, List<Integer> errorRoadIdList, String vehiclePlate, Integer[] roadIdArray) { // 每一辆车产生记录在100条记录以内 // 即最多过100个监控点 // 即最多过50条路 // 这里可以根据需要调节 for (int n = 0; n < random.nextInt(Common.ROAD_NUM); n++) { int roadId = roadIdArray[random.nextInt(roadIdArray.length)]; Integer[] monitorIdArray = new Integer[2]; Integer[] cameraIdArray = new Integer[2]; boolean isAllError = false; int monitorId = 0; if (errorRoadIdList.contains(roadId)) { // System.out.println("find error road.... " + roadId + // " for vehicle : " + vehiclePlate); if (roadId % 2 == 0) { // 监控设备全部坏掉 isAllError = true; } else { // 部分坏掉 monitorIdArray[0] = roadId * 2 - 1; monitorIdArray[1] = roadId * 2 - 1; monitorId = monitorIdArray[random.nextInt(monitorIdArray.length)]; cameraIdArray[0] = roadId * 4 - 3; cameraIdArray[1] = roadId * 4 - 2; } } else { monitorIdArray[0] = roadId * 2 - 1; monitorIdArray[1] = roadId * 2; monitorId = monitorIdArray[random.nextInt(monitorIdArray.length)]; cameraIdArray[0] = monitorId * 2 - 1; cameraIdArray[1] = monitorId * 2; } if (!isAllError) { // 16/01/2019 10:20:30 SCN89000J 124 10002 20004 40007 contentSb.append(date).append(Common.BLANK).append(StringUtils.fulfuill(String.valueOf(random.nextInt(25)))).append(Common.COLON).append(StringUtils.fulfuill(String.valueOf(random.nextInt(61)))).append(Common.COLON).append(StringUtils.fulfuill(String.valueOf(random.nextInt(61)))).append(Common.SEPARATOR); contentSb.append(vehiclePlate).append(Common.SEPARATOR); contentSb.append((random.nextInt(Common.MAX_SPEED) + 1)).append(Common.SEPARATOR); contentSb.append(roadId).append(Common.SEPARATOR); contentSb.append(monitorId).append(Common.SEPARATOR); contentSb.append(cameraIdArray[random.nextInt(cameraIdArray.length)]).append(Common.LINE_BREAK); } } return contentSb.toString(); } } class FileUtils implements Serializable { private static final long serialVersionUID = 1L; public static boolean createFile(String pathFileName) { try { File file = new File(pathFileName); if (file.exists()) { System.err.println("Find file" + pathFileName + ", system will delete it now!!!"); file.delete(); } boolean createNewFile = file.createNewFile(); System.err.println("create file " + pathFileName + " success!"); return createNewFile; } catch (IOException e) { e.printStackTrace(); } return false; } public static void saveFile(String pathFileName, String newContent) { FileOutputStream fos = null; OutputStreamWriter osw = null; PrintWriter pw = null; try { String content = newContent; File file = new File(pathFileName); fos = new FileOutputStream(file, true); osw = new OutputStreamWriter(fos, Common.CHARSETNAME_UTF_8); pw = new PrintWriter(osw); pw.write(content); pw.close(); osw.close(); fos.close(); } catch (IOException e) { e.printStackTrace(); } finally { if (pw != null) { pw.close(); } if (osw != null) { try { osw.close(); } catch (IOException e) { e.printStackTrace(); } } if (fos != null) { try { fos.close(); } catch (IOException e) { e.printStackTrace(); } } } } } class StringUtils { public static String fulfuill(String str) { if (str.length() == 1) { return Common.ZERO + str; } return str; } } /** * From wiki: * https://en.wikipedia.org/wiki/Vehicle_registration_plates_of_Singapore<br> * * A typical vehicle registration number comes in the format "SKV 6201 B":<br> * * <li>S – Vehicle class ("S", with some exceptions, stands for a private * vehicle since 1984)</li><br> * <li>KV – Alphabetical series ("I" and "O" are not used to avoid confusion * with "1" and "0")</li><br> * <li>6201 – Numerical series</li><br> * <li>B – Checksum letter ("F","I", "N", "O", "Q", "V" and "W" are never used * as checksum letters; absent on special government vehicle plates and events * vehicle plates)</li> * */ class VehiclePlateGenerateSG implements Serializable { private static final long serialVersionUID = -8006144823705880339L; public static Random random = new Random(); // 主要目的就是使得产生的数字字符不在这个list里面 // 如果在这个list里面找到，那么需要重新产生 private static List<String> uniqueList = new ArrayList<String>(); public static String generatePlate() { // System.out.println(ALPHABETICAL_ARRAY[18]);// S String alphabeticalSeries = getAlphabeticalStr(Common.ALPHABETICAL_ARRAY, random) + getAlphabeticalStr(Common.ALPHABETICAL_ARRAY, random); // System.out.println(alphabeticalSeries);//KV String numbericalSeries = getNumericalSeriesStr(random); // System.out.println(numbericalSeries);//62010 String checksumLetter = getChecksumLetterStr(Common.ALPHABETICAL_ARRAY, random); // System.out.println(checksumLetter);//B String singaporeVehiclePlate = Common.ALPHABETICAL_ARRAY[18] + alphabeticalSeries + numbericalSeries + checksumLetter; return singaporeVehiclePlate; } private static String getAlphabeticalStr(String[] ALPHABETICAL_ARRAY, Random random) { String alphabeticalStr = Common.ALPHABETICAL_ARRAY[random.nextInt(Common.ALPHABETICAL_ARRAY.length)]; // "I", "O" if (!(alphabeticalStr.equals(Common.ALPHABETICAL_ARRAY[8]) || alphabeticalStr.equals(Common.ALPHABETICAL_ARRAY[14]))) { return alphabeticalStr; } else { return getAlphabeticalStr(Common.ALPHABETICAL_ARRAY, random); } } private static String getNumericalSeriesStr(Random random) { // 为了区别真实的车牌，我们把数字设置为5位 String numericalStr = random.nextInt(10) + "" + random.nextInt(10) + "" + random.nextInt(10) + "" + random.nextInt(10) + "" + random.nextInt(10); if (uniqueList.contains(numericalStr)) { // 如果存在，则重新产生 return getNumericalSeriesStr(random); } else { uniqueList.add(numericalStr); return numericalStr; } } private static String getChecksumLetterStr(String[] ALPHABETICAL_ARRAY, Random random) { String checksumLetter = ALPHABETICAL_ARRAY[random.nextInt(ALPHABETICAL_ARRAY.length)]; // "F","I", "N", "O", "Q", "V" and "W" if (!(checksumLetter.equals(Common.ALPHABETICAL_ARRAY[5]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[8]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[13]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[14]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[16]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[21]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[22]))) { return checksumLetter; } else { return getChecksumLetterStr(ALPHABETICAL_ARRAY, random); } } } class Common implements Serializable { private static final long serialVersionUID = 1L; public static String VEHICLE_LOG = "./vehicle_log"; public static String ROAD_MONITOR_CAMERA_RELATIONSHIP = "./road_monitor_camera_relationship"; public static final String[] ALPHABETICAL_ARRAY = new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" }; public static final String DATE_FORMAT_YYYYMMDD = "dd/MM/yyyy"; public static final String CHARSETNAME_UTF_8 = "UTF-8"; public static final String SEPARATOR = " "; public static final String LINE_BREAK = " "; public static final String BLANK = " "; public static final String COLON = ":"; public static final String ZERO = "0"; //车辆数 public static final int VEHICLE_NUMBER = 100000; //道路数 public static final int ROAD_NUM = 400; public static final int ROAD_ERROR_NUM = 8; //最大车速 public static final int MAX_SPEED = 300; }
2.1.编译执行
:wq javac DataGenerate.java java DataGenerate --运行完成，会生成下面两个文件 /root/vehicle_dir/vehicle_log /root/vehicle_dir/road_monitor_camera_relationship
2.2.车辆记录log样本
16/01/2019 14:06:44 SVM35185L 258 10295 20590 41179
16/01/2019 15:56:25 SVM35185L 110 10288 20575 41149
16/01/2019 02:22:29 SVM35185L 28 10109 20217 40436
16/01/2019 24:29:59 SSK43417H 254 10281 20562 41123
16/01/2019 07:36:54 SSK43417H 149 10124 20247 40495
16/01/2019 12:21:30 SSK43417H 196 10211 20421 40843
16/01/2019 12:42:43 SSK43417H 92 10308 20615 41230
16/01/2019 02:57:59 SDV20274X 206 10166 20332 40663
16/01/2019 11:60:17 SDV20274X 191 10372 20744 41488
16/01/2019 00:09:06 SDV20274X 197 10094 20188 40374
16/01/2019 21:18:30 SDV20274X 294 10101 20201 40401
16/01/2019 11:23:38 SDV20274X 74 10163 20325 40652
16/01/2019 04:35:16 SDV20274X 53 10077 20153 40305
16/01/2019 20:56:56 SDV20274X 31 10113 20226 40449
16/01/2019 16:50:11 SEN89218Y 58 10202 20404 40808
16/01/2019 18:34:47 SEN89218Y 113 10042 20083 40168
16/01/2019 02:25:52 SEN89218Y 35 10051 20101 40204
16/01/2019 24:08:52 SEN89218Y 77 10165 20330 40657
2.3.道路-监控-摄像头关系样本
10001 20001 40001
10001 20001 40002
10001 20002 40003
10001 20002 40004
10002 20003 40005
10002 20003 40006
10002 20004 40007
10002 20004 40008
10003 20005 40009
10003 20005 40010
10003 20006 40011
10003 20006 40012
10004 20007 40013
10004 20007 40014
10004 20008 40015
10004 20008 40016
3.在Hive中创建table并且导入数据
-- 创建table，并且把结果数据导入到Hive table里面 cd /root/vehicle_dir/ vi hive_vehicle.sql --1.drop t_vehicle_log drop table IF EXISTS t_vehicle_log; --2.create t_vehicle_log CREATE TABLE t_vehicle_log( date_time string , vehicle_plate string , vehicle_speed int , road_id string , monitor_id string , camera_id string )ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' LINES TERMINATED BY ' '; --3.load data into t_vehicle_log load data local inpath '/root/vehicle_dir/vehicle_log' into table t_vehicle_log; --4.drop t_road_monitor_camera_relationship drop table IF EXISTS t_road_monitor_camera_relationship; --5.create t_road_monitor_camera_relationship CREATE TABLE t_road_monitor_camera_relationship( road_id string , monitor_id string , camera_id string )ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' LINES TERMINATED BY ' '; --6.load data into t_road_monitor_camera_relationship load data local inpath '/root/vehicle_dir/road_monitor_camera_relationship' into table t_road_monitor_camera_relationship; --7.drop t_monitor_camera drop table IF EXISTS t_monitor_camera; --8.create t_monitor_camera create table t_monitor_camera( monitor_id string , cameranum int, workingcameranum int, notWorkingCameraNum int )ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY ' '; --9.load data from other table into t_monitor_camera from (select monitor_id, count(distinct camera_id) cameraNum from t_road_monitor_camera_relationship group by monitor_id) t1 left outer join (select monitor_id, NVL(count(distinct camera_id), 0) workingCameraNum from t_vehicle_log group by monitor_id) t2 on t1.monitor_id=t2.monitor_id insert into table t_monitor_camera select t1.monitor_id, t1.cameraNum cameraNum, NVL(t2.workingCameraNum, 0) workingCameraNum,NVL((t1.cameraNum - NVL(t2.workingCameraNum, 0)), 0) notWorkingCameraNum;
4.编写Sqoop配置文件
--配置sqoop：hive数据导入到mysql中 --注意： --export-dir /user/hive/warehouse/t_monitor_camera/ 这里的地址可以在hive中， --通过desc formatted t_monitor_camera 查看 --Location: hdfs://mycluster/user/hive/warehouse/t_monitor_camera cd /root/vehicle_dir/ vi hive_to_mysql_for_vehicle export --connect jdbc:mysql://node1:3306/sqoop_db --username root --password '!QAZ2wsx3edc' --table t_hive_to_mysql_for_vehicle -m 1 --columns monitor_id,camera_num,working_camera_num,not_working_camera_num --fields-terminated-by '|' --export-dir /user/hive/warehouse/t_monitor_camera/ ：wq
5.在Mysql中创建table
--在mysql里面创建表 mysql -u root -p !QAZ2wsx3edc use sqoop_db; --如果有则删除 DROP TABLE IF EXISTS t_hive_to_mysql_for_vehicle; CREATE TABLE t_hive_to_mysql_for_vehicle (monitor_id VARCHAR(5), camera_num INT, working_camera_num INT, not_working_camera_num INT);
6.编写自动化可执行脚本
--编辑可执行脚本 cd /root/vehicle_dir/ vi hive_to_mysql_vehicle.sh echo 'job begin' cd /home/hive/bin ./hive -f /root/vehicle_dir/hive_vehicle.sql echo 'begin to inport to mysql' sqoop --options-file /root/vehicle_dir/hive_to_mysql_for_vehicle echo 'done.' :wq
7.赋予脚本可执行属性
--赋予脚本可执行属性 chmod +x hive_to_mysql_vehicle.sh
8.执行脚本
--执行脚本 ./hive_to_mysql_vehicle.sh
9.结果
9.1.执行脚本前，检查mysql table
--执行脚本之前，查询t_hive_to_mysql_for_vehicle mysql> select * from t_hive_to_mysql_for_vehicle; Empty set (0.00 sec)
9.2.执行脚本
[root@node1 vehicle_dir]# ./hive_to_mysql_vehicle.sh job begin 19/01/16 01:00:53 WARN conf.HiveConf: DEPRECATED: hive.metastore.ds.retry.* no longer has any effect. Use hive.hmshandler.retry.* instead Logging initialized using configuration in jar:file:/root/apache-hive-0.13.1-bin/lib/hive-common-0.13.1.jar!/hive-log4j.properties SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/root/hadoop-2.5.1/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/root/apache-hive-0.13.1-bin/lib/slf4j-log4j12-1.6.4.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] OK Time taken: 1.432 seconds OK Time taken: 0.306 seconds Copying data from file:/root/vehicle_dir/vehicle_log Copying file: file:/root/vehicle_dir/vehicle_log Loading data to table default.t_vehicle_log Table default.t_vehicle_log stats: [numFiles=1, numRows=0, totalSize=121817379, rawDataSize=0] OK Time taken: 5.958 seconds OK Time taken: 0.101 seconds OK Time taken: 0.042 seconds Copying data from file:/root/vehicle_dir/road_monitor_camera_relationship Copying file: file:/root/vehicle_dir/road_monitor_camera_relationship Loading data to table default.t_road_monitor_camera_relationship Table default.t_road_monitor_camera_relationship stats: [numFiles=1, numRows=0, totalSize=28799, rawDataSize=0] OK Time taken: 0.637 seconds OK Time taken: 0.094 seconds OK Time taken: 0.071 seconds Total jobs = 4 Launching Job 1 out of 4 Number of reduce tasks not specified. Estimated from input data size: 1 In order to change the average load for a reducer (in bytes): set hive.exec.reducers.bytes.per.reducer=<number> In order to limit the maximum number of reducers: set hive.exec.reducers.max=<number> In order to set a constant number of reducers: set mapreduce.job.reduces=<number> Starting Job = job_1547622676146_0025, Tracking URL = http://node1:8088/proxy/application_1547622676146_0025/ Kill Command = /home/hadoop-2.5/bin/hadoop job -kill job_1547622676146_0025 Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 1 2019-01-16 01:01:29,337 Stage-1 map = 0%, reduce = 0% 2019-01-16 01:01:40,583 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 2.06 sec 2019-01-16 01:02:27,667 Stage-1 map = 100%, reduce = 100%, Cumulative CPU 4.65 sec MapReduce Total cumulative CPU time: 4 seconds 650 msec Ended Job = job_1547622676146_0025 Launching Job 2 out of 4 Number of reduce tasks not specified. Estimated from input data size: 1 In order to change the average load for a reducer (in bytes): set hive.exec.reducers.bytes.per.reducer=<number> In order to limit the maximum number of reducers: set hive.exec.reducers.max=<number> In order to set a constant number of reducers: set mapreduce.job.reduces=<number> Starting Job = job_1547622676146_0026, Tracking URL = http://node1:8088/proxy/application_1547622676146_0026/ Kill Command = /home/hadoop-2.5/bin/hadoop job -kill job_1547622676146_0026 Hadoop job information for Stage-4: number of mappers: 1; number of reducers: 1 2019-01-16 01:02:43,341 Stage-4 map = 0%, reduce = 0% 2019-01-16 01:03:01,206 Stage-4 map = 100%, reduce = 0%, Cumulative CPU 6.99 sec 2019-01-16 01:03:10,440 Stage-4 map = 100%, reduce = 100%, Cumulative CPU 9.8 sec MapReduce Total cumulative CPU time: 9 seconds 800 msec Ended Job = job_1547622676146_0026 Stage-7 is selected by condition resolver. Stage-2 is filtered out by condition resolver. SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/root/hadoop-2.5.1/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/root/apache-hive-0.13.1-bin/lib/slf4j-log4j12-1.6.4.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] 19/01/16 01:03:16 WARN conf.Configuration: file:/tmp/root/hive_2019-01-16_01-01-07_044_1858062061865079126-1/-local-10011/jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval; Ignoring. 19/01/16 01:03:16 WARN conf.Configuration: file:/tmp/root/hive_2019-01-16_01-01-07_044_1858062061865079126-1/-local-10011/jobconf.xml:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts; Ignoring. 19/01/16 01:03:17 WARN conf.HiveConf: DEPRECATED: hive.metastore.ds.retry.* no longer has any effect. Use hive.hmshandler.retry.* instead Execution log at: /tmp/root/root_20190116010101_c2813672-251f-4087-a452-9c26df2565e8.log 2019-01-16 01:03:17 Starting to launch local task to process map join; maximum memory = 477102080 2019-01-16 01:03:18 Dump the side-table into file: file:/tmp/root/hive_2019-01-16_01-01-07_044_1858062061865079126-1/-local-10004/HashTable-Stage-5/MapJoin-mapfile01--.hashtable 2019-01-16 01:03:18 Uploaded 1 File to: file:/tmp/root/hive_2019-01-16_01-01-07_044_1858062061865079126-1/-local-10004/HashTable-Stage-5/MapJoin-mapfile01--.hashtable (20083 bytes) 2019-01-16 01:03:18 End of local task; Time Taken: 1.033 sec. Execution completed successfully MapredLocal task succeeded Launching Job 4 out of 4 Number of reduce tasks is set to 0 since there's no reduce operator Starting Job = job_1547622676146_0027, Tracking URL = http://node1:8088/proxy/application_1547622676146_0027/ Kill Command = /home/hadoop-2.5/bin/hadoop job -kill job_1547622676146_0027 Hadoop job information for Stage-5: number of mappers: 1; number of reducers: 0 2019-01-16 01:03:28,599 Stage-5 map = 0%, reduce = 0% 2019-01-16 01:03:41,568 Stage-5 map = 100%, reduce = 0%, Cumulative CPU 1.83 sec MapReduce Total cumulative CPU time: 1 seconds 830 msec Ended Job = job_1547622676146_0027 Loading data to table default.t_monitor_camera Table default.t_monitor_camera stats: [numFiles=1, numRows=800, totalSize=9600, rawDataSize=8800] MapReduce Jobs Launched: Job 0: Map: 1 Reduce: 1 Cumulative CPU: 4.65 sec HDFS Read: 29057 HDFS Write: 19476 SUCCESS Job 1: Map: 1 Reduce: 1 Cumulative CPU: 9.8 sec HDFS Read: 121817595 HDFS Write: 19212 SUCCESS Job 2: Map: 1 Cumulative CPU: 1.83 sec HDFS Read: 19834 HDFS Write: 9683 SUCCESS Total MapReduce CPU Time Spent: 16 seconds 280 msec OK _col0 _col1 _col2 _col3 Time taken: 155.84 seconds begin to inport to mysql 19/01/16 01:03:44 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6 19/01/16 01:03:44 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead. 19/01/16 01:03:44 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset. 19/01/16 01:03:44 INFO tool.CodeGenTool: Beginning code generation 19/01/16 01:03:45 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `t_hive_to_mysql_for_vehicle` AS t LIMIT 1 19/01/16 01:03:45 INFO manager.SqlManager: Executing SQL statement: SELECT t.* FROM `t_hive_to_mysql_for_vehicle` AS t LIMIT 1 19/01/16 01:03:45 INFO orm.CompilationManager: HADOOP_MAPRED_HOME is /home/hadoop-2.5 Note: /tmp/sqoop-root/compile/abc965816faa58aeae64d67acf60c1af/t_hive_to_mysql_for_vehicle.java uses or overrides a deprecated API. Note: Recompile with -Xlint:deprecation for details. 19/01/16 01:03:46 INFO orm.CompilationManager: Writing jar file: /tmp/sqoop-root/compile/abc965816faa58aeae64d67acf60c1af/t_hive_to_mysql_for_vehicle.jar 19/01/16 01:03:46 INFO mapreduce.ExportJobBase: Beginning export of t_hive_to_mysql_for_vehicle SLF4J: Class path contains multiple SLF4J bindings. SLF4J: Found binding in [jar:file:/root/hadoop-2.5.1/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: Found binding in [jar:file:/root/hbase-0.98.9-hadoop2/lib/slf4j-log4j12-1.6.4.jar!/org/slf4j/impl/StaticLoggerBinder.class] SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] 19/01/16 01:03:47 INFO Configuration.deprecation: mapred.jar is deprecated. Instead, use mapreduce.job.jar 19/01/16 01:03:48 INFO Configuration.deprecation: mapred.reduce.tasks.speculative.execution is deprecated. Instead, use mapreduce.reduce.speculative 19/01/16 01:03:48 INFO Configuration.deprecation: mapred.map.tasks.speculative.execution is deprecated. Instead, use mapreduce.map.speculative 19/01/16 01:03:48 INFO Configuration.deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 19/01/16 01:03:48 INFO client.RMProxy: Connecting to ResourceManager at node1/192.168.79.138:8032 19/01/16 01:03:52 INFO input.FileInputFormat: Total input paths to process : 1 19/01/16 01:03:52 INFO input.FileInputFormat: Total input paths to process : 1 19/01/16 01:03:52 INFO mapreduce.JobSubmitter: number of splits:1 19/01/16 01:03:52 INFO Configuration.deprecation: mapred.map.tasks.speculative.execution is deprecated. Instead, use mapreduce.map.speculative 19/01/16 01:03:53 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1547622676146_0028 19/01/16 01:03:53 INFO impl.YarnClientImpl: Submitted application application_1547622676146_0028 19/01/16 01:03:53 INFO mapreduce.Job: The url to track the job: http://node1:8088/proxy/application_1547622676146_0028/ 19/01/16 01:03:53 INFO mapreduce.Job: Running job: job_1547622676146_0028 19/01/16 01:04:01 INFO mapreduce.Job: Job job_1547622676146_0028 running in uber mode : false 19/01/16 01:04:01 INFO mapreduce.Job: map 0% reduce 0% 19/01/16 01:04:08 INFO mapreduce.Job: map 100% reduce 0% 19/01/16 01:04:09 INFO mapreduce.Job: Job job_1547622676146_0028 completed successfully 19/01/16 01:04:09 INFO mapreduce.Job: Counters: 30 File System Counters FILE: Number of bytes read=0 FILE: Number of bytes written=116265 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=9746 HDFS: Number of bytes written=0 HDFS: Number of read operations=4 HDFS: Number of large read operations=0 HDFS: Number of write operations=0 Job Counters Launched map tasks=1 Rack-local map tasks=1 Total time spent by all maps in occupied slots (ms)=5229 Total time spent by all reduces in occupied slots (ms)=0 Total time spent by all map tasks (ms)=5229 Total vcore-seconds taken by all map tasks=5229 Total megabyte-seconds taken by all map tasks=5354496 Map-Reduce Framework Map input records=800 Map output records=800 Input split bytes=143 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=29 CPU time spent (ms)=1150 Physical memory (bytes) snapshot=184467456 Virtual memory (bytes) snapshot=902938624 Total committed heap usage (bytes)=106954752 File Input Format Counters Bytes Read=0 File Output Format Counters Bytes Written=0 19/01/16 01:04:09 INFO mapreduce.ExportJobBase: Transferred 9.5176 KB in 21.8085 seconds (446.8896 bytes/sec) 19/01/16 01:04:09 INFO mapreduce.ExportJobBase: Exported 800 records. done. [root@node1 vehicle_dir]# mysql -u root -p Enter password: ERROR 1045 (28000): Access denied for user 'root'@'localhost' (using password: YES) [root@node1 vehicle_dir]# mysql -u root -p Enter password: Welcome to the MySQL monitor. Commands end with ; or g. Your MySQL connection id is 617 Server version: 5.7.24 MySQL Community Server (GPL) Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective owners. Type 'help;' or 'h' for help. Type 'c' to clear the current input statement. mysql> use sqoop_db; Reading table information for completion of table and column names You can turn off this feature to get a quicker startup with -A Database changed mysql> select * from t_hive_to_mysql_for_vehicle; +------------+------------+--------------------+------------------------+ | monitor_id | camera_num | working_camera_num | not_working_camera_num | +------------+------------+--------------------+------------------------+ | 20001 | 2 | 2 | 0 | | 20002 | 2 | 2 | 0 | | 20003 | 2 | 2 | 0 | | 20004 | 2 | 2 | 0 | | 20005 | 2 | 2 | 0 | | 20006 | 2 | 2 | 0 | | 20007 | 2 | 2 | 0 | | 20008 | 2 | 2 | 0 | | 20009 | 2 | 2 | 0 | | 20010 | 2 | 2 | 0 | | 20011 | 2 | 2 | 0 | | 20012 | 2 | 2 | 0 | | 20013 | 2 | 2 | 0 | | 20014 | 2 | 2 | 0 | | 20015 | 2 | 2 | 0 | | 20016 | 2 | 2 | 0 | | 20017 | 2 | 2 | 0 | | 20018 | 2 | 2 | 0 | | 20019 | 2 | 2 | 0 | | 20020 | 2 | 2 | 0 | | 20021 | 2 | 2 | 0 | | 20022 | 2 | 2 | 0 | | 20023 | 2 | 2 | 0 | | 20024 | 2 | 2 | 0 | | 20025 | 2 | 2 | 0 | | 20026 | 2 | 2 | 0 | | 20027 | 2 | 2 | 0 | | 20028 | 2 | 2 | 0 | | 20029 | 2 | 2 | 0 | | 20030 | 2 | 2 | 0 | | 20031 | 2 | 2 | 0 | | 20032 | 2 | 2 | 0 | | 20033 | 2 | 2 | 0 | | 20034 | 2 | 2 | 0 | | 20035 | 2 | 2 | 0 | | 20036 | 2 | 2 | 0 | | 20037 | 2 | 2 | 0 | | 20038 | 2 | 2 | 0 | | 20039 | 2 | 2 | 0 | | 20040 | 2 | 2 | 0 | | 20041 | 2 | 2 | 0 | | 20042 | 2 | 2 | 0 | | 20043 | 2 | 2 | 0 | | 20044 | 2 | 2 | 0 | | 20045 | 2 | 2 | 0 | | 20046 | 2 | 2 | 0 | | 20047 | 2 | 2 | 0 | | 20048 | 2 | 2 | 0 | | 20049 | 2 | 2 | 0 | | 20050 | 2 | 2 | 0 | | 20051 | 2 | 2 | 0 | | 20052 | 2 | 2 | 0 | | 20053 | 2 | 2 | 0 | | 20054 | 2 | 2 | 0 | | 20055 | 2 | 2 | 0 | | 20056 | 2 | 2 | 0 | | 20057 | 2 | 2 | 0 | | 20058 | 2 | 2 | 0 | | 20059 | 2 | 2 | 0 | | 20060 | 2 | 2 | 0 | | 20061 | 2 | 2 | 0 | | 20062 | 2 | 2 | 0 | | 20063 | 2 | 2 | 0 | | 20064 | 2 | 2 | 0 | | 20065 | 2 | 2 | 0 | | 20066 | 2 | 2 | 0 | | 20067 | 2 | 2 | 0 | | 20068 | 2 | 2 | 0 | | 20069 | 2 | 2 | 0 | | 20070 | 2 | 2 | 0 | | 20071 | 2 | 2 | 0 | | 20072 | 2 | 2 | 0 | | 20073 | 2 | 2 | 0 | | 20074 | 2 | 2 | 0 | | 20075 | 2 | 2 | 0 | | 20076 | 2 | 2 | 0 | | 20077 | 2 | 2 | 0 | | 20078 | 2 | 2 | 0 | | 20079 | 2 | 2 | 0 | | 20080 | 2 | 2 | 0 | | 20081 | 2 | 2 | 0 | | 20082 | 2 | 2 | 0 | | 20083 | 2 | 2 | 0 | | 20084 | 2 | 2 | 0 | | 20085 | 2 | 2 | 0 | | 20086 | 2 | 2 | 0 | | 20087 | 2 | 2 | 0 | | 20088 | 2 | 2 | 0 | | 20089 | 2 | 2 | 0 | | 20090 | 2 | 0 | 2 | | 20091 | 2 | 2 | 0 | | 20092 | 2 | 2 | 0 | | 20093 | 2 | 2 | 0 | | 20094 | 2 | 2 | 0 | | 20095 | 2 | 2 | 0 | | 20096 | 2 | 2 | 0 | | 20097 | 2 | 2 | 0 | | 20098 | 2 | 2 | 0 | | 20099 | 2 | 0 | 2 | | 20100 | 2 | 0 | 2 | | 20101 | 2 | 2 | 0 | | 20102 | 2 | 2 | 0 | | 20103 | 2 | 2 | 0 | | 20104 | 2 | 2 | 0 | | 20105 | 2 | 2 | 0 | | 20106 | 2 | 2 | 0 | | 20107 | 2 | 2 | 0 | | 20108 | 2 | 2 | 0 | | 20109 | 2 | 2 | 0 | | 20110 | 2 | 2 | 0 | | 20111 | 2 | 2 | 0 | | 20112 | 2 | 2 | 0 | | 20113 | 2 | 2 | 0 | | 20114 | 2 | 2 | 0 | | 20115 | 2 | 2 | 0 | | 20116 | 2 | 2 | 0 | | 20117 | 2 | 2 | 0 | | 20118 | 2 | 2 | 0 | | 20119 | 2 | 2 | 0 | | 20120 | 2 | 2 | 0 | | 20121 | 2 | 2 | 0 | | 20122 | 2 | 2 | 0 | | 20123 | 2 | 2 | 0 | | 20124 | 2 | 2 | 0 | | 20125 | 2 | 2 | 0 | | 20126 | 2 | 2 | 0 | | 20127 | 2 | 2 | 0 | | 20128 | 2 | 2 | 0 | | 20129 | 2 | 2 | 0 | | 20130 | 2 | 0 | 2 | | 20131 | 2 | 2 | 0 | | 20132 | 2 | 2 | 0 | | 20133 | 2 | 2 | 0 | | 20134 | 2 | 2 | 0 | | 20135 | 2 | 2 | 0 | | 20136 | 2 | 2 | 0 | | 20137 | 2 | 2 | 0 | | 20138 | 2 | 2 | 0 | | 20139 | 2 | 2 | 0 | | 20140 | 2 | 2 | 0 | | 20141 | 2 | 2 | 0 | | 20142 | 2 | 2 | 0 | | 20143 | 2 | 2 | 0 | | 20144 | 2 | 2 | 0 | | 20145 | 2 | 2 | 0 | | 20146 | 2 | 2 | 0 | | 20147 | 2 | 2 | 0 | | 20148 | 2 | 2 | 0 | | 20149 | 2 | 2 | 0 | | 20150 | 2 | 2 | 0 | | 20151 | 2 | 2 | 0 | | 20152 | 2 | 2 | 0 | | 20153 | 2 | 2 | 0 | | 20154 | 2 | 2 | 0 | | 20155 | 2 | 2 | 0 | | 20156 | 2 | 2 | 0 | | 20157 | 2 | 2 | 0 | | 20158 | 2 | 2 | 0 | | 20159 | 2 | 2 | 0 | | 20160 | 2 | 2 | 0 | | 20161 | 2 | 2 | 0 | | 20162 | 2 | 2 | 0 | | 20163 | 2 | 2 | 0 | | 20164 | 2 | 2 | 0 | | 20165 | 2 | 2 | 0 | | 20166 | 2 | 2 | 0 | | 20167 | 2 | 2 | 0 | | 20168 | 2 | 2 | 0 | | 20169 | 2 | 2 | 0 | | 20170 | 2 | 2 | 0 | | 20171 | 2 | 2 | 0 | | 20172 | 2 | 2 | 0 | | 20173 | 2 | 2 | 0 | | 20174 | 2 | 2 | 0 | | 20175 | 2 | 2 | 0 | | 20176 | 2 | 2 | 0 | | 20177 | 2 | 2 | 0 | | 20178 | 2 | 0 | 2 | | 20179 | 2 | 2 | 0 | | 20180 | 2 | 2 | 0 | | 20181 | 2 | 2 | 0 | | 20182 | 2 | 2 | 0 | | 20183 | 2 | 2 | 0 | | 20184 | 2 | 2 | 0 | | 20185 | 2 | 2 | 0 | | 20186 | 2 | 2 | 0 | | 20187 | 2 | 2 | 0 | | 20188 | 2 | 2 | 0 | | 20189 | 2 | 2 | 0 | | 20190 | 2 | 2 | 0 | | 20191 | 2 | 2 | 0 | | 20192 | 2 | 2 | 0 | | 20193 | 2 | 2 | 0 | | 20194 | 2 | 2 | 0 | | 20195 | 2 | 2 | 0 | | 20196 | 2 | 2 | 0 | | 20197 | 2 | 2 | 0 | | 20198 | 2 | 2 | 0 | | 20199 | 2 | 2 | 0 | | 20200 | 2 | 2 | 0 | | 20201 | 2 | 2 | 0 | | 20202 | 2 | 2 | 0 | | 20203 | 2 | 2 | 0 | | 20204 | 2 | 2 | 0 | | 20205 | 2 | 2 | 0 | | 20206 | 2 | 2 | 0 | | 20207 | 2 | 2 | 0 | | 20208 | 2 | 2 | 0 | | 20209 | 2 | 2 | 0 | | 20210 | 2 | 2 | 0 | | 20211 | 2 | 2 | 0 | | 20212 | 2 | 2 | 0 | | 20213 | 2 | 2 | 0 | | 20214 | 2 | 2 | 0 | | 20215 | 2 | 2 | 0 | | 20216 | 2 | 2 | 0 | | 20217 | 2 | 2 | 0 | | 20218 | 2 | 2 | 0 | | 20219 | 2 | 2 | 0 | | 20220 | 2 | 2 | 0 | | 20221 | 2 | 2 | 0 | | 20222 | 2 | 2 | 0 | | 20223 | 2 | 2 | 0 | | 20224 | 2 | 2 | 0 | | 20225 | 2 | 2 | 0 | | 20226 | 2 | 2 | 0 | | 20227 | 2 | 2 | 0 | | 20228 | 2 | 2 | 0 | | 20229 | 2 | 2 | 0 | | 20230 | 2 | 2 | 0 | | 20231 | 2 | 2 | 0 | | 20232 | 2 | 2 | 0 | | 20233 | 2 | 2 | 0 | | 20234 | 2 | 2 | 0 | | 20235 | 2 | 2 | 0 | | 20236 | 2 | 2 | 0 | | 20237 | 2 | 2 | 0 | | 20238 | 2 | 2 | 0 | | 20239 | 2 | 2 | 0 | | 20240 | 2 | 2 | 0 | | 20241 | 2 | 2 | 0 | | 20242 | 2 | 2 | 0 | | 20243 | 2 | 2 | 0 | | 20244 | 2 | 2 | 0 | | 20245 | 2 | 2 | 0 | | 20246 | 2 | 2 | 0 | | 20247 | 2 | 2 | 0 | | 20248 | 2 | 2 | 0 | | 20249 | 2 | 2 | 0 | | 20250 | 2 | 2 | 0 | | 20251 | 2 | 2 | 0 | | 20252 | 2 | 2 | 0 | | 20253 | 2 | 2 | 0 | | 20254 | 2 | 2 | 0 | | 20255 | 2 | 2 | 0 | | 20256 | 2 | 2 | 0 | | 20257 | 2 | 2 | 0 | | 20258 | 2 | 2 | 0 | | 20259 | 2 | 2 | 0 | | 20260 | 2 | 2 | 0 | | 20261 | 2 | 2 | 0 | | 20262 | 2 | 2 | 0 | | 20263 | 2 | 2 | 0 | | 20264 | 2 | 2 | 0 | | 20265 | 2 | 2 | 0 | | 20266 | 2 | 2 | 0 | | 20267 | 2 | 2 | 0 | | 20268 | 2 | 2 | 0 | | 20269 | 2 | 2 | 0 | | 20270 | 2 | 2 | 0 | | 20271 | 2 | 2 | 0 | | 20272 | 2 | 2 | 0 | | 20273 | 2 | 2 | 0 | | 20274 | 2 | 2 | 0 | | 20275 | 2 | 2 | 0 | | 20276 | 2 | 2 | 0 | | 20277 | 2 | 2 | 0 | | 20278 | 2 | 2 | 0 | | 20279 | 2 | 2 | 0 | | 20280 | 2 | 2 | 0 | | 20281 | 2 | 2 | 0 | | 20282 | 2 | 2 | 0 | | 20283 | 2 | 2 | 0 | | 20284 | 2 | 2 | 0 | | 20285 | 2 | 2 | 0 | | 20286 | 2 | 2 | 0 | | 20287 | 2 | 2 | 0 | | 20288 | 2 | 2 | 0 | | 20289 | 2 | 2 | 0 | | 20290 | 2 | 2 | 0 | | 20291 | 2 | 2 | 0 | | 20292 | 2 | 2 | 0 | | 20293 | 2 | 2 | 0 | | 20294 | 2 | 2 | 0 | | 20295 | 2 | 2 | 0 | | 20296 | 2 | 2 | 0 | | 20297 | 2 | 2 | 0 | | 20298 | 2 | 2 | 0 | | 20299 | 2 | 2 | 0 | | 20300 | 2 | 2 | 0 | | 20301 | 2 | 2 | 0 | | 20302 | 2 | 2 | 0 | | 20303 | 2 | 2 | 0 | | 20304 | 2 | 2 | 0 | | 20305 | 2 | 2 | 0 | | 20306 | 2 | 2 | 0 | | 20307 | 2 | 2 | 0 | | 20308 | 2 | 2 | 0 | | 20309 | 2 | 2 | 0 | | 20310 | 2 | 2 | 0 | | 20311 | 2 | 2 | 0 | | 20312 | 2 | 2 | 0 | | 20313 | 2 | 2 | 0 | | 20314 | 2 | 2 | 0 | | 20315 | 2 | 2 | 0 | | 20316 | 2 | 2 | 0 | | 20317 | 2 | 2 | 0 | | 20318 | 2 | 2 | 0 | | 20319 | 2 | 2 | 0 | | 20320 | 2 | 2 | 0 | | 20321 | 2 | 2 | 0 | | 20322 | 2 | 2 | 0 | | 20323 | 2 | 2 | 0 | | 20324 | 2 | 2 | 0 | | 20325 | 2 | 2 | 0 | | 20326 | 2 | 2 | 0 | | 20327 | 2 | 2 | 0 | | 20328 | 2 | 2 | 0 | | 20329 | 2 | 2 | 0 | | 20330 | 2 | 2 | 0 | | 20331 | 2 | 2 | 0 | | 20332 | 2 | 2 | 0 | | 20333 | 2 | 2 | 0 | | 20334 | 2 | 2 | 0 | | 20335 | 2 | 2 | 0 | | 20336 | 2 | 2 | 0 | | 20337 | 2 | 2 | 0 | | 20338 | 2 | 2 | 0 | | 20339 | 2 | 2 | 0 | | 20340 | 2 | 2 | 0 | | 20341 | 2 | 2 | 0 | | 20342 | 2 | 2 | 0 | | 20343 | 2 | 2 | 0 | | 20344 | 2 | 2 | 0 | | 20345 | 2 | 2 | 0 | | 20346 | 2 | 2 | 0 | | 20347 | 2 | 2 | 0 | | 20348 | 2 | 2 | 0 | | 20349 | 2 | 2 | 0 | | 20350 | 2 | 2 | 0 | | 20351 | 2 | 2 | 0 | | 20352 | 2 | 2 | 0 | | 20353 | 2 | 2 | 0 | | 20354 | 2 | 0 | 2 | | 20355 | 2 | 2 | 0 | | 20356 | 2 | 2 | 0 | | 20357 | 2 | 2 | 0 | | 20358 | 2 | 2 | 0 | | 20359 | 2 | 2 | 0 | | 20360 | 2 | 2 | 0 | | 20361 | 2 | 2 | 0 | | 20362 | 2 | 2 | 0 | | 20363 | 2 | 0 | 2 | | 20364 | 2 | 0 | 2 | | 20365 | 2 | 2 | 0 | | 20366 | 2 | 2 | 0 | | 20367 | 2 | 2 | 0 | | 20368 | 2 | 2 | 0 | | 20369 | 2 | 2 | 0 | | 20370 | 2 | 2 | 0 | | 20371 | 2 | 0 | 2 | | 20372 | 2 | 0 | 2 | | 20373 | 2 | 2 | 0 | | 20374 | 2 | 2 | 0 | | 20375 | 2 | 2 | 0 | | 20376 | 2 | 2 | 0 | | 20377 | 2 | 2 | 0 | | 20378 | 2 | 2 | 0 | | 20379 | 2 | 2 | 0 | | 20380 | 2 | 2 | 0 | | 20381 | 2 | 2 | 0 | | 20382 | 2 | 2 | 0 | | 20383 | 2 | 2 | 0 | | 20384 | 2 | 2 | 0 | | 20385 | 2 | 2 | 0 | | 20386 | 2 | 2 | 0 | | 20387 | 2 | 2 | 0 | | 20388 | 2 | 2 | 0 | | 20389 | 2 | 2 | 0 | | 20390 | 2 | 2 | 0 | | 20391 | 2 | 2 | 0 | | 20392 | 2 | 2 | 0 | | 20393 | 2 | 2 | 0 | | 20394 | 2 | 2 | 0 | | 20395 | 2 | 2 | 0 | | 20396 | 2 | 2 | 0 | | 20397 | 2 | 2 | 0 | | 20398 | 2 | 2 | 0 | | 20399 | 2 | 2 | 0 | | 20400 | 2 | 2 | 0 | | 20401 | 2 | 2 | 0 | | 20402 | 2 | 2 | 0 | | 20403 | 2 | 2 | 0 | | 20404 | 2 | 2 | 0 | | 20405 | 2 | 2 | 0 | | 20406 | 2 | 2 | 0 | | 20407 | 2 | 2 | 0 | | 20408 | 2 | 2 | 0 | | 20409 | 2 | 2 | 0 | | 20410 | 2 | 2 | 0 | | 20411 | 2 | 2 | 0 | | 20412 | 2 | 2 | 0 | | 20413 | 2 | 2 | 0 | | 20414 | 2 | 2 | 0 | | 20415 | 2 | 2 | 0 | | 20416 | 2 | 2 | 0 | | 20417 | 2 | 2 | 0 | | 20418 | 2 | 2 | 0 | | 20419 | 2 | 2 | 0 | | 20420 | 2 | 2 | 0 | | 20421 | 2 | 2 | 0 | | 20422 | 2 | 2 | 0 | | 20423 | 2 | 2 | 0 | | 20424 | 2 | 2 | 0 | | 20425 | 2 | 2 | 0 | | 20426 | 2 | 2 | 0 | | 20427 | 2 | 2 | 0 | | 20428 | 2 | 2 | 0 | | 20429 | 2 | 2 | 0 | | 20430 | 2 | 2 | 0 | | 20431 | 2 | 2 | 0 | | 20432 | 2 | 2 | 0 | | 20433 | 2 | 2 | 0 | | 20434 | 2 | 2 | 0 | | 20435 | 2 | 2 | 0 | | 20436 | 2 | 2 | 0 | | 20437 | 2 | 2 | 0 | | 20438 | 2 | 2 | 0 | | 20439 | 2 | 2 | 0 | | 20440 | 2 | 2 | 0 | | 20441 | 2 | 2 | 0 | | 20442 | 2 | 2 | 0 | | 20443 | 2 | 2 | 0 | | 20444 | 2 | 2 | 0 | | 20445 | 2 | 2 | 0 | | 20446 | 2 | 2 | 0 | | 20447 | 2 | 2 | 0 | | 20448 | 2 | 2 | 0 | | 20449 | 2 | 2 | 0 | | 20450 | 2 | 2 | 0 | | 20451 | 2 | 2 | 0 | | 20452 | 2 | 2 | 0 | | 20453 | 2 | 2 | 0 | | 20454 | 2 | 2 | 0 | | 20455 | 2 | 2 | 0 | | 20456 | 2 | 2 | 0 | | 20457 | 2 | 2 | 0 | | 20458 | 2 | 2 | 0 | | 20459 | 2 | 2 | 0 | | 20460 | 2 | 2 | 0 | | 20461 | 2 | 2 | 0 | | 20462 | 2 | 2 | 0 | | 20463 | 2 | 2 | 0 | | 20464 | 2 | 2 | 0 | | 20465 | 2 | 2 | 0 | | 20466 | 2 | 2 | 0 | | 20467 | 2 | 2 | 0 | | 20468 | 2 | 2 | 0 | | 20469 | 2 | 2 | 0 | | 20470 | 2 | 2 | 0 | | 20471 | 2 | 2 | 0 | | 20472 | 2 | 2 | 0 | | 20473 | 2 | 2 | 0 | | 20474 | 2 | 2 | 0 | | 20475 | 2 | 2 | 0 | | 20476 | 2 | 2 | 0 | | 20477 | 2 | 2 | 0 | | 20478 | 2 | 2 | 0 | | 20479 | 2 | 2 | 0 | | 20480 | 2 | 2 | 0 | | 20481 | 2 | 2 | 0 | | 20482 | 2 | 2 | 0 | | 20483 | 2 | 2 | 0 | | 20484 | 2 | 2 | 0 | | 20485 | 2 | 2 | 0 | | 20486 | 2 | 2 | 0 | | 20487 | 2 | 2 | 0 | | 20488 | 2 | 2 | 0 | | 20489 | 2 | 2 | 0 | | 20490 | 2 | 2 | 0 | | 20491 | 2 | 2 | 0 | | 20492 | 2 | 2 | 0 | | 20493 | 2 | 2 | 0 | | 20494 | 2 | 2 | 0 | | 20495 | 2 | 2 | 0 | | 20496 | 2 | 2 | 0 | | 20497 | 2 | 2 | 0 | | 20498 | 2 | 2 | 0 | | 20499 | 2 | 2 | 0 | | 20500 | 2 | 2 | 0 | | 20501 | 2 | 2 | 0 | | 20502 | 2 | 2 | 0 | | 20503 | 2 | 2 | 0 | | 20504 | 2 | 2 | 0 | | 20505 | 2 | 2 | 0 | | 20506 | 2 | 2 | 0 | | 20507 | 2 | 2 | 0 | | 20508 | 2 | 2 | 0 | | 20509 | 2 | 2 | 0 | | 20510 | 2 | 2 | 0 | | 20511 | 2 | 2 | 0 | | 20512 | 2 | 2 | 0 | | 20513 | 2 | 2 | 0 | | 20514 | 2 | 2 | 0 | | 20515 | 2 | 2 | 0 | | 20516 | 2 | 2 | 0 | | 20517 | 2 | 2 | 0 | | 20518 | 2 | 2 | 0 | | 20519 | 2 | 2 | 0 | | 20520 | 2 | 2 | 0 | | 20521 | 2 | 2 | 0 | | 20522 | 2 | 2 | 0 | | 20523 | 2 | 2 | 0 | | 20524 | 2 | 2 | 0 | | 20525 | 2 | 2 | 0 | | 20526 | 2 | 2 | 0 | | 20527 | 2 | 2 | 0 | | 20528 | 2 | 2 | 0 | | 20529 | 2 | 2 | 0 | | 20530 | 2 | 2 | 0 | | 20531 | 2 | 2 | 0 | | 20532 | 2 | 2 | 0 | | 20533 | 2 | 2 | 0 | | 20534 | 2 | 2 | 0 | | 20535 | 2 | 2 | 0 | | 20536 | 2 | 2 | 0 | | 20537 | 2 | 2 | 0 | | 20538 | 2 | 2 | 0 | | 20539 | 2 | 2 | 0 | | 20540 | 2 | 2 | 0 | | 20541 | 2 | 2 | 0 | | 20542 | 2 | 0 | 2 | | 20543 | 2 | 2 | 0 | | 20544 | 2 | 2 | 0 | | 20545 | 2 | 2 | 0 | | 20546 | 2 | 2 | 0 | | 20547 | 2 | 2 | 0 | | 20548 | 2 | 2 | 0 | | 20549 | 2 | 2 | 0 | | 20550 | 2 | 2 | 0 | | 20551 | 2 | 2 | 0 | | 20552 | 2 | 2 | 0 | | 20553 | 2 | 2 | 0 | | 20554 | 2 | 2 | 0 | | 20555 | 2 | 2 | 0 | | 20556 | 2 | 2 | 0 | | 20557 | 2 | 2 | 0 | | 20558 | 2 | 2 | 0 | | 20559 | 2 | 2 | 0 | | 20560 | 2 | 2 | 0 | | 20561 | 2 | 2 | 0 | | 20562 | 2 | 2 | 0 | | 20563 | 2 | 2 | 0 | | 20564 | 2 | 2 | 0 | | 20565 | 2 | 2 | 0 | | 20566 | 2 | 2 | 0 | | 20567 | 2 | 2 | 0 | | 20568 | 2 | 2 | 0 | | 20569 | 2 | 2 | 0 | | 20570 | 2 | 2 | 0 | | 20571 | 2 | 2 | 0 | | 20572 | 2 | 2 | 0 | | 20573 | 2 | 2 | 0 | | 20574 | 2 | 2 | 0 | | 20575 | 2 | 2 | 0 | | 20576 | 2 | 2 | 0 | | 20577 | 2 | 2 | 0 | | 20578 | 2 | 2 | 0 | | 20579 | 2 | 2 | 0 | | 20580 | 2 | 2 | 0 | | 20581 | 2 | 2 | 0 | | 20582 | 2 | 2 | 0 | | 20583 | 2 | 2 | 0 | | 20584 | 2 | 2 | 0 | | 20585 | 2 | 2 | 0 | | 20586 | 2 | 2 | 0 | | 20587 | 2 | 2 | 0 | | 20588 | 2 | 2 | 0 | | 20589 | 2 | 2 | 0 | | 20590 | 2 | 2 | 0 | | 20591 | 2 | 2 | 0 | | 20592 | 2 | 2 | 0 | | 20593 | 2 | 2 | 0 | | 20594 | 2 | 2 | 0 | | 20595 | 2 | 2 | 0 | | 20596 | 2 | 2 | 0 | | 20597 | 2 | 2 | 0 | | 20598 | 2 | 2 | 0 | | 20599 | 2 | 2 | 0 | | 20600 | 2 | 2 | 0 | | 20601 | 2 | 2 | 0 | | 20602 | 2 | 2 | 0 | | 20603 | 2 | 2 | 0 | | 20604 | 2 | 2 | 0 | | 20605 | 2 | 2 | 0 | | 20606 | 2 | 2 | 0 | | 20607 | 2 | 2 | 0 | | 20608 | 2 | 2 | 0 | | 20609 | 2 | 2 | 0 | | 20610 | 2 | 2 | 0 | | 20611 | 2 | 2 | 0 | | 20612 | 2 | 2 | 0 | | 20613 | 2 | 2 | 0 | | 20614 | 2 | 2 | 0 | | 20615 | 2 | 2 | 0 | | 20616 | 2 | 2 | 0 | | 20617 | 2 | 2 | 0 | | 20618 | 2 | 2 | 0 | | 20619 | 2 | 2 | 0 | | 20620 | 2 | 2 | 0 | | 20621 | 2 | 2 | 0 | | 20622 | 2 | 2 | 0 | | 20623 | 2 | 2 | 0 | | 20624 | 2 | 2 | 0 | | 20625 | 2 | 2 | 0 | | 20626 | 2 | 2 | 0 | | 20627 | 2 | 2 | 0 | | 20628 | 2 | 2 | 0 | | 20629 | 2 | 2 | 0 | | 20630 | 2 | 2 | 0 | | 20631 | 2 | 2 | 0 | | 20632 | 2 | 2 | 0 | | 20633 | 2 | 2 | 0 | | 20634 | 2 | 2 | 0 | | 20635 | 2 | 2 | 0 | | 20636 | 2 | 2 | 0 | | 20637 | 2 | 2 | 0 | | 20638 | 2 | 2 | 0 | | 20639 | 2 | 2 | 0 | | 20640 | 2 | 2 | 0 | | 20641 | 2 | 2 | 0 | | 20642 | 2 | 2 | 0 | | 20643 | 2 | 2 | 0 | | 20644 | 2 | 2 | 0 | | 20645 | 2 | 2 | 0 | | 20646 | 2 | 2 | 0 | | 20647 | 2 | 2 | 0 | | 20648 | 2 | 2 | 0 | | 20649 | 2 | 2 | 0 | | 20650 | 2 | 2 | 0 | | 20651 | 2 | 2 | 0 | | 20652 | 2 | 2 | 0 | | 20653 | 2 | 2 | 0 | | 20654 | 2 | 2 | 0 | | 20655 | 2 | 2 | 0 | | 20656 | 2 | 2 | 0 | | 20657 | 2 | 2 | 0 | | 20658 | 2 | 2 | 0 | | 20659 | 2 | 2 | 0 | | 20660 | 2 | 2 | 0 | | 20661 | 2 | 2 | 0 | | 20662 | 2 | 2 | 0 | | 20663 | 2 | 2 | 0 | | 20664 | 2 | 2 | 0 | | 20665 | 2 | 2 | 0 | | 20666 | 2 | 2 | 0 | | 20667 | 2 | 2 | 0 | | 20668 | 2 | 2 | 0 | | 20669 | 2 | 2 | 0 | | 20670 | 2 | 2 | 0 | | 20671 | 2 | 2 | 0 | | 20672 | 2 | 2 | 0 | | 20673 | 2 | 2 | 0 | | 20674 | 2 | 2 | 0 | | 20675 | 2 | 2 | 0 | | 20676 | 2 | 2 | 0 | | 20677 | 2 | 2 | 0 | | 20678 | 2 | 2 | 0 | | 20679 | 2 | 2 | 0 | | 20680 | 2 | 2 | 0 | | 20681 | 2 | 2 | 0 | | 20682 | 2 | 2 | 0 | | 20683 | 2 | 2 | 0 | | 20684 | 2 | 2 | 0 | | 20685 | 2 | 2 | 0 | | 20686 | 2 | 2 | 0 | | 20687 | 2 | 2 | 0 | | 20688 | 2 | 2 | 0 | | 20689 | 2 | 2 | 0 | | 20690 | 2 | 2 | 0 | | 20691 | 2 | 2 | 0 | | 20692 | 2 | 2 | 0 | | 20693 | 2 | 2 | 0 | | 20694 | 2 | 2 | 0 | | 20695 | 2 | 2 | 0 | | 20696 | 2 | 2 | 0 | | 20697 | 2 | 2 | 0 | | 20698 | 2 | 2 | 0 | | 20699 | 2 | 2 | 0 | | 20700 | 2 | 2 | 0 | | 20701 | 2 | 2 | 0 | | 20702 | 2 | 2 | 0 | | 20703 | 2 | 2 | 0 | | 20704 | 2 | 2 | 0 | | 20705 | 2 | 2 | 0 | | 20706 | 2 | 2 | 0 | | 20707 | 2 | 2 | 0 | | 20708 | 2 | 2 | 0 | | 20709 | 2 | 2 | 0 | | 20710 | 2 | 2 | 0 | | 20711 | 2 | 2 | 0 | | 20712 | 2 | 2 | 0 | | 20713 | 2 | 2 | 0 | | 20714 | 2 | 2 | 0 | | 20715 | 2 | 2 | 0 | | 20716 | 2 | 2 | 0 | | 20717 | 2 | 2 | 0 | | 20718 | 2 | 2 | 0 | | 20719 | 2 | 2 | 0 | | 20720 | 2 | 2 | 0 | | 20721 | 2 | 2 | 0 | | 20722 | 2 | 2 | 0 | | 20723 | 2 | 2 | 0 | | 20724 | 2 | 2 | 0 | | 20725 | 2 | 2 | 0 | | 20726 | 2 | 2 | 0 | | 20727 | 2 | 2 | 0 | | 20728 | 2 | 2 | 0 | | 20729 | 2 | 2 | 0 | | 20730 | 2 | 2 | 0 | | 20731 | 2 | 2 | 0 | | 20732 | 2 | 2 | 0 | | 20733 | 2 | 2 | 0 | | 20734 | 2 | 2 | 0 | | 20735 | 2 | 2 | 0 | | 20736 | 2 | 2 | 0 | | 20737 | 2 | 2 | 0 | | 20738 | 2 | 2 | 0 | | 20739 | 2 | 2 | 0 | | 20740 | 2 | 2 | 0 | | 20741 | 2 | 2 | 0 | | 20742 | 2 | 2 | 0 | | 20743 | 2 | 2 | 0 | | 20744 | 2 | 2 | 0 | | 20745 | 2 | 2 | 0 | | 20746 | 2 | 2 | 0 | | 20747 | 2 | 2 | 0 | | 20748 | 2 | 2 | 0 | | 20749 | 2 | 2 | 0 | | 20750 | 2 | 2 | 0 | | 20751 | 2 | 2 | 0 | | 20752 | 2 | 2 | 0 | | 20753 | 2 | 2 | 0 | | 20754 | 2 | 2 | 0 | | 20755 | 2 | 2 | 0 | | 20756 | 2 | 2 | 0 | | 20757 | 2 | 2 | 0 | | 20758 | 2 | 2 | 0 | | 20759 | 2 | 2 | 0 | | 20760 | 2 | 2 | 0 | | 20761 | 2 | 2 | 0 | | 20762 | 2 | 2 | 0 | | 20763 | 2 | 2 | 0 | | 20764 | 2 | 2 | 0 | | 20765 | 2 | 2 | 0 | | 20766 | 2 | 2 | 0 | | 20767 | 2 | 2 | 0 | | 20768 | 2 | 2 | 0 | | 20769 | 2 | 2 | 0 | | 20770 | 2 | 2 | 0 | | 20771 | 2 | 2 | 0 | | 20772 | 2 | 2 | 0 | | 20773 | 2 | 2 | 0 | | 20774 | 2 | 2 | 0 | | 20775 | 2 | 2 | 0 | | 20776 | 2 | 2 | 0 | | 20777 | 2 | 2 | 0 | | 20778 | 2 | 2 | 0 | | 20779 | 2 | 2 | 0 | | 20780 | 2 | 2 | 0 | | 20781 | 2 | 2 | 0 | | 20782 | 2 | 2 | 0 | | 20783 | 2 | 2 | 0 | | 20784 | 2 | 2 | 0 | | 20785 | 2 | 2 | 0 | | 20786 | 2 | 2 | 0 | | 20787 | 2 | 2 | 0 | | 20788 | 2 | 2 | 0 | | 20789 | 2 | 2 | 0 | | 20790 | 2 | 2 | 0 | | 20791 | 2 | 2 | 0 | | 20792 | 2 | 2 | 0 | | 20793 | 2 | 2 | 0 | | 20794 | 2 | 2 | 0 | | 20795 | 2 | 2 | 0 | | 20796 | 2 | 2 | 0 | | 20797 | 2 | 2 | 0 | | 20798 | 2 | 2 | 0 | | 20799 | 2 | 2 | 0 | | 20800 | 2 | 2 | 0 | +------------+------------+--------------------+------------------------+ 800 rows in set (0.00 sec)
9.3.异常的监控点/摄像头情况
--异常的监控点/摄像头情况 mysql> select * from t_hive_to_mysql_for_vehicle where not_working_camera_num > 0; +------------+------------+--------------------+------------------------+ | monitor_id | camera_num | working_camera_num | not_working_camera_num | +------------+------------+--------------------+------------------------+ | 20090 | 2 | 0 | 2 | | 20099 | 2 | 0 | 2 | | 20100 | 2 | 0 | 2 | | 20130 | 2 | 0 | 2 | | 20178 | 2 | 0 | 2 | | 20354 | 2 | 0 | 2 | | 20363 | 2 | 0 | 2 | | 20364 | 2 | 0 | 2 | | 20371 | 2 | 0 | 2 | | 20372 | 2 | 0 | 2 | | 20542 | 2 | 0 | 2 | +------------+------------+--------------------+------------------------+ 11 rows in set (0.00 sec)
这个时候，就可以从Mysql中查询数据，显示在我们的Web UI中。
B.在所有监控点里面，通过车辆最多的10个监控点是什么
--需求: 在所有监控点里面，通过车辆最多的10个监控点是什么？ -- cd /root/vehicle_dir/ vi hive_vehicle_top10.sql drop table IF EXISTS t_top10_monitor; CREATE TABLE t_top10_monitor( monitor_id string , vehicleNum int )ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' LINES TERMINATED BY ' '; from t_vehicle_log insert into table t_top10_monitor select monitor_id, count(vehicle_plate) vehicleNum group by monitor_id order by vehicleNum desc limit 10; drop table IF EXISTS t_top10_monitor_details; --top10监控点的车辆具体信息 CREATE TABLE t_top10_monitor_details( date_time string , vehicle_plate string , vehicle_speed int , road_id string , monitor_id string , camera_id string )ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' LINES TERMINATED BY ' '; from t_vehicle_log t1 insert into table t_top10_monitor_details select t1.date_time,t1.vehicle_plate,t1.vehicle_speed,t1.road_id,t1.monitor_id,t1.camera_id where t1.monitor_id in (select t2.monitor_id from t_top10_monitor t2); :wq cd /root/vehicle_dir/ vi hive_to_mysql_for_top10_monitor export --connect jdbc:mysql://node1:3306/sqoop_db --username root --password '!QAZ2wsx3edc' --table t_hive_to_mysql_for_top10_monitor -m 1 --columns monitor_id,vehicle_num --fields-terminated-by ' ' --export-dir /user/hive/warehouse/t_top10_monitor/ ：wq vi hive_to_mysql_for_top10_details export --connect jdbc:mysql://node1:3306/sqoop_db --username root --password '!QAZ2wsx3edc' --table t_hive_to_mysql_for_top10_details -m 1 --columns date_time,vehicle_plate,vehicle_speed,road_id,monitor_id,camera_id --fields-terminated-by ' ' --export-dir /user/hive/warehouse/t_top10_monitor_details/ ：wq --在mysql里面创建表 mysql -u root -p !QAZ2wsx3edc use sqoop_db; --如果有则删除 DROP TABLE IF EXISTS t_hive_to_mysql_for_top10_monitor; CREATE TABLE t_hive_to_mysql_for_top10_monitor (monitor_id VARCHAR(5), vehicle_num INT); DROP TABLE IF EXISTS t_hive_to_mysql_for_top10_details; CREATE TABLE t_hive_to_mysql_for_top10_details (date_time VARCHAR(20),vehicle_plate VARCHAR(9),vehicle_speed INT,road_id VARCHAR(5),monitor_id VARCHAR(5),camera_id VARCHAR(5)); --编辑自动化执行脚本 cd /root/vehicle_dir/ vi hive_vehicle_top10.sh echo 'begin job.' cd /home/hive/bin echo 'processing........ hive data .......................' ./hive -f /root/vehicle_dir/hive_vehicle_top10.sql echo 'export data into mysql' echo 'processing........ hive to mysql for top10 .......................' sqoop --options-file /root/vehicle_dir/hive_to_mysql_for_top10_monitor echo 'processing........ hive to mysql for top10 details.......................' sqoop --options-file /root/vehicle_dir/hive_to_mysql_for_top10_details echo 'Done.........................' :wq --添加执行权限 chmod +x hive_vehicle_top10.sh --执行 ./hive_vehicle_top10.sh --结果 mysql> select monitor_id,count(vehicle_plate) vehicleNum from t_hive_to_mysql_for_top10_details group by monitor_id order by vehicleNum desc; +------------+------------+ | monitor_id | vehicleNum | +------------+------------+ | 20353 | 6107 | | 20541 | 6063 | | 20177 | 6006 | | 20089 | 5983 | | 20129 | 5858 | | 20147 | 3159 | | 20003 | 3146 | | 20638 | 3119 | | 20659 | 3117 | | 20531 | 3113 | +------------+------------+ 10 rows in set (0.09 sec) mysql> select * from t_hive_to_mysql_for_top10_monitor; +------------+-------------+ | monitor_id | vehicle_num | +------------+-------------+ | 20353 | 6107 | | 20541 | 6063 | | 20177 | 6006 | | 20089 | 5983 | | 20129 | 5858 | | 20147 | 3159 | | 20003 | 3146 | | 20638 | 3119 | | 20659 | 3117 | | 20531 | 3113 | +------------+-------------+ 10 rows in set (0.00 sec)
C.在所有监控点里面，超速（max speed: 250）车辆最多的10个监控点是什么
--需求: 在所有监控点里面，超速（max speed: 250）车辆最多的10个监控点是什么？ cd /root/vehicle_dir/ vi hive_vehicle_top10_speed.sql drop table if exists t_top10_speed_monitor; create table t_top10_speed_monitor( monitor_id string, vehicleNum int )ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' LINES TERMINATED BY ' '; from t_vehicle_log insert into table t_top10_speed_monitor select monitor_id, count(vehicle_plate) vehicleNum where vehicle_speed >= 250 group by monitor_id order by vehicleNum desc limit 10; :wq --编写sqoop配置文件 vi hive_to_mysql_for_top10_speed export --connect jdbc:mysql://node1:3306/sqoop_db --username root --password '!QAZ2wsx3edc' --table t_hive_to_mysql_for_top10_speed -m 1 --columns monitor_id,vehicle_num --fields-terminated-by ' ' --export-dir /user/hive/warehouse/t_top10_speed_monitor/ :wq --在mysql里面创建表 mysql -u root -p !QAZ2wsx3edc use sqoop_db; --如果有则删除 DROP TABLE IF EXISTS t_hive_to_mysql_for_top10_speed; CREATE TABLE t_hive_to_mysql_for_top10_speed (monitor_id VARCHAR(5), vehicle_num INT); --编辑自动化执行脚本 cd /root/vehicle_dir/ vi hive_vehicle_top10_speed.sh echo 'begin job.' cd /home/hive/bin echo 'processing........ hive data .......................' ./hive -f /root/vehicle_dir/hive_vehicle_top10_speed.sql echo 'export data into mysql' echo 'processing........ hive to mysql for top10 .......................' sqoop --options-file /root/vehicle_dir/hive_to_mysql_for_top10_speed echo 'Done.........................' :wq --添加执行权限 chmod +x hive_vehicle_top10_speed.sh --执行 ./hive_vehicle_top10_speed.sh --效果： mysql> select * from t_hive_to_mysql_for_top10_speed; +------------+-------------+ | monitor_id | vehicle_num | +------------+-------------+ | 20353 | 1053 | | 20129 | 1037 | | 20089 | 1035 | | 20541 | 1008 | | 20177 | 1005 | | 20678 | 582 | | 20006 | 573 | | 20304 | 571 | | 20122 | 569 | | 20074 | 567 | +------------+-------------+ 10 rows in set (0.00 sec)
D:使用Hive分桶方式，随机抽取20辆车
--需求: 使用Hive分桶方式，随机抽取20辆车 --Hive 分桶 ---- https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Sampling --分桶的目的在于将同一个目录里面的文件拆分成多个文件 --分桶表时对列值取哈希值的方式，将不同数据放到不同文件中存储。 --对于Hive中给每一个表，分区都可以进一步分桶 --由列的哈希值除以桶的个数来决定每条数据话费在哪个桶中 --场景： --数据抽样 --开启分桶 set hive.enforce.bucketing=true; hive> set hive.enforce.bucketing; hive.enforce.bucketing=false hive> set hive.enforce.bucketing=true; hive> set hive.enforce.bucketing; hive.enforce.bucketing=true --抽样 select * from t_bucket tablesample(bucket 1 out of 4 on columns); --tablesample(bucket 1 out of 4 on columns) --1 : 表示从第1个bucket开始抽取数据 --4 ： 表示必须为该表总的bucket数的倍数或因子 --e.g. bucket总数为32 --tablesample(bucket 2 out of 4) --表示从第2个bucket开始，每隔4个bucket抽取一次，总共抽取32/4=8次 --2,6,10,14,18,22,26,30 --tablesample(bucket 3 out of 8) --表示从第3个bucket开始，每隔8个bucket抽取一次，总共抽取32/8=4次 --3,11,19,27 --tablesample(bucket 3 out of 64) --表示从第3个bucket开始，每隔64个bucket抽取一次，总共抽取32/64=1/2次 --抽取第3个bucket里面的1/2数据即可 --创建表 create table t_user_info( id int, name string, age int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; --test data cd vi hive_test_data_user_info 1,Tom,34 2,John,32 3,Susan,23 4,Make,21 5,Jack,19 :wq --加载数据 load data local inpath '/root/hive_test_data_user_info' into table t_user_info; --创建分桶表 create table t_user_info_bucket( id int, name string, age int ) clustered by (age) into 4 buckets ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; --导入数据 from t_user_info insert into table t_user_info_bucket select id, name, age; --抽样数据 --从第1个bucket开始，每隔2个bucket抽一次，总共抽取4/2=2次 --1,3 select * from t_user_info_bucket tablesample(bucket 1 out of 2 on age); hive> select * from t_user_info_bucket tablesample(bucket 1 out of 2 on age); Total jobs = 1 Launching Job 1 out of 1 Number of reduce tasks is set to 0 since there's no reduce operator Starting Job = job_1547180022884_0010, Tracking URL = http://node1:8088/proxy/application_1547180022884_0010/ Kill Command = /home/hadoop-2.5/bin/hadoop job -kill job_1547180022884_0010 Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0 2019-01-11 01:47:42,857 Stage-1 map = 0%, reduce = 0% 2019-01-11 01:47:52,331 Stage-1 map = 100%, reduce = 0%, Cumulative CPU 4.22 sec MapReduce Total cumulative CPU time: 4 seconds 220 msec Ended Job = job_1547180022884_0010 MapReduce Jobs Launched: Job 0: Map: 1 Cumulative CPU: 4.22 sec HDFS Read: 318 HDFS Write: 19 SUCCESS Total MapReduce CPU Time Spent: 4 seconds 220 msec OK 2 John 32 1 Tom 34 Time taken: 16.995 seconds, Fetched: 2 row(s) --随机抽取sql cd /root/vehicle_dir/ vi hive_random_20_vehicle.sql set hive.enforce.bucketing=true; --1.drop t_vehicle_log_bucket drop table IF EXISTS t_vehicle_log_bucket; --2.create t_vehicle_log_bucket CREATE TABLE t_vehicle_log_bucket( date_time string , vehicle_plate string , vehicle_speed int , road_id string , monitor_id string , camera_id string )clustered by (vehicle_speed) into 40 buckets ROW FORMAT DELIMITED FIELDS TERMINATED BY ' ' LINES TERMINATED BY ' '; --这步操作很花时间，所以我这里只是在500条记录里面抽取 --真是环境中，可以全部导入 from t_vehicle_log insert into table t_vehicle_log_bucket select date_time,vehicle_plate,vehicle_speed,road_id,monitor_id,camera_id limit 500; select * from t_vehicle_log_bucket tablesample(bucket 1 out of 2 on vehicle_speed) limit 20; :wq --编写自动化脚本 cd /root/vehicle_dir/ vi hive_random_20_vehicle.sh echo 'begin job' cd /home/hive/bin ./hive -f /root/vehicle_dir/hive_random_20_vehicle.sql echo 'Done' :wq --添加执行权限 chmod +x hive_random_20_vehicle.sh --执行 ./hive_random_20_vehicle.sh --效果： select * from t_vehicle_log_bucket tablesample(bucket 1 out of 2 on vehicle_speed) limit 20; 16/01/2019 23:15:27 SUN88422R 280 10302 20604 41207 16/01/2019 00:48:19 SDZ49313B 80 10074 20147 40294 16/01/2019 20:11:42 SKS85506P 280 10006 20011 40022 16/01/2019 01:12:19 SUN88422R 120 10303 20606 41211 16/01/2019 05:51:22 SBW11362J 240 10117 20234 40468 16/01/2019 16:27:52 SJG66878X 200 10244 20488 40976 16/01/2019 08:12:52 SFT94124G 40 10210 20420 40839 16/01/2019 14:06:49 SFW82834K 200 10281 20561 41121 16/01/2019 20:43:35 SZH07875C 40 10027 20054 40108 16/01/2019 18:16:17 SCB31874U 200 10195 20390 40780 16/01/2019 09:18:33 SZS23949D 280 10160 20320 40640 16/01/2019 18:09:22 SZH07875C 120 10248 20496 40992 16/01/2019 12:04:22 SHY07731S 160 10048 20095 40190 16/01/2019 04:02:43 STR82103U 240 10359 20717 41433 16/01/2019 23:10:47 SBW11362J 242 10292 20583 41166 16/01/2019 01:10:33 SBW11362J 162 10257 20514 41028 16/01/2019 06:18:09 SJG66878X 242 10216 20432 40864 16/01/2019 10:17:01 SFW14362S 282 10389 20777 41554 16/01/2019 14:12:42 SUN88422R 122 10340 20680 41360 16/01/2019 10:25:30 SUS22167R 122 10190 20379 40758
========================================================
More reading,and english is important.
I'm Hongten
大哥哥大姐姐，觉得有用打赏点哦！你的支持是我最大的动力。谢谢。
Hongten博客排名在100名以内。粉丝过千。
Hongten出品，必是精品。
E | hongtenzone@foxmail.com B | http://www.cnblogs.com/hongten
========================================================