2017 级课堂测试试卷—数据清洗

石家庄铁道大学2019年秋季

  2017 级课堂测试试卷—数据清洗

课程名称： 大型数据库应用技术  任课教师： 王建民   考试时间： 100 分钟

Result文件数据说明：

Ip：106.39.41.166,（城市）Date：10/Nov/2016:00:01:02 +0800,（日期）Day：10,（天数）Traffic: 54 ,（流量）Type: video,（类型：视频video或文章article）

Id: 8701（视频或者文章的id）

测试要求：

1、 数据清洗：按照进行数据清洗，并将清洗后的数据导入hive数据库中。

两阶段数据清洗：

（1）第一阶段：把需要的信息从原始日志中提取出来

ip:    199.30.25.88time:  10/Nov/2016:00:01:03 +0800traffic:  62文章： article/11325视频： video/3235

（2）第二阶段：根据提取出来的信息做精细化操作

ip--->城市 city（IP）date--> time:2016-11-10 00:01:03day: 10traffic:62type:article/videid:11325

（3）hive数据库表结构:

create table data(  ip string,  time string , day string, traffic bigint,type string, id   string )

2、数据处理：

·统计地区最受欢迎的视频/文章的Top10访问次数 （video/article）

·按照地市统计最受欢迎的Top10课程 （ip）

·按照流量统计最受欢迎的Top10课程 （traffic）

3、数据可视化：将统计结果倒入MySql数据库中，通过图形化展示的方式展现出来。

******************************************************************************

说明：

运行环境：MyEclipse(linux外面)

我的理解是有错误的，这些代码都是之前那11个改编的。但确实是能实现简单的清洗。感谢韩代表知道我配置环境*.*

数据清洗：

package mapreduce1;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;  
public class Result_1{  

    static int Sum=0;
    public static class Map extends Mapper<Object , Text , Text,Text>{  
    private static Text Name =new Text();  
    private static Text num=new Text();  
    public void map(Object key,Text value,Context context) throws IOException, InterruptedException{  
    String line=value.toString();  
    String arr[]=line.split(",");  
        Name.set(arr[0]);
        String trm =arr[3].trim();
        num.set(trm);
        System.out.println(num);
    context.write(Name,num);  
    }  
    }  
    public static class Reduce extends Reducer< Text, Text,Text, Text>{  
    int i=0;
    public void reduce(Text key,Iterable<Text> values,Context context) throws IOException, InterruptedException{  
        Text num=new Text();
        for(Text val:values){  
            num=val;
            Sum+=1;
            }  
        String mid=new String();
        mid=String.valueOf(Sum);
        mid=num.toString()+"	"+mid;
        num.set(mid);
        context.write(key,num);
        System.out.println(Sum);
        }  
         }  
    public static int run()throws IOException, ClassNotFoundException, InterruptedException
    {
        Configuration conf=new Configuration();  
        conf.set("fs.defaultFS", "hdfs://192.168.1.100:9000");
        FileSystem fs =FileSystem.get(conf);
        Job job =new Job(conf,"Result_1");  
        job.setJarByClass(Result_1.class);  
        job.setMapperClass(Map.class);  
        job.setReducerClass(Reduce.class);  
        job.setOutputKeyClass(Text.class);  
        job.setOutputValueClass(Text.class);  
        job.setInputFormatClass(TextInputFormat.class);  
        job.setOutputFormatClass(TextOutputFormat.class);  
        Path in=new Path("hdfs://192.168.1.100:9000/mymapreduce1/in/result.txt");  
        Path out=new Path("hdfs://192.168.1.100:9000/mymapreduce1/out_result");  
        FileInputFormat.addInputPath(job,in);  
        fs.delete(out,true);
        FileOutputFormat.setOutputPath(job,out);  
        return(job.waitForCompletion(true) ? 0 : 1);  
    }
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{  
    
            run();
        }  
        }  

结果：

简单排序
  1 package mapreduce1;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class Result_2 {

    public static List<String> Names=new ArrayList<String>();
    public static  List<String> Values=new ArrayList<String>();
    public static  List<String> Texts=new ArrayList<String>();
    public static class Sort extends WritableComparator {
        public Sort(){
        //这里就是看你map中填的输出key是什么数据类型，就给什么类型
        super(IntWritable.class,true);
        }
        @Override
        public int compare(WritableComparable a, WritableComparable b) {
        return -a.compareTo(b);//加个负号就是倒序，把负号去掉就是正序。
        }
        }
    public static class Map extends Mapper<Object , Text , IntWritable,Text >{  
    private static Text Name=new Text();
    private static IntWritable num=new IntWritable();
    public void map(Object key,Text value,Context context)throws IOException, InterruptedException
    {
         String line=value.toString();  
         String mid=new String();
            String arr[]=line.split("	");  
            if(!arr[0].startsWith(" "))
            {
                  num.set(Integer.parseInt(arr[2]));  
                  mid=arr[0]+"	"+arr[1];
                  Name.set(mid);
                  context.write(num, Name);
            }
          
    }
    }
    public static class Reduce extends Reducer< IntWritable, Text, Text, IntWritable>{  
        private static IntWritable result= new IntWritable();  
        int i=0;
        
         public void reduce(IntWritable key,Iterable<Text> values,Context context) throws IOException, InterruptedException{  
                for(Text val:values){  
                    
                    if(i<10)
                    {i=i+1;
                    String mid=new String();
                    mid=val.toString();
                    String arr[]=mid.split("	");
                    Texts.add(arr[1]);
                        Names.add(arr[0]);
                        Values.add(key.toString());
                    }
                context.write(val,key);  
                }  
    }
    }

  
    
 
    
    public static int run()throws IOException, ClassNotFoundException, InterruptedException{
        Configuration conf=new Configuration();  
        conf.set("fs.defaultFS", "hdfs://192.168.1.100:9000");
        FileSystem fs =FileSystem.get(conf);
        Job job =new Job(conf,"Result_2");  
        job.setJarByClass(Result_2.class);  
        job.setMapperClass(Map.class);  
        job.setReducerClass(Reduce.class);  
        job.setSortComparatorClass(Sort.class);
        job.setOutputKeyClass(IntWritable.class);  
        job.setOutputValueClass(Text.class);  
        job.setInputFormatClass(TextInputFormat.class);  
        job.setOutputFormatClass(TextOutputFormat.class);  
        Path in=new Path("hdfs://192.168.1.100:9000/mymapreduce1/out_result/part-r-00000");  
        Path out=new Path("hdfs://192.168.1.100:9000/mymapreduce1/out_result1");  
        FileInputFormat.addInputPath(job,in);  
        fs.delete(out,true);
        FileOutputFormat.setOutputPath(job,out);  
       return(job.waitForCompletion(true) ? 0 : 1);  
        
       
        }
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{  
          run();
           for(String n:Names)
            {
                System.out.println(n);
               }
          } 
}

结果：