[Java] 数据分析--数据预处理

数据结构

• 键-值对：HashMap

import java.io.File;
import java.io.FileNotFoundException;
import java.util.HashMap;
import java.util.Scanner;

public class HashMapExample {
    public static void main(String[] args) {
        File dataFile = new File("data/Countries.dat");
        HashMap<String,Integer> dataset = new HashMap();
        try {
            Scanner input = new Scanner(dataFile);
            while (input.hasNext()) {
                String country = input.next();
                int population = input.nextInt();
                dataset.put(country, population);
            }
        } catch (FileNotFoundException e) {
            System.out.println(e);
        }
        System.out.printf("dataset.size(): %d%n", dataset.size());
        System.out.printf("dataset.get("Peru"): %,d%n", dataset.get("Peru"));
    }
}

文件处理

• csv文件
  • 将Map数据存入csv文件　　

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;

public class FromMapToExcel {
    public static void main(String[] args) {
        Map<String,Integer> map = new TreeMap();
        load(map, "data/Countries.dat");
        print(map);
        storeXL(map, "data/Countries.xls", "Countries Worksheet");
    }
    
    /** Loads the data from the specified file into the specified map.
    */
    public static void load(Map map, String fileSpec) {
        File file = new File(fileSpec);
        try {
            Scanner input = new Scanner(file);
            while (input.hasNext()) {
                String country = input.next();
                int population = input.nextInt();
                map.put(country, population);
            }
        } catch (FileNotFoundException e) {
            System.out.println(e);
        }
    }
    
    public static void print(Map map) {
        Set countries = map.keySet();
        for (Object country : countries) {
            Object population = map.get(country);
            System.out.printf("%-10s%,12d%n", country, population);
        }
    }
    
    /** Stores the specified map in the specified worksheet of 
        the specified Excel workbook file.
     * @param map
     * @param fileSpec
     * @param sheet
    */
    public static void storeXL(Map map, String fileSpec, String sheet) {
        try {
            FileOutputStream out = new FileOutputStream(fileSpec);
            HSSFWorkbook workbook = new HSSFWorkbook();
            HSSFSheet worksheet = workbook.createSheet(sheet);
            Set countries = map.keySet();
            short rowNum = 0;
            for (Object country : countries) {
                Object population = map.get(country);
                HSSFRow row = worksheet.createRow(rowNum);
                row.createCell(0).setCellValue((String)country);
                row.createCell(1).setCellValue((Integer)population);
                ++rowNum;
            }
            workbook.write(out);
            out.flush();
            out.close();
        } catch (FileNotFoundException e) {
            System.err.println(e);
        } catch (IOException e) {
            System.err.println(e);
        }
    }
}

• • 读取csv文件

import java.io.File;
import java.io.FileNotFoundException;
import java.util.HashMap;
import java.util.Scanner;

public class ReadingCSVFiles {
    public static void main(String[] args) {
        File dataFile = new File("data/Countries.csv");
        try {
            Scanner input = new Scanner(dataFile);
            input.useDelimiter(",|\s");
            String column1 = input.next();
            String column2 = input.next();
            System.out.printf("%-10s%12s%n", column1, column2);
            while (input.hasNext()) {
                String country = input.next();
                int population = input.nextInt();
                System.out.printf("%-10s%,12d%n", country, population);
            }
        } catch (FileNotFoundException e) {
            System.out.println(e);
        }
    }
}

• • 读取csv到Map

import static dawj.ch02.FromMapToExcel.print;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Row;

public class FromExcelToMap {
    public static void main(String[] args) {
        Map map = loadXL("data/Countries.xls", "Countries Worksheet");
        print(map);
    }
    
    /** Returns a Map object containing the data from the specified 
        worksheet in the specified Excel file.
    */
    public static Map loadXL(String fileSpec, String sheetName) {
        Map<String,Integer> map = new TreeMap();
        try {
            FileInputStream stream = new FileInputStream(fileSpec);
            HSSFWorkbook workbook = new HSSFWorkbook(stream);
            HSSFSheet worksheet = workbook.getSheet(sheetName);
            DataFormatter formatter = new DataFormatter();
            for (Row row : worksheet) {
                HSSFRow hssfRow = (HSSFRow)row;
                HSSFCell cell = hssfRow.getCell(0);
                String country = cell.getStringCellValue();
                cell = hssfRow.getCell(1);
                String str = formatter.formatCellValue(cell);
                int population = (int)Integer.getInteger(str);
                map.put(country, population);
            }
        } catch (FileNotFoundException e) {
            System.err.println(e);
        } catch (IOException e) {
            System.err.println(e);
        }
        return map;
    }
}

• 解析JSON文件

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import javax.json.Json;
import javax.json.stream.JsonParser;
import javax.json.stream.JsonParser.Event;

public class ParsingJSONFiles {
    public static void main(String[] args) {
        File dataFile = new File("data/Books.json");
        try {
            InputStream stream = new FileInputStream(dataFile);
            JsonParser parser = Json.createParser(stream);
            Event event = parser.next();  // advance past START_OBJECT
            HashMap<String,Object> map = getMap(parser);
            System.out.println(map);
            stream.close();
        } catch (FileNotFoundException e) {
            System.out.println(e);
        } catch (IOException e) {
            System.out.println(e);
        }
    }
    
    /*  Returns the HashMap parsed by the specified parser.
        Called when event.equals(event.START_OBJECT):
    */
    public static HashMap getMap(JsonParser parser) {
        HashMap<String,Object> map = new HashMap();
        Event event = parser.next();  // advance past START_OBJECT
        String key = parser.getString();
        event = parser.next();       // advance past KEY_NAME
        while (!event.equals(Event.END_OBJECT)) {
            if (event.equals(Event.VALUE_STRING)) {
                String value = parser.getString();
                map.put(key, value);
            } else if (event.equals(Event.VALUE_NUMBER)) {
                Integer value = parser.getInt();
                map.put(key, value);
            } else if (event.equals(Event.START_ARRAY)) {
                ArrayList<String> list = getList(parser);
                map.put(key, list);
            }
            event = parser.next();
            if (event.equals(Event.END_OBJECT)) {
                break;
            }
            key = parser.getString();
            event = parser.next();
        }
        return map;
    }
    
    /*  Returns the ArrayList parsed by the specified parser.
        Called when event.equals(event.START_ARRAY):
    */
    public static ArrayList getList(JsonParser parser) {
        ArrayList list = new ArrayList();
        Event event = parser.next();  // advance past START_ARRAY
        while (!event.equals(Event.END_ARRAY)) {
            if (event.equals(Event.VALUE_STRING)) {
                list.add(parser.getString());
                event = parser.next();
            } else if (event.equals(Event.START_OBJECT)) {
                HashMap<String,Object> map = getMap(parser);
                list.add(map);
                event = parser.next();
            } else if (event.equals(Event.START_ARRAY)) {
                ArrayList subList = getList(parser);   //  recursion
                list.add(subList);
                event = parser.next();
            }
        }
        return list;
    }
}

数据处理

• 生成测试数据集

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.util.Random;

public class GeneratingTestData {
    private static final int ROWS = 8, COLS = 5;
    private static final Random RANDOM = new Random();
    
    public static void main(String[] args) {
        File outputFile = new File("data/Output.csv");
        try {
            PrintWriter writer = new PrintWriter(outputFile);
            for (int i = 0; i < ROWS; i++) {
                for (int j = 0; j < COLS-1; j++) {
                    writer.printf("%.6f,", RANDOM.nextDouble());
                }
                writer.printf("%.6f%n", RANDOM.nextDouble());
            }
            writer.close();
        } catch (FileNotFoundException e) {
            System.err.println(e);
        }
    }
}

• 数据过滤
  • 需求：选择国土面积超过100万平米的内陆国家
  • 过程：数据为dat格式，先定义对应简单类country，再写程序将dat中数据存在country的Set中，最后做筛选

Country.java

import java.util.HashSet;
import java.util.Scanner;

class Country {
    protected String name;
    protected int population;
    protected int area;
    protected boolean landlocked;

    /*  Constructs a new Country object from the next line being scanned.
        If there are no more lines, the new object's fields are left null.
    */
    public Country(Scanner in) {
        if (in.hasNextLine()) {
            this.name = in.next();
            this.population = in.nextInt();
            this.area = in.nextInt();
            this.landlocked = in.nextBoolean();
        }
    }

    @Override
    public String toString() {
        return String.format("%-10s %,12d %,12d %b", 
                name, population, area, landlocked);
    }
}

FilteringData.java

import java.io.File;
import java.io.FileNotFoundException;
import java.util.HashSet;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;

public class FilteringData {
    private static final int MIN_AREA = 1000000;  // one million 
    public static void main(String[] args) {
        File file = new File("data/Countries.dat");
        Set<Country> dataset = readDataset(file);
        
        for (Country country : dataset) {
            if (country.landlocked && country.area >= MIN_AREA) {
                System.out.println(country);
            }
        }
    }
    
    public static Set readDataset(File file) {
        Set<Country> set = new HashSet();
        try {
            Scanner input = new Scanner(file);
            input.nextLine();  // read past headers
            while (input.hasNextLine()) {
                set.add(new Country(input));
            }
            input.close();
        } catch (FileNotFoundException e) {
            System.out.println(e);
        }
        return set;
    }
}

• 排序
  • 需求：将contries.dat中数据按population进行排序
  • 实现：将数据存入TreeMap
  • 注意：关键字段必须唯一，即两个国家人口不能相同

import java.io.File;
import java.io.FileNotFoundException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;

public class SortingData {
    public static void main(String[] args) {
        File file = new File("src/main/java/com/hongfeng/SortingData/Countries.dat");
        TreeMap<Integer,String> dataset = new TreeMap();
        try {
            Scanner input = new Scanner(file);
            while (input.hasNext()) {
                String x = input.next();
                int y = input.nextInt();
                dataset.put(y, x);
            }
            input.close();
        } catch (FileNotFoundException e) {
            System.out.println(e);
        }
        print(dataset);
    }
    
    public static void print(TreeMap<Integer,String> map) {
        for (Integer key : map.keySet()) {
            System.out.printf("%,12d  %-16s%n", key, map.get(key));
        }
    }
}

• 合并
  • 需求：将多个排好序的文件合并为单个排好序的文件
  • country类继承Comparable，定义从文件创建对象的构造方法，以及比较方法
  • 扫描两个文件，比较，存入新文件，一个文件扫描完后，另一个文件逐项扫描即可

Country.java

import java.util.Scanner;

class Country implements Comparable{
    protected String name;
    protected int population;

    /*  Constructs a new Country object from the next line being scanned.
        If there are no more lines, the new object's fields are left null.
    */
    public Country(Scanner in) {
        if (in.hasNextLine()) {
            this.name = in.next();
            this.population = in.nextInt();
        }
    }

    public boolean isNull(){
        return this.name == null;
    }

    @Override
    public int compareTo(Object object){
        Country that = (Country)object;
        return this.population - that.population;
    }

    @Override
    public String toString() {
        return String.format("%-10s %,12d",
                name, population);
    }
}

MergingFiles

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.util.Scanner;

public class MergingFiles {
    public static void main(String[] args) {
        File inFile1 = new File("data/Countries1.dat");
        File inFile2 = new File("data/Countries2.dat");
        File outFile = new File("data/Countries.dat");
        try {
            Scanner in1 = new Scanner(inFile1);
            Scanner in2 = new Scanner(inFile2);
            PrintWriter out = new PrintWriter(outFile);
            Country country1 = new Country(in1);
            Country country2 = new Country(in2);  
            System.out.println(country1.hashCode());
            System.out.println(country2.hashCode());
            while (!country1.isNull() && !country2.isNull()) {
                if (country1.compareTo(country2) < 0) {
                    out.println(country1);
                    country1 = new Country(in1);
                } else {
                    out.println(country2);
                    country2 = new Country(in2);
                }
            }
            while (!country1.isNull()) {
                out.println(country1);
                country1 = new Country(in1);
            }
            while (!country2.isNull()) {
                out.println(country2);
                country2 = new Country(in2);
            }
            in1.close();
            in2.close();
            out.close();
        } catch (FileNotFoundException e) {
            System.out.println(e);
        }
    }
}