zoukankan      html  css  js  c++  java
  • java 链接mysql 产生500W数据模拟生成环境

    java 插入数据到mysql 通过sqoop 导入到hive 中,kylin模拟见cube 时间和 数据膨胀率 kylin 数据插入到 HBase

    Kylin

    HBase 1.1.3

    Hive 1.2.1

    Hadoop 2.5.1


    create table infoagetime(
    prod_name char(10),
    prod_id SMALLINT,
    ods_date DATE
    )

    数据格式

    oPmgBZxldW    75    2016-09-04
    WSSAnnZrNy    57    2016-09-09


    本人Java水平有限,大家自行改善,尤其是批量插入mysql语句


    另工程中需要导入 mysql jar :   mysql-connector-java-5.1.6.jar

    V2

    package com.wubaiwan.instmysql;
    
    import java.io.File;
    import java.io.FileOutputStream;
    import java.sql.DriverManager;
    import java.util.Random;
    
    public class YiBaiWan {
    
    	public static String getRandomString(int length) { // length表示生成字符串的长度
    		String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
    		Random random = new Random();
    		StringBuffer sb = new StringBuffer();
    		for (int i = 0; i < length; i++) {
    			int number = random.nextInt(base.length());
    			sb.append(base.charAt(number));
    		}
    		return sb.toString();
    	}
    
    	public static final String url = "jdbc:mysql://192.168.184.168/hive2";
    	public static final String name = "com.mysql.jdbc.Driver";
    	public static final String user = "root";
    	public static final String password = "123";
    
    	public static java.sql.Connection conn = null;
    	public static java.sql.PreparedStatement pst = null;
    	static FileOutputStream out = null;
    
    	public static void main(String args[]) {
    
    		try {
    			Class.forName(name);
    			conn = DriverManager.getConnection(url, user, password);// 获取连接
    			conn.setAutoCommit(false);
    		} catch (Exception e1) {
    			// TODO Auto-generated catch block
    			e1.printStackTrace();
    		} // 指定连接类型
    
    		
    
    		Random random = new Random();
    		// int k = random.nextInt();
    		// System.out.println(k);
    		int x = 0;
    		try {
    			out = new FileOutputStream(new File("D:/mysql.txt"));
    			String sql = "insert into infoagetime(prod_name,prod_id,ods_date) values(?,?,?)";
    			// System.out.println(sql);
    			pst = conn.prepareStatement(sql);
    
    			while (x < 5000000) {
    				// System.out.println((int)(Math.random()*100));
    				// System.out.println(getRandomString(10) + ',' +
    				// (int)(Math.random()*100));
    				// String k = getRandomString(10) + ',' +
    				// (int)(Math.random()*100 )+ ','+ "2016-09-0" +
    				// (int)(random.nextInt(9)%9 + 1)+"
    " ;
    				// System.out.println(k);
    				// out.write(k.getBytes());
    
    				pst.setString(1, getRandomString(10));
    				pst.setInt(2, (int) (Math.random() * 100));
    				pst.setString(3, "2016-09-0" + (int) (random.nextInt(9) % 9 + 1));
    
    				pst.addBatch();
    
    				if (x % 1000 == 0) {
    					pst.executeBatch();
    					//pst.executeUpdate(arg0)
    				}
    
    				x++;
    			}
    			System.out.println("Commit");
    			pst.executeBatch();
    			conn.commit();
    			out.close();
    			conn.close();
    			pst.close();
    			
    		} catch (Exception e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		}finally{
    			
    			
    		}
    		
    		
    
    	}
    
    }



    ======================================

    V1  这个版本会有java memory 异常

    package com.wubaiwan.instmysql;
    
    import java.io.File;  
    import java.io.FileOutputStream;
    import java.sql.DriverManager;
    import java.util.Random;
     
    public class YiBaiWan {  
          
          
        public static String getRandomString(int length) { //length表示生成字符串的长度      
               String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";         
               Random random = new Random();         
               StringBuffer sb = new StringBuffer();         
               for (int i = 0; i < length; i++) {         
                   int number = random.nextInt(base.length());         
                   sb.append(base.charAt(number));         
               }         
               return sb.toString();         
            }    
          
        public static final String url = "jdbc:mysql://192.168.184.168/hive2";    
        public static final String name = "com.mysql.jdbc.Driver";    
        public static final String user = "root";    
        public static final String password = "123";    
          
        public static java.sql.Connection conn = null;    
        public static java.sql.PreparedStatement pst = null;    
          
        public static void main(String args[]){  
              
            try {
    			Class.forName(name);
    			conn = DriverManager.getConnection(url, user, password);//获取连接 
    		} catch (Exception e1) {
    			// TODO Auto-generated catch block
    			e1.printStackTrace();
    		}//指定连接类型   
               
            FileOutputStream out = null;     
              
            Random random = new Random();  
            //int k = random.nextInt();  
            //System.out.println(k);  
            int x = 0;  
            try {  
                out = new FileOutputStream(new File("D:/mysql.txt"));  
              
              
            while(x <5000000){  
                //System.out.println((int)(Math.random()*100));  
                //System.out.println(getRandomString(10) + ',' + (int)(Math.random()*100));  
                //String k = getRandomString(10) + ',' + (int)(Math.random()*100 )+ ','+ "2016-09-0" + (int)(random.nextInt(9)%9 + 1)+"
    " ;  
                //System.out.println(k);  
                //out.write(k.getBytes());  
                String sql = "insert into infoagetime(prod_name,prod_id,ods_date) values('"+getRandomString(10)+ "',"+(int)(Math.random()*100 )+","+ "'2016-09-0" + (int)(random.nextInt(9)%9 + 1) + "')"   ;
                //System.out.println(sql);
                pst = conn.prepareStatement(sql);  
                pst.executeUpdate();  
                if(x%10000 == 0 ) {
                    System.out.println("Commit");
                    conn.commit();
                }
                x++;  
                }   
            out.close();    
            conn.close();   
            pst.close();  
            }catch (Exception e) {  
                    // TODO Auto-generated catch block  
                    e.printStackTrace();  
                }  
              
               
      
        }  
      
    }  


  • 相关阅读:
    Spring Cloud Hystrix Dashboard的使用 5.1.3
    Spring Cloud Hystrix 服务容错保护 5.1
    Spring Cloud Ribbon 客户端负载均衡 4.3
    Spring Cloud 如何实现服务间的调用 4.2.3
    hadoop3.1集成yarn ha
    hadoop3.1 hdfs的api使用
    hadoop3.1 ha高可用部署
    hadoop3.1 分布式集群部署
    hadoop3.1伪分布式部署
    KVM(八)使用 libvirt 迁移 QEMU/KVM 虚机和 Nova 虚机
  • 原文地址:https://www.cnblogs.com/TendToBigData/p/10501376.html
Copyright © 2011-2022 走看看