sparksql dataset

java

/** 
 *2.0之后使用sparksession即可,不需要再去创建sqlcontext
 *@author Tele
 *
 */
public class Demo {
    private static SparkConf conf = new SparkConf().setAppName("dataframedemo").setMaster("local");
    private static JavaSparkContext jsc = new JavaSparkContext(conf);
    
    private static SparkSession session = new SparkSession(jsc.sc());
    
    //创建sparksession(可以不创建jsc)
//    private static SparkSession session = SparkSession.builder().appName("asd").master("local").getOrCreate();
    
    
//    private static SQLContext sqlContext = new SQLContext(session);
    
    
    public static void main(String[] args) {
    
        //在java中返回是dataset,在scala中返回的是dataframe
    //    sqlContext.read().json("xx")
        
        Dataset<Row> dataset = session.read().json("./src/main/java/cn/tele/spark_sql/dataframe/students.json");
        //输出全部数据
        dataset.show();
        
        
        //输出某一列
        dataset.select("id").show();
        dataset.select(dataset.col("id")).show();
        
        
        //判断id>2
        dataset.select(dataset.col("id").gt(2)).show();
        
        //判断id>=2
        dataset.select(dataset.col("id").geq(2)).show();
        
        //将age += 100
        dataset.select(dataset.col("age").plus(100)).show();
        
        //输出元数据
        dataset.printSchema();
        
        
        jsc.close();
        
        
    }
}

scala

object Demo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("demo").setMaster("local")
    val sc = new SparkContext(conf)
    
    val sqlcontext = new SQLContext(sc)
    
    
    val dataframe = sqlcontext.read.json("./src/main/scala/cn/tele/spark_sql/dataframe/students.json")
    
    
    dataframe.select(dataframe.col("id")).show()
    
    
    dataframe.select(dataframe.col("id").plus(1)).show()
    
    dataframe.select(dataframe.col("id") + 1).show()
    
    dataframe.select("id","name").show()
    
    dataframe.select(dataframe.col("id").geq("2")).show()
    
    dataframe.printSchema()
    
  }
}