zoukankan      html  css  js  c++  java
  • dataframe去除null、NaN和空字符串

    去除null、NaN

    去除 dataframe 中的 nullNaN 有方法 drop ,用 dataframe.na 找出带有 nullNaN 的行,用 drop 删除行:

    import org.apache.spark.{SparkConf, SparkContext}
    import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}
    /**
      * Created by TTyb on 2017/10/12.
      */
    object test3 {
      def main(args: Array[String]): Unit = {
        val conf = new SparkConf().setAppName("TTyb").setMaster("local")
        val sc = new SparkContext(conf)
        val spark=new SQLContext(sc)
        val sentenceDataFrame = spark.createDataFrame(Seq(
          (1, "asf"),
          (2, "2143"),
          (3, "rfds"),
          (4, null),
          (5, "")
        )).toDF("label", "sentence")
        sentenceDataFrame.show()
        sentenceDataFrame.na.drop().show()
      }
    }
    

    去除空字符串

    去除空字符串用 dataframe.where

    import org.apache.spark.{SparkConf, SparkContext}
    import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}
    /**
      * Created by TTyb on 2017/10/12.
      */
    object test3 {
      def main(args: Array[String]): Unit = {
        val conf = new SparkConf().setAppName("TTyb").setMaster("local")
        val sc = new SparkContext(conf)
        val spark=new SQLContext(sc)
        val sentenceDataFrame = spark.createDataFrame(Seq(
          (1, "asf"),
          (2, "2143"),
          (3, "rfds"),
          (4, null),
          (5, "")
        )).toDF("label", "sentence")
        sentenceDataFrame.show()
        // sentenceDataFrame.na.drop().show()
        sentenceDataFrame.where("sentence <> ''").show()
      }
    }
    
  • 相关阅读:
    九度-题目1197:奇偶校验
    九度-题目1073:杨辉三角形
    九度-题目1072:有多少不同的面值组合?
    同步异步,阻塞非阻塞
    注解方式配置bean
    监听器
    自定义系统初始化器
    构建流
    数值流
    流的使用
  • 原文地址:https://www.cnblogs.com/TTyb/p/7655129.html
Copyright © 2011-2022 走看看