Spark中DataFrame去除NaN、null以及空字符串数据

去除null、NaN:

//创建sparkSession(打包在集群上运行要删除master) val sparkConf: SparkConf = new SparkConf().setAppName(this.getClass.getSimpleName.filter(!_.equals('$'))) //获取sparkContext val sparkContext = new SparkContext(sparkConf) //设置日志级别 sparkContext.setLogLevel("WARN") //获取sqlContext val spark: SQLContext = new SQLContext(sparkContext) //读取数据 val data: DataFrame = spark.read.format("jdbc") .option("url", "jdbc:mysql://10.213.111.XXX:23306/buf_amr_all") .option("dbtable", "pdwqy_pms_yx_sbxx") .option("user", "qjjc") .option("password", "XXX") .load() //去除null和NaN data.na.drop().show()

去掉空字符串:

//创建sparkSession(打包在集群上运行要删除master) val sparkConf: SparkConf = new SparkConf().setAppName(this.getClass.getSimpleName.filter(!_.equals('$'))) //获取sparkContext val sparkContext = new SparkContext(sparkConf) //设置日志级别 sparkContext.setLogLevel("WARN") //获取sqlContext val spark: SQLContext = new SQLContext(sparkContext) //读取数据 val data: DataFrame = spark.read.format("jdbc") .option("url", "jdbc:mysql://10.213.111.XXX:23306/buf_amr_all") .option("dbtable", "pdwqy_pms_yx_sbxx") .option("user", "qjjc") .option("password", "XXX") .load() //去掉空字符串 data.where("sentence <> ''").show()