zoukankan      html  css  js  c++  java
  • RDDTest.scala

    /**
     * Created by root on 9/7/15.
     */
    import org.apache.spark.SparkContext
    import org.apache.spark.SparkConf
    
    object RDDTest {
      def main(args: Array[String]) {
        val conf = new SparkConf().setAppName("RDDTest").setMaster("local")
        val sc = new SparkContext(conf)
        val lines = sc.textFile("/home/slh/data/rddtest.txt")
        //count the word
        val lineLengths = lines.map(s => s.length)  //rdd
        val totalLength = lineLengths.reduce((a, b) => a + b)
        println("total length: " + totalLength)
    
        //get the word count
        val word_count = lines.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
        //word_count.saveAsTextFile("/home/slh/data/rddresult0")
    
        //get the sum
        val sum = lines.flatMap(line => line.split(" ")).map(word => (1, word)).reduceByKey((a, b) => a + b)
        //sum.saveAsTextFile("/home/slh/data/rddresult1")
        //the result is (1,3343566777879717727)
        //println("sum: " + sum)
    
        //accumulator
        val accum = sc.accumulator(0, "My Accumulator")
        sc.parallelize(Array(1,2,3,4)).foreach(x => accum += x)
        println("Accumulator of Array(1,2,3,4) : " + accum.value)
      }
    }
  • 相关阅读:
    字符串、列表(操作)
    数据类型
    第二周 第四部分
    第二周 第三部分
    第二周第二部分
    特征缩放和标准化 设置学习率大小 正则方程
    梯度下降 coursera.org
    监督学习和无监督学习
    手写数字问题
    pytorch基础
  • 原文地址:https://www.cnblogs.com/sunflower627/p/4794669.html
Copyright © 2011-2022 走看看