zoukankan      html  css  js  c++  java
  • spark包

    spark-assembly-1.5.2-hadoop2.6.0.jar

    http://blog.csdn.net/ronaldo4511/article/details/53035494

    http://doc.okbase.net/qindongliang1922/archive/172752.html

    name := "spark-hello"
    
    version := "1.0"
    
    scalaVersion := "2.11.7"
    //使用公司的私服
    resolvers += "Local Maven Repository" at "http://dev.bizbook-inc.com:8083/nexus/content/groups/public/"
    //使用内部仓储
    externalResolvers := Resolver.withDefaultResolvers(resolvers.value, mavenCentral = false)
    //Hadoop的依赖
    libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "2.7.1"
    //Spark的依赖
    libraryDependencies += "org.apache.spark" % "spark-core_2.11" % "1.4.1"
    //Spark SQL 依赖
    libraryDependencies += "org.apache.spark" % "spark-sql_2.11" % "1.4.1"
    //java servlet 依赖
    libraryDependencies += "javax.servlet" % "javax.servlet-api" % "3.0.1"
        


    demo1:使用Scala读取HDFS的数据: 

     /** *
        * Spark读取来自HDFS的数据
        */
    def readDataFromHDFS(): Unit ={
        //以standalone方式运行,提交到远程的spark集群上面
        val conf = new SparkConf().setMaster("spark://h1:7077").setAppName("load hdfs data")
        conf.setJars(Seq(jarPaths));
        //得到一个Sprak上下文
        val sc = new SparkContext(conf)
        val textFile=sc.textFile("hdfs://h1:8020/user/webmaster/crawldb/etl_monitor/part-m-00000")
        //获取第一条数据
        //val data=textFile.first()
       // println(data)
        //遍历打印
          /**

    libraryDependencies+="org.apache.spark"%"spark-sql_2.11"%"2.1.0"%"provided"

    libraryDependencies+="org.apache.spark"%"spark-mllib_2.11"%"2.1.0"%"provided"

    libraryDependencies+="org.apache.spark"%"spark-graphx_2.11"%"2.1.0"%"provided"

    libraryDependencies+="org.apache.spark"%"spark-streaming_2.11"%"2.1.0"%"provided"

    libraryDependencies+="com.github.scopt"%"scopt_2.11"%"3.5.0"

    libraryDependencies+="org.apache.spark"%"spark-streaming-kafka-0-8_2.11"%"2.1.0"

    libraryDependencies+="org.apache.spark"%"spark-streaming-flume_2.11"%"2.1.0"

  • 相关阅读:
    SpringBoot发送邮箱验证码
    判断一个数是否为2的整数次幂
    [模板] 虚树 && bzoj2286-[Sdoi2011]消耗战
    [模板] K-D Tree
    [模板] 平衡树: Splay, 非旋Treap, 替罪羊树
    对于约数个数上界的估计
    luogu3702-[SDOI2017]序列计数
    [模板] 线性基
    [模板] 区间mex && 区间元素种数
    bzoj4367-[IOI2014]holiday假期
  • 原文地址:https://www.cnblogs.com/wcLT/p/6606597.html
Copyright © 2011-2022 走看看