zoukankan      html  css  js  c++  java
  • 【Spark】帮你搞明白怎么通过SparkSQL整合Hive


    一、创建maven工程,导包

    <properties>
        <scala.version>2.11.8</scala.version>
        <spark.version>2.2.0</spark.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>2.2.0</version>
        </dependency>
    </dependencies>
    <build>
        <sourceDirectory>src/main/scala</sourceDirectory>
        <testSourceDirectory>src/test/scala</testSourceDirectory>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.0</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                    <!--    <verbal>true</verbal>-->
                </configuration>
            </plugin>
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.0</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                        <configuration>
                            <args>
                                <arg>-dependencyfile</arg>
                                <arg>${project.build.directory}/.scala_dependencies</arg>
                            </args>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.1.1</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass></mainClass>
                                </transformer>
                            </transformers>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
    

    二、开发代码

    package cn.itcast.sparksql.demo1
    
    import org.apache.spark.SparkContext
    import org.apache.spark.sql.SparkSession
    
    object SparkSQLWithHive {
      def main(args: Array[String]): Unit = {
        // 获取SparkSession
        val sparkSession: SparkSession = SparkSession.builder().appName("sparksql_with_hive").master("local[2]").config("spark.driver.host", "localhost").enableHiveSupport().getOrCreate()
    
        // 获取SparkContext  这主要是为了筛选日志,是可选项,可以不创建
        val sparkContext: SparkContext = sparkSession.sparkContext
        sparkContext.setLogLevel("WARN")
    
        // 直接创建
        sparkSession.sql("create table if not exists student(id Int,name String,age Int) row format delimited fields terminated by ','")
        // 导入数据
        sparkSession.sql("load data local inpath './datas/student.csv' overwrite into table student")
        // 查询数据
        sparkSession.sql("select * from student").show()
    
        sparkContext.stop()
        sparkSession.close()
      }
    
    }
    
    控制台结果
    
    +---+--------+---+
    | id|    name|age|
    +---+--------+---+
    |  1|zhangsan| 18|
    |  2|    lisi| 28|
    |  3|      王五| 50|
    +---+--------+---+
    
    
    Process finished with exit code 0
    
  • 相关阅读:
    AtCoder Beginner Contest 167
    AtCoder Beginner Contest 166
    AtCoder Beginner Contest 165
    AtCoder Beginner Contest 164
    AtCoder Beginner Contest 163
    AtCoder Beginner Contest 162
    AtCoder Beginner Contest 161
    AtCoder Beginner Contest 160
    AtCoder Beginner Contest 159
    自定义Mybatis自动生成代码规则
  • 原文地址:https://www.cnblogs.com/zzzsw0412/p/12772386.html
Copyright © 2011-2022 走看看