zoukankan      html  css  js  c++  java
  • 创建序列化器,序列化管理器,closureSerializer

    创建序列化器,序列化管理器,closureSerializer

        //通过反射创建序列化对象
        // Create an instance of the class with the given name, possibly initializing it with our conf
        def instantiateClass[T](className: String): T = {
          val cls = Utils.classForName(className)
          // Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just
          // SparkConf, then one taking no arguments
          try {
            cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE)
              .newInstance(conf, new java.lang.Boolean(isDriver))
              .asInstanceOf[T]
          } catch {
            case _: NoSuchMethodException =>
              try {
                cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T]
              } catch {
                case _: NoSuchMethodException =>
                  cls.getConstructor().newInstance().asInstanceOf[T]
              }
          }
        }
    
        // Create an instance of the class named by the given SparkConf property, or defaultClassName
        // if the property is not set, possibly initializing it with our conf
        def instantiateClassFromConf[T](propertyName: String, defaultClassName: String): T = {
          instantiateClass[T](conf.get(propertyName, defaultClassName))
        }
    
        val serializer = instantiateClassFromConf[Serializer](
          "spark.serializer", "org.apache.spark.serializer.JavaSerializer")
        logDebug(s"Using serializer: ${serializer.getClass}")
    
        val serializerManager = new SerializerManager(serializer, conf, ioEncryptionKey)
    
        val closureSerializer = new JavaSerializer(conf)
    View Code

    closureSerializer 

  • 相关阅读:
    Hive 2.1.1安装配置
    vi / vim 删除以及其它命令
    『MySQL』时间戳转换
    update 中实现子查询
    hive lateral view 与 explode详解
    xpath定位方法详解
    Python int与string之间的转化
    ORM、SQLAchemy
    python bottle 简介
    python wsgi 简介
  • 原文地址:https://www.cnblogs.com/chengbao/p/10624917.html
Copyright © 2011-2022 走看看