zoukankan      html  css  js  c++  java
  • 创建序列化器,序列化管理器,closureSerializer

    创建序列化器,序列化管理器,closureSerializer

        //通过反射创建序列化对象
        // Create an instance of the class with the given name, possibly initializing it with our conf
        def instantiateClass[T](className: String): T = {
          val cls = Utils.classForName(className)
          // Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just
          // SparkConf, then one taking no arguments
          try {
            cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE)
              .newInstance(conf, new java.lang.Boolean(isDriver))
              .asInstanceOf[T]
          } catch {
            case _: NoSuchMethodException =>
              try {
                cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T]
              } catch {
                case _: NoSuchMethodException =>
                  cls.getConstructor().newInstance().asInstanceOf[T]
              }
          }
        }
    
        // Create an instance of the class named by the given SparkConf property, or defaultClassName
        // if the property is not set, possibly initializing it with our conf
        def instantiateClassFromConf[T](propertyName: String, defaultClassName: String): T = {
          instantiateClass[T](conf.get(propertyName, defaultClassName))
        }
    
        val serializer = instantiateClassFromConf[Serializer](
          "spark.serializer", "org.apache.spark.serializer.JavaSerializer")
        logDebug(s"Using serializer: ${serializer.getClass}")
    
        val serializerManager = new SerializerManager(serializer, conf, ioEncryptionKey)
    
        val closureSerializer = new JavaSerializer(conf)
    View Code

    closureSerializer 

  • 相关阅读:
    索引信息统计
    删除脚本
    归档脚本
    SQL2005四个排名函数(row_number、rank、dense_rank和ntile)的比较
    用js输出同样字符出现的次数
    grid布局
    统计字符出现的次数
    百度地图
    For each...in,For...in ,For...of的Examples和Explanation
    Object的起源
  • 原文地址:https://www.cnblogs.com/chengbao/p/10624917.html
Copyright © 2011-2022 走看看