zoukankan      html  css  js  c++  java
  • 创建序列化器,序列化管理器,closureSerializer

    创建序列化器,序列化管理器,closureSerializer

        //通过反射创建序列化对象
        // Create an instance of the class with the given name, possibly initializing it with our conf
        def instantiateClass[T](className: String): T = {
          val cls = Utils.classForName(className)
          // Look for a constructor taking a SparkConf and a boolean isDriver, then one taking just
          // SparkConf, then one taking no arguments
          try {
            cls.getConstructor(classOf[SparkConf], java.lang.Boolean.TYPE)
              .newInstance(conf, new java.lang.Boolean(isDriver))
              .asInstanceOf[T]
          } catch {
            case _: NoSuchMethodException =>
              try {
                cls.getConstructor(classOf[SparkConf]).newInstance(conf).asInstanceOf[T]
              } catch {
                case _: NoSuchMethodException =>
                  cls.getConstructor().newInstance().asInstanceOf[T]
              }
          }
        }
    
        // Create an instance of the class named by the given SparkConf property, or defaultClassName
        // if the property is not set, possibly initializing it with our conf
        def instantiateClassFromConf[T](propertyName: String, defaultClassName: String): T = {
          instantiateClass[T](conf.get(propertyName, defaultClassName))
        }
    
        val serializer = instantiateClassFromConf[Serializer](
          "spark.serializer", "org.apache.spark.serializer.JavaSerializer")
        logDebug(s"Using serializer: ${serializer.getClass}")
    
        val serializerManager = new SerializerManager(serializer, conf, ioEncryptionKey)
    
        val closureSerializer = new JavaSerializer(conf)
    View Code

    closureSerializer 

  • 相关阅读:
    南邮OJA题
    Executors工厂类创建线程池的底层实现
    Linux kernel 中的per_cpu宏
    [置顶] 高并发服务器的设计内存池的设计
    数据列表DataList模板之实例
    软件开发无敌心得篇
    嵌入式学习笔记之UART通信协议
    正则表达式 进阶(二)
    11687 Digits
    DELPHI接口转化为COM接口
  • 原文地址:https://www.cnblogs.com/chengbao/p/10624917.html
Copyright © 2011-2022 走看看