zoukankan      html  css  js  c++  java
  • Hive学习笔记——metastore listener

    除了使用hive hook来记录hive上用户的操作之外,还可以使用hive metastore listener来进行记录,参考:

    https://towardsdatascience.com/apache-hive-hooks-and-metastore-listeners-a-tale-of-your-metadata-903b751ee99f
    

    hive metastore的接口有3种,分别是

    Property Abstract class
    hive.metastore.pre.event.listeners org.apache.hadoop.hive.metastore.MetaStorePreEventListener
    hive.metastore.end.function.listeners org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener
    hive.metastore.event.listeners org.apache.hadoop.hive.metastore.MetaStoreEventListener

    下面的代码中继承了MetaStoreEventListener,实现的功能是在建表和修改表的时候,日志输出一下table的元数据信息

    package com.bigdata.hive;
    
    import com.fasterxml.jackson.databind.ObjectMapper;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
    import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
    import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    import java.io.IOException;
    
    public class MyListener extends MetaStoreEventListener {
    
        private static final Logger LOGGER = LoggerFactory.getLogger(MyListener.class);
        private static final ObjectMapper objMapper = new ObjectMapper();
    
        public MyListener(Configuration config) {
            super(config);
            logWithHeader(" created ");
        }
    
        @Override
        public void onCreateTable(CreateTableEvent event) {
            logWithHeader(event.getTable());
        }
    
        @Override
        public void onAlterTable(AlterTableEvent event) {
            logWithHeader(event.getOldTable());
            logWithHeader(event.getNewTable());
        }
    
        private void logWithHeader(Object obj) {
            LOGGER.info("[CustomListener][Thread: " + Thread.currentThread().getName() + "] | " + objToStr(obj));
        }
    
        private String objToStr(Object obj) {
            try {
                return objMapper.writeValueAsString(obj);
            } catch (IOException e) {
                LOGGER.error("Error on conversion", e);
            }
            return null;
        }
    
    }
    

    进行打包

    mvn clean package
    

    将jar包放到/var/lib/hive目录下,并修改用户组

    lintong@master:/var/lib/hive$ ls -al| grep jar
    -rw-r--r--   1 hive hive 363002613 12月 26 16:30 bigdata-1.0-SNAPSHOT-jar-with-dependencies.jar
    

    在CDH界面上配置metastore listener,注意是在metastore的配置项中,而不是hiveserver2的

    然后重启hive

    之后在HUE上运行建表语句

    create table test_table2(id int,name string)
    

    查看hive metastore的日志,可以看到日志中打印了所创建的hive表的元数据信息

    lintong@master:/var/log/hive$ tail -f hadoop-cmf-hive-HIVEMETASTORE-master.log.out | grep Listen
    2021-12-26 21:43:53,737 INFO  com.bigdata.hive.MyListener: [main]: [CustomListener][Thread: main] | " created "
    2021-12-26 21:45:15,224 INFO  com.bigdata.hive.MyListener: [pool-5-thread-14]: [CustomListener][Thread: pool-5-thread-14] | {"tableName":"CM_TEST_TABLE","dbName":"cloudera_manager_metastore_canary_test_db_hive_hivemetastore_0a5c4e82140edb166b3b3c29b6817024","owner":null,"createTime":1640526315,"lastAccessTime":0,"retention":0,"sd":{"cols":[{"name":"s","type":"string","comment":"test string","setType":true,"setComment":true,"setName":true},{"name":"f","type":"float","comment":"test float","setType":true,"setComment":true,"setName":true},{"name":"a","type":"array<map<string,struct<p1:int,p2:int>>>","comment":"test complex type","setType":true,"setComment":true,"setName":true}],"location":"hdfs://master:8020/user/hue/.cloudera_manager_hive_metastore_canary/hive_HIVEMETASTORE_0a5c4e82140edb166b3b3c29b6817024/cm_test_table","inputFormat":null,"outputFormat":null,"compressed":false,"numBuckets":1,"serdeInfo":{"name":"CM_TEST_TABLE","serializationLib":null,"parameters":{"serialization.format":"1"},"setSerializationLib":false,"setParameters":true,"parametersSize":1,"setName":true},"bucketCols":[],"sortCols":[],"parameters":{},"skewedInfo":null,"storedAsSubDirectories":false,"setParameters":true,"setSkewedInfo":false,"colsSize":3,"colsIterator":[{"name":"s","type":"string","comment":"test string","setType":true,"setComment":true,"setName":true},{"name":"f","type":"float","comment":"test float","setType":true,"setComment":true,"setName":true},{"name":"a","type":"array<map<string,struct<p1:int,p2:int>>>","comment":"test complex type","setType":true,"setComment":true,"setName":true}],"setCols":true,"setLocation":true,"setInputFormat":false,"setOutputFormat":false,"setCompressed":true,"setNumBuckets":true,"setSerdeInfo":true,"bucketColsSize":0,"bucketColsIterator":[],"setBucketCols":true,"sortColsSize":0,"sortColsIterator":[],"setSortCols":true,"setStoredAsSubDirectories":false,"parametersSize":0},"partitionKeys":[{"name":"p1","type":"string","comment":"partition-key-1","setType":true,"setComment":true,"setName":true},{"name":"p2","type":"int","comment":"partition-key-2","setType":true,"setComment":true,"setName":true}],"parameters":{"transient_lastDdlTime":"1640526315"},"viewOriginalText":null,"viewExpandedText":null,"tableType":null,"privileges":null,"temporary":false,"ownerType":"USER","partitionKeysSize":2,"setParameters":true,"setTableType":false,"setTableName":true,"setOwner":false,"setRetention":true,"partitionKeysIterator":[{"name":"p1","type":"string","comment":"partition-key-1","setType":true,"setComment":true,"setName":true},{"name":"p2","type":"int","comment":"partition-key-2","setType":true,"setComment":true,"setName":true}],"setPartitionKeys":true,"setViewOriginalText":false,"setViewExpandedText":false,"setPrivileges":false,"setTemporary":false,"setOwnerType":true,"setDbName":true,"setCreateTime":true,"setLastAccessTime":true,"setSd":true,"parametersSize":1}
    
  • 相关阅读:
    git stash功能的使用
    git tag的应用
    git merge 与 git rebase的区别?
    git的一些操作命令
    docker的常用操作
    lvs搭建dr负载均衡集群
    centos8安装lvs
    centos8安装docker
    centos8用firewalld搭建防火墙
    openresty上安装waf
  • 原文地址:https://www.cnblogs.com/tonglin0325/p/14615980.html
Copyright © 2011-2022 走看看