zoukankan      html  css  js  c++  java
  • 用newLISP读取Hive的元数据

    思想是通过调用hive -e命令,然后解析返回的结果。

    以下的hive.lsp文件是一个样例,提供了读取本地hive数据库名称。表名,表结构和依据hive表的创建语句生成相应的MySQL的创建语句。

    最新代码參考我在gitlab.com上的开源项目:

    https://gitlab.com/newlisp/armory.git

    ;; hive.lsp
    
    ;; @module hive
    ;; @description hive module provides some features for manuplating local hive database
    ;; @note must make sure the hive env are correct
    ;; @location hive.lsp
    ;; @version 0.1.0
    ;; @author Dean Chen
    ;; @example
    ;; (load "/opt/newlisp_util/env.lsp")
    ;; (HIVE:show-dbs)
    ;; (HIVE:show-tables db-name)
    ;; (HIVE:show-create-table-statment db-name table-name)
    
    (context 'HIVE)
    
    ;; @syntax (HIVE:show-dbs)
    (define (show-dbs)
      (exec (format "hive -e 'show databases' 2>&1")))
    
    ;; @syntax (HIVE:show-tables db-name)
    (define (show-tables db-name)
      (exec (format "hive -e 'use %s; show tables' 2>&1" db-name)))
    
    ;; @syntax (HIVE:describe-table db-name table-name)
    (define (describe-table db-name table-name)
      (exec (format "hive -e 'use %s; describe %s' 2>&1" db-name table-name)))
    
    ;; @syntax (HIVE:show-create-table db-name table-name)
    (define (show-create-table db-name table-name)
      (exec (format "hive -e 'use %s; show create table %s' 2>&1" db-name table-name)))
    
    ;; @syntax (HIVE:gen-mysql-create-table hive-db-name hive-table-name)
    ;; @note this method use list index to locate CREATE EXTERNAL statement
    ;; it could be improved later for getting more compatibilty
    (define (gen-mysql-create-table db-name table-name)
      (set 's-list (show-create-table db-name table-name))
      (set 'header (replace "CREATE EXTERNAL" (s-list 6) "CREATE"))
      (set 'mysql-create-table-statement "")
      (set 'mysql-create-table-statement (append mysql-create-table-statement header))
      (set 'body (slice s-list 7))
      (catch (dolist (field body)
    	   (if (regex "PARTITIONED BY" field)
    	       (begin
    		(set 'sub-length (- (length mysql-create-table-statement) 1))
    		(set 'mysql-create-table-statement (slice mysql-create-table-statement 0 sub-length))
    		(throw mysql-create-table-statement))
    	     (begin
    	      (set 'column-list (parse field ","))
    	      (dolist (column column-list)
    		(set 'column-name (parse (trim column) " "))
    		(if column-name
    		    (begin
    		     (set 'mysql-create-table-statement (append mysql-create-table-statement (column-name 0)))
    		     (set 'mysql-create-table-statement (append mysql-create-table-statement " "))
    		     (set 'mysql-create-table-statement (append mysql-create-table-statement (replace "string" (column-name 1) "varchar(1024)")))
    		     (set 'mysql-create-table-statement (append mysql-create-table-statement ","))
    		     )
    		  )))))))


  • 相关阅读:
    Codeforces Round #251 (Div. 2) A
    topcoder SRM 623 DIV2 CatAndRat
    topcoder SRM 623 DIV2 CatchTheBeatEasy
    topcoder SRM 622 DIV2 FibonacciDiv2
    topcoder SRM 622 DIV2 BoxesDiv2
    Leetcode Linked List Cycle II
    leetcode Linked List Cycle
    Leetcode Search Insert Position
    关于vim插件
    Codeforces Round #248 (Div. 2) B. Kuriyama Mirai's Stones
  • 原文地址:https://www.cnblogs.com/zfyouxi/p/5178262.html
Copyright © 2011-2022 走看看