Presto实战(转)

原文url: https://my.oschina.net/guol/blog/891156

介绍

        Presto是一个开源的分布式SQL查询引擎，适用于交互式分析查询，数据量支持GB到PB字节。Presto的设计和编写完全是为了解决像Facebook这样规模的商业数据仓库的交互式分析和处理速度的问题。Presto支持在线数据查询，包括Hive, Cassandra, Mysql关系数据库以及专有数据存储。也支持Redis，Mongodb，Kafak这样的系统通过SQL语句来查询数据。一条Presto查询可以将多个数据源的数据进行合并，可以跨越整个组织进行分析。

        第一次接触Presto，还是0.150，现在版本已经更新到0.174，可见presto的更新还是很活跃的，社区氛围也不错。

依赖

        Mac OS X or Linux
        Java 8 Update 92 or higher (8u92+), 64-bit
        Maven 3.3.9+ (for building)
        Python 2.4+ (for running with the launcher script)

架构

        coordinator

            presto中的coordinator主要是控制worker节点的，一般称为调度节点

        worker

            presto中的worker主要是工作节点，具体的查询都是在worker节点上执行的。

部署

    下载

cd /opt/programs
wget 'https://repo1.maven.org/maven2/com/facebook/presto/presto-server/0.174/presto-server-0.174.tar.gz'

    coordinator部署

#进入下载目录
cd /opt/programs

#解压
tar czvf presto-server-0.174.tar.gz -C presto_174

#进入主目录
cd presto_174

#创建配置目录
mkdir etc

#etc目录的基础配置文件，均需要手动创建
##jvm.config
-server
-Xmx16G
-XX:+UseG1GC
-XX:G1HeapRegionSize=32M
-XX:+UseGCOverheadLimit
-XX:+ExplicitGCInvokesConcurrent
-XX:+HeapDumpOnOutOfMemoryError
-XX:OnOutOfMemoryError=kill -9 %p

##log.properties
com.facebook.presto=INFO

##node.properties
node.environment=production
node.id=109
node.data-dir=/tmp/presto/data

##config.properties
coordinator=true
node-scheduler.include-coordinator=false
http-server.http.port=9999
query.max-memory=20GB
query.max-memory-per-node=4GB
discovery-server.enabled=true
discovery.uri=http://192.168.1.109:9999

    worker部署

#进入下载目录
cd /opt/programs

#解压
tar czvf presto-server-0.174.tar.gz -C presto_174

#进入主目录
cd presto_174

#创建配置目录
mkdir etc

#etc目录的基础配置文件，均需要手动创建
##jvm.config
-server
-Xmx16G
-XX:+UseG1GC
-XX:G1HeapRegionSize=32M
-XX:+UseGCOverheadLimit
-XX:+ExplicitGCInvokesConcurrent
-XX:+HeapDumpOnOutOfMemoryError
-XX:OnOutOfMemoryError=kill -9 %p

##log.properties
com.facebook.presto=INFO

##node.properties
node.environment=production
node.id=135
node.data-dir=/tmp/presto/data

##config.properties
coordinator=false
http-server.http.port=9999
query.max-memory=20GB
query.max-memory-per-node=4GB
discovery.uri=http://192.168.1.109:9999

    Presto CLI部署

#进入bin
cd /opt/programs/presto_174/bin

#下载
wget 'https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/0.174/presto-cli-0.174-executable.jar'
mv presto-cli-0.174-executable.jar presto-cli
chmod 755 presto-cli

    Presto UI

        http://192.168.1.109:9999

    Presto init.d

#!/bin/bash
#
#Author: dalu, Date: 2016/12/8
#
###################################
# chkconfig read settings
# chkconfig: - 99 50
# description: presto start script
# processname: presto server
###################################
export PATH=/opt/programs/jdk1.8.0_111/bin/:$PATH
#java home
JAVA_HOME="/opt/programs/jdk1.8.0_111"

#app home
APP_HOME='/opt/programs/presto_174/'

#app name
APP_NAME='PrestoServer'

#app log dir
APP_LOG="$APP_HOME/var/logs/"

#java main function
APP_MAINCLASS="PrestoServer"

#start cmd
CMD="/opt/programs/presto_174/bin/launcher"


###################################
#check PrestoServer app is running
###################################
psid=0

checkpid(){
    javaps=`$JAVA_HOME/bin/jps -l | grep $APP_MAINCLASS`
    if [ -n "$javaps" ];then
	psid=`echo $javaps | awk '{print $1}'`
    else
	psid=0
    fi
}

###################################
# start PrestoServer app 
##################################

start(){
    checkpid
    if [ $psid -ne 0 ];then
	echo "================================"
	echo "warn: $APP_NAME already started! (pid=$psid)"
	echo "================================"
    else
	echo -n "Starting $APP_NAME ..."
	$CMD --pid-file=$APP_LOG/launcher.pid  --launcher-log-file=$APP_LOG/launcher.log --server-log-file=$APP_LOG/server.log start 2>&1
	checkpid
	if [ $psid -ne 0 ];then
	    echo "(pid=$psid) [OK]"
	else
	    echo "[Failed]"
	fi
    fi
}


##################################
# stop PrestoServer app
##################################

stop(){
    checkpid
    if [ $psid -ne 0 ];then
	echo -n "Stopping $APP_NAME ...(pid=$psid) "
	presto_pid=`$JAVA_HOME/bin/jps -l | grep $APP_MAINCLASS | awk '{print $1}'`
	kill $presto_pid
	if [ $? -eq 0 ];then
	    echo "[OK]"
	else
	    kill -9 $presto_pid
	    sleep 5
	    checkpid
	    if [ $psid -ne 0 ];then
		echo -n "[Failed]"	   
	    fi
	fi 
    else
	echo "================================"
	echo "warn: $APP_NAME is not running"
	echo "================================"
    fi
}

case "$1" in
    'start')
	start
	;;
    'stop')
	stop
	;;
    'restart')
	stop
	start
	;;
    *)
	echo "Usage: $0 {start|stop|restart}"
	exit 1
esac

exit 0

redis访问测试

    配置

#创建catalog
mkdir -p /opt/programs/presto_174/etc/catalog

#进入catalog
cd /opt/programs/presto_174/etc/catalog

#创建redis.properties
connector.name=redis
redis.table-names=antnest
redis.nodes=192.168.1.109:6379
redis.password=dalu
redis.default-schema=redis
redis.database-index=0
redis.table-description-dir=etc/redis
redis.hide-internal-columns=false

    创建redis映射

#创建redis映射目录
mkdir -p /opt/programs/presto_174/etc/redis

#进入映射目录
cd /opt/programs/presto_174/etc/redis

#创建映射文件redis.json
{
    "tableName": "antnest",
    "schemaName": "redis",
    "key": {
        "dataFormat": "raw",
        "fields": [
            {
		"name": "key",
		"type": "VARCHAR"
	    }
        ]
    },
    "value": {
        "dataFormat": "raw",
        "fields": [
            {
		"name": "value",
		"type": "VARCHAR"
	    }
       ]
    }
}

        ps：上述所有配置文件在coordinator和worker都需要拷贝一份，然后重启coordinator和worker上的presto进程    

    测试

cd /opt/programs/presto_174/bin
./presto-cli --server 192.168.1.109:9999 --catalog redis

        

mongodb访问测试

    配置　：　可以创建多个，如　mongoUser.properties  ,       xxxx.properties

#在catalog目录创建mongodb配置mongodb.properties
connector.name=mongodb
mongodb.seeds=192.168.1.109:12001
mongodb.credentials=admin:dalu@admin
mongodb.socket-keep-alive=true
mongodb.schema-collection=admin

    测试

kafka访问测试

    配置

#在catalog目录创建kafka配置
connector.name=kafka
kafka.table-names=json_data
kafka.nodes=192.168.1.109:9092
kafka.hide-internal-columns=false
kafka.table-description-dir=etc/kafka
kafka.default-schema=kafka

    配置映射

#创建kafka映射目录
mkdir -p /opt/programs/presto_174/etc/kafka

#创建kafka映射文件json_data.json
{
    "tableName": "json_data",
    "schemaName": "kafka",
    "topicName": "json_data",
    "key": {
        "dataFormat": "raw",
        "fields": [
            {
                "name": "kafka_key",
                "type": "BIGINT",
                "dataFormat": "LONG",
                "hidden": "false"
            }
        ]
    },
    "message": {
        "dataFormat": "json",
        "fields": [
            {
                "name": "name",
		"mapping": "name",
                "type": "VARCHAR"
            },
	    {
                "name": "phone",
                "mapping": "phone",
                "type": "VARCHAR"
            }
        ]
    }
}

    测试

第三方WEB

    airpal

        airpal不建议使用，很久没有更新了，版本还在0.1，一直没有变化，优点是有认证，采用apache shiro认证，UI界面不错。

    yanagishima

        1.x版本的时候，还比较搓，作者比较勤奋，一直在更新，现在已经更新到3.0版本了，界面也比airpal好看了

        下载

cd /opt/programs
wget 'wget https://bintray.com/artifact/download/wyukawa/generic/yanagishima-3.0.zip'
unzip yanagishima-3.0.zip

        配置

cd /opt/programs/yanagishima-3.0/conf

#配置文件yanagishima.properties
jetty.port=8088
presto.query.max-run-time-seconds=1800
presto.max-result-file-byte-size=1073741824
presto.datasources=presto1
presto.coordinator.server.presto1=http://192.168.1.109:9999
catalog.presto1=mongodb
schema.presto1=webspider
select.limit=500

        访问

            首页

            选择

            结果