昨天有大概提过关于使用postgres s3 fdw 用来分析csv 数据的，今天简单试用下
环境准备
- docker-compose 文件
version: "3"
services:
postgres-fdw:
image: dalongrong/pgspider:pg_cron
ports:
- "5433:5432"
environment:
- "POSTGRES_PASSWORD=dalong"
postgres:
image: dalongrong/pg-s3-fdw:latest
environment:
- POSTGRES_PASSWORD=dalong
ports:
- 5432:5432
minio:
image: minio/minio
environment:
- MINIO_ACCESS_KEY=dalongrong
- MINIO_SECRET_KEY=dalongrong
ports:
- 9000:9000
command: server /data
- 启动&&初始化数据
启动
docker-compose up -d
minio 导入数据
csv 文件地址：wget http://examples.citusdata.com/customer_reviews_1998.csv.gz
注意文件需要解压，同时csv 文件需要添加头，内容为：
放首行
"customer_id","review_date","review_rating","review_votes","review_helpful_votes","product_id","product_title","product_sales_rank","product_group","product_category","product_subcategory","similar_product_ids
启用pg s3 fdw 扩展
// 创建扩展
CREATE EXTENSION multicorn;
// 创建server
CREATE SERVER multicorn_es FOREIGN DATA WRAPPER multicorn
OPTIONS (
wrapper 's3_fdw.S3ForeignDataWrapper'
);
// 创建外部表
CREATE FOREIGN TABLE customer_reviews(
customer_id TEXT,
review_date DATE,
review_rating INTEGER,
review_votes INTEGER,
review_helpful_votes INTEGER,
product_id CHAR(10),
product_title TEXT,
product_sales_rank BIGINT,
product_group TEXT,
product_category TEXT,
product_subcategory TEXT,
similar_product_ids CHAR(10)[])
SERVER multicorn_es
OPTIONS
(
bucket_name 'apps',
object_name 'customer_reviews.csv',
access_key 'dalongrong',
secret_key 'dalongrong',
endpoint_url 'http://minio:9000'
);
// 一个简单的数据加速（提高查询）
select * into customer_reviews3 from customer_reviews;
cube.js 项目
- 创建
cubejs create s3-app -d postgres
- 启动
yarn dev
- 效果
- 查询展示demo
说明
以上是一个简单的试用，实际上我们可以集成pg_cron 进行简单的调度，实现一些比较灵活的数据处理,当前此扩展对于csv 头的处理以及csv 分割格式
上没有配置选项，后边可以自己修改下
参考资料
https://github.com/smomni/s3_fdw_py
https://github.com/rongfengliang/pg-s3-fdw-learning
https://github.com/rongfengliang/pgspider-docker