代码改变世界
[登录 · 注册]
  • pgspider clickhouse fdw v2 版本
  • 此版本是基于clickhouse 的http 以及binary 协议更灵活高效

    环境准备

    • dockefile
    FROM dalongrong/pgspider:base as build
    WORKDIR /app
    RUN apt-get update && apt-get install -y libcurl4-openssl-dev automake autoconf libtool pkg-config  uuid-dev  cmake git build-essential gcc openssl
    RUN git clone https://github.com/adjust/clickhouse_fdw.git /app/postgresql-11.6/contrib/clickhousedb_fdw
    RUN cd /app/postgresql-11.6/contrib/clickhousedb_fdw && mkdir build && cd build  
        && cmake .. && make && make install
    FROM debian:stretch-slim
    ENV GOSU_VERSION 1.11
    RUN apt-get update && apt-get install -y  uuid-dev wget openssl libreadline-dev libcurl4-openssl-dev
    # explicitly set user/group IDs
    RUN set -eux; 
        groupadd -r postgres --gid=999; 
    # https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35
        useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; 
    # also create the postgres user's home directory with appropriate permissions
    # see https://github.com/docker-library/postgres/issues/274
        mkdir -p /var/lib/postgresql; 
        chown -R postgres:postgres /var/lib/postgresql
    RUN wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" 
       && chmod +x /usr/local/bin/gosu 
       && gosu nobody true
    # make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
    RUN set -eux; 
       if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then 
       # if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales)
       grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; 
       sed -ri '//usr/share/locale/d' /etc/dpkg/dpkg.cfg.d/docker; 
       ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; 
       fi; 
       apt-get update; apt-get install -y locales; rm -rf /var/lib/apt/lists/*; 
       localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
    ENV LANG en_US.utf8
    # install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift)
    # https://github.com/docker-library/postgres/issues/359
    # https://cwrap.org/nss_wrapper.html
    RUN set -eux; 
       apt-get update; 
       apt-get install -y --no-install-recommends libnss-wrapper; 
       rm -rf /var/lib/apt/lists/*
    RUN mkdir /docker-entrypoint-initdb.d && mkdir -p /opt/clickhousedb
    COPY  --from=build  /usr/local/pgspider /usr/local/pgspider
    RUN sed -ri "s!^#?(listen_addresses)s*=s*S+.*!1 = '*'!" /usr/local/pgspider/share/postgresql/postgresql.conf.sample; 
       grep -F "listen_addresses = '*'" /usr/local/pgspider/share/postgresql/postgresql.conf.sample
    RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql  && chmod 2777 /var/run/postgresql
    ENV PATH $PATH:/usr/local/pgspider/bin
    ENV PGDATA /var/lib/postgresql/data
    RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"
    VOLUME /var/lib/postgresql/data
    COPY docker-entrypoint.sh /usr/local/bin/
    RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat
    ENTRYPOINT ["docker-entrypoint.sh"]
    EXPOSE 5432
    CMD ["postgres"]
    • docker-compose 文件
    version: "3"
    services:
        server:
         image: yandex/clickhouse-server
         ports:
         - "8123:8123"
         - "9000:9000"
         - "9009:9009"
         ulimits:
          nproc: 65535
          nofile:
           soft: 262144
           hard: 262144
        client:
          image: yandex/clickhouse-client
          command: ['--host', 'server']
        pgspider:
          image: dalongrong/pgspider:clickhourse2
          ports: 
          - "5432:5432"
          environment: 
          - "POSTGRES_PASSWORD=dalong"
    • 启动&&数据初始化
    docker-compose up -d
    CREATE TABLE wikistat
    (
        project String,
        subproject String,
        hits UInt64,
        size UInt64
    ) ENGINE = Log;
    导入数据:
    serverhost=192.168.0.103 // replace with your hostid
    docker run -i yandex/clickhouse-client  --format_csv_delimiter="|" --host ${serverhost} --query="INSERT INTO default.wikistat FORMAT CSV" < ./data/info.csv

    使用

    • 创建扩展以及用户映射
    CREATE EXTENSION clickhouse_fdw;
    CREATE SERVER clickhouse_svr FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'default', host 'server');
    CREATE USER MAPPING FOR postgres SERVER clickhouse_svr
           OPTIONS (user 'default', password '');
    • 导入schema
    IMPORT FOREIGN SCHEMA "default" FROM SERVER clickhouse_svr INTO public;
    • 查询
    select * from wikistat;
    • 效果

    说明

    https://github.com/adjust/clickhouse_fdw 的fdw 相比Percona有很多提升,是一个不错的选择

    参考资料

    https://github.com/Percona-Lab/clickhousedb_fdw
    https://github.com/adjust/clickhouse_fdw
    https://github.com/rongfengliang/pgspider-docker

  • 【推广】 阿里云小站-上云优惠聚集地(新老客户同享)更有每天限时秒杀!
    【推广】 云服务器低至0.95折 1核2G ECS云服务器8.1元/月
    【推广】 阿里云老用户升级四重礼遇享6.5折限时折扣!
  • 原文:https://www.cnblogs.com/rongfengliang/p/13096808.html
走看看 - 开发者的网上家园