pgspider gzip fdw试用(集成gzip+http+graphql-engine)

gzip 也是一个在实际中比较有用的处理工具，可以减少数据传输，以下是集成gzip http 以及plv8 的处理

gzip Docker 镜像

• Dockerfile

FROM dalongrong/pgspider:base as build

WORKDIR /app

RUN apt-get update && apt-get install -y libssl-dev libz-dev pkg-config

RUN git clone https://github.com/pramsey/pgsql-gzip.git /app/postgresql-11.6/contrib/pgsql-gzip

RUN cd /app/postgresql-11.6/contrib/pgsql-gzip && make && make install

​

FROM debian:stretch-slim

ENV GOSU_VERSION 1.11

RUN apt-get update && apt-get install -y wget libssl-dev libz-dev libreadline-dev

# explicitly set user/group IDs

RUN set -eux; 

 groupadd -r postgres --gid=999; 

# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35

 useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; 

# also create the postgres user's home directory with appropriate permissions

# see https://github.com/docker-library/postgres/issues/274

 mkdir -p /var/lib/postgresql; 

 chown -R postgres:postgres /var/lib/postgresql

​

RUN wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" 

   && chmod +x /usr/local/bin/gosu 

   && gosu nobody true

# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default

RUN set -eux; 

   if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then 

   # if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales)

   grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; 

   sed -ri '//usr/share/locale/d' /etc/dpkg/dpkg.cfg.d/docker; 

   ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; 

   fi; 

   apt-get update; apt-get install -y locales; rm -rf /var/lib/apt/lists/*; 

   localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8

ENV LANG en_US.utf8

​

# install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift)

# https://github.com/docker-library/postgres/issues/359

# https://cwrap.org/nss_wrapper.html

RUN set -eux; 

   apt-get update; 

   apt-get install -y --no-install-recommends libnss-wrapper; 

   rm -rf /var/lib/apt/lists/*

​

RUN mkdir /docker-entrypoint-initdb.d

COPY --from=build /usr/local/pgspider /usr/local/pgspider

RUN sed -ri "s!^#?(listen_addresses)s*=s*S+.*!1 = '*'!" /usr/local/pgspider/share/postgresql/postgresql.conf.sample; 

   grep -F "listen_addresses = '*'" /usr/local/pgspider/share/postgresql/postgresql.conf.sample

RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql

ENV PATH $PATH:/usr/local/pgspider/bin

ENV PGDATA /var/lib/postgresql/data

RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"

VOLUME /var/lib/postgresql/data

COPY docker-entrypoint.sh /usr/local/bin/

RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat

ENTRYPOINT ["docker-entrypoint.sh"]

​

EXPOSE 5432

CMD ["postgres"]

• 集成http 的docker 镜像

FROM dalongrong/pgspider:base as build

WORKDIR /app

RUN apt-get update && apt-get install -y libssl-dev libz-dev pkg-config libcurl4-openssl-dev

RUN git clone https://github.com/pramsey/pgsql-gzip.git /app/postgresql-11.6/contrib/pgsql-gzip

RUN git clone https://github.com/pramsey/pgsql-http.git /app/postgresql-11.6/contrib/pgsql-http

RUN cd /app/postgresql-11.6/contrib/pgsql-gzip && make && make install

RUN cd /app/postgresql-11.6/contrib/pgsql-http && make && make install

​

​

FROM debian:stretch-slim

ENV GOSU_VERSION 1.11

RUN apt-get update && apt-get install -y wget libssl-dev libcurl4-openssl-dev libz-dev libreadline-dev

# explicitly set user/group IDs

RUN set -eux; 

   groupadd -r postgres --gid=999; 

# https://salsa.debian.org/postgresql/postgresql-common/blob/997d842ee744687d99a2b2d95c1083a2615c79e8/debian/postgresql-common.postinst#L32-35

   useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; 

# also create the postgres user's home directory with appropriate permissions

# see https://github.com/docker-library/postgres/issues/274

   mkdir -p /var/lib/postgresql; 

   chown -R postgres:postgres /var/lib/postgresql

​

RUN wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" 

   && chmod +x /usr/local/bin/gosu 

   && gosu nobody true

# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default

RUN set -eux; 

   if [ -f /etc/dpkg/dpkg.cfg.d/docker ]; then 

   # if this file exists, we're likely in "debian:xxx-slim", and locales are thus being excluded so we need to remove that exclusion (since we need locales)

   grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; 

   sed -ri '//usr/share/locale/d' /etc/dpkg/dpkg.cfg.d/docker; 

   ! grep -q '/usr/share/locale' /etc/dpkg/dpkg.cfg.d/docker; 

   fi; 

   apt-get update; apt-get install -y locales; rm -rf /var/lib/apt/lists/*; 

   localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8

ENV LANG en_US.utf8

​

# install "nss_wrapper" in case we need to fake "/etc/passwd" and "/etc/group" (especially for OpenShift)

# https://github.com/docker-library/postgres/issues/359

# https://cwrap.org/nss_wrapper.html

RUN set -eux; 

   apt-get update; 

   apt-get install -y --no-install-recommends libnss-wrapper; 

   rm -rf /var/lib/apt/lists/*

​

RUN mkdir /docker-entrypoint-initdb.d

COPY --from=build /usr/local/pgspider /usr/local/pgspider

RUN sed -ri "s!^#?(listen_addresses)s*=s*S+.*!1 = '*'!" /usr/local/pgspider/share/postgresql/postgresql.conf.sample; 

   grep -F "listen_addresses = '*'" /usr/local/pgspider/share/postgresql/postgresql.conf.sample

RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql

ENV PATH $PATH:/usr/local/pgspider/bin

ENV PGDATA /var/lib/postgresql/data

RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"

VOLUME /var/lib/postgresql/data

COPY docker-entrypoint.sh /usr/local/bin/

RUN ln -s usr/local/bin/docker-entrypoint.sh / # backwards compat

ENTRYPOINT ["docker-entrypoint.sh"]

​

EXPOSE 5432

CMD ["postgres"]

 

• 集成plv8 的镜像
  基于已有的基础镜像添加plv8 支持，因为构建方式使用了静态连接库

 

FROM dalongrong/pgspider:gzip-http

RUN apt-get update && apt-get install -y libc++1

COPY --from=dalongrong/pgspider:plv8 /usr/local/pgspider/lib/postgresql/plv8-2.3.12.so /usr/local/pgspider/lib/postgresql/plv8-2.3.12.so

COPY --from=dalongrong/pgspider:plv8 /usr/local/pgspider/share/postgresql/extension /usr/local/pgspider/share/postgresql/extension

集成使用

• docker-compose 文件

version: "3"

services: 

  graphql-engine:

    image: hasura/graphql-engine:v1.1.0

    ports:

    - "8080:8080"

    environment:

      HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:dalong@pgspider-fdw:5432/postgres

      HASURA_GRAPHQL_ENABLE_CONSOLE: "true" # set to "false" to disable console

      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log

  pgspider-fdw:

    image: dalongrong/pgspider:gzip-http-plv8

    ports: 

    - "5432:5432"

    environment: 

    - "POSTGRES_PASSWORD=dalong"

• 启动

docker-compose up -d

• 使用扩展

// create extension

create extension plv8;

create extension http;

create extension gzip;

// create fucntion

CREATE OR REPLACE FUNCTION plv8_eval_test()

RETURNS text AS $$

  var result = eval("1+2")

  return JSON.stringify(result);

$$ LANGUAGE plv8 IMMUTABLE STRICT;

// select

 create table appdemos as 

  select encode(gzip(content),'hex') as gzip ,plv8_eval_test() FROM http_get('http://httpbin.org/ip');

• hasura 集成

说明

以上是几个扩展的集成，我们可以灵活的扩展pg 对于数据的支持能力

参考资料

https://github.com/rongfengliang/pgspider-docker
https://github.com/hasura/graphql-engine/releases
https://github.com/pramsey/pgsql-gzip
https://github.com/pramsey/pgsql-http
https://hub.docker.com/repository/docker/dalongrong/pgspider