cube.js s3 FileRepository 新功能

为了更好的集成cube.js 多租户的能力，进行了一些调整，支持基于配置参数的s3 schema 访问处理
同时加强了数据桶过滤的支持

核心修改的代码

主要是关于构造函数以及minio链接部分

 

constructor(config) {

        const { ...restConfig } = config || {};

        this.config = {

            endPoint: process.env.CUBEJS_S3_ENDPOINT,

            port: parseInt(process.env.CUBEJS_S3_PORT),

            useSSL: process.env.CUBEJS_S3_SSL == "true" ? true : false,

            accessKey: process.env.CUBEJS_S3_ACCESSKEY,

            secretKey: process.env.CUBEJS_s3_SECRETKEY,

            bucket:process.env.CUBEJS_S3_BUCKET,

            objectPrefix:"", // s3 object search  prefix default set ""

            ...restConfig,

        };

        this.minioClient = new Minio.Client({

            endPoint: this.config.endPoint,

            port: this.config.port,

            useSSL: this.config.useSSL,

            accessKey: this.config.accessKey,

            secretKey: this.config.secretKey

        });

    }

    async dataSchemaFiles(includeDependencies = true) {

        var self = this

        var bucket = self.config.bucket

        var objectPrefix = self.config.objectPrefix

        var Files = await streamToPromise(self.minioClient.listObjectsV2(bucket, objectPrefix, true))

        var fileContents = []

        for (const file of Files) {

            try {

                const fileBuffer = await streamToPromise(await self.minioClient.getObject(bucket, file.name))

                let fileItemContent = fileBuffer.toString('utf-8');

                fileContents.push({ fileName: file.name, content: fileItemContent })

            }

            catch (e) {

                console.log(e)

            }

        }

        if (includeDependencies) {

            fileContents = fileContents.concat(await this.readModules());

        }

        return fileContents;

    }

参考使用

新的调整支持基于环境变量以及配置参数的模式

• env 模式
  .env 文件

CUBEJS_S3_ENDPOINT=localhost

CUBEJS_S3_PORT=9000

CUBEJS_S3_SSL=false

CUBEJS_S3_ACCESSKEY=minio

CUBEJS_s3_SECRETKEY=minio123

CUBEJS_S3_BUCKET=demo

cube.js 配置
使用了一个自己开发的dremio driver

 

// Cube.js configuration options: https://cube.dev/docs/config

const {DremioDriver,DremioQuery} = require("@dalongrong/mydremio-driver")

const MyS3FileRepository = require("@dalongrong/cube-s3repository")

 

module.exports = {

    dialectFactory: (dataSource) => {

        // need config  datasource  for multitenant env

        return DremioQuery

    },

    queryTransformer: (query, { securityContext }) => {

        console.log(securityContext)

        return query;

      },

    dbType: ({ dataSource } = {}) => {

        return "mydremio"

    },

    driverFactory: ({ dataSource } = {}) => {

        return new DremioDriver({})

    },

    repositoryFactory: ({securityContext})=>{

        return new MyS3FileRepository.S3FileRepository({

            objectPrefix:"appdemo/"

        })

    }

};

参考资料

https://github.com/rongfengliang/cube.js-s3-filerepository
https://www.npmjs.com/package/@dalongrong/cube-s3repository