edgedb 内部pg 数据存储的探索 (三) 源码包setup.py 文件

edgedb 是基于python开发的，同时集成了cython 以下为包的setup.py 配置，从里面我们可以看到关于edgedb 的一些依赖
以及构建过程

setup.py 源码

整体配置不算很多，500 多行，主要是cython extension 配置以及pg 构建配置，以及pg extension 配置,其中添加了关于pg 以及
pg 扩展build 的自定义cmdclass

• 代码

#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2008-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


import os.path
import pathlib
import platform
import shutil
import subprocess
import textwrap

from distutils import extension as distutils_extension
from distutils.command import build as distutils_build
from distutils.command import build_ext as distutils_build_ext

import setuptools
from setuptools.command import develop as setuptools_develop


RUNTIME_DEPS = [
    'asyncpg~=0.18.2',
    'click~=6.7',
    'httptools>=0.0.13',
    'immutables>=0.9',
    'parsing~=1.6.1',
    'prompt_toolkit>=2.0.0',
    'psutil~=5.6.1',
    'Pygments~=2.3.0',
    'setproctitle~=1.1.10',
    'setuptools_scm~=3.2.0',
    'uvloop~=0.12.2',

    'graphql-core~=2.1.0',
    'promise~=2.2.0',
]

CYTHON_DEPENDENCY = 'Cython==0.29.6'

DOCS_DEPS = [
    'Sphinx~=2.0.0',
    'lxml~=4.2.5',
    'sphinxcontrib-asyncio~=0.2.0',
]

BUILD_DEPS = [
    CYTHON_DEPENDENCY,
]

EXTRA_DEPS = {
    'test': [
        'flake8~=3.7.5',
        'pycodestyle~=2.5.0',
        'coverage~=4.5.2',
        'requests-xml~=0.2.3',
        'lxml',
        'requests-xml',
    ] + DOCS_DEPS,

    'docs': DOCS_DEPS,
}

EXT_CFLAGS = ['-O2']
EXT_LDFLAGS = []


if platform.uname().system != 'Windows':
    EXT_CFLAGS.extend([
        '-std=c99', '-fsigned-char', '-Wall', '-Wsign-compare', '-Wconversion'
    ])


def _compile_parsers(build_lib, inplace=False):
    import parsing

    import edb.edgeql.parser.grammar.single as edgeql_spec
    import edb.edgeql.parser.grammar.block as edgeql_spec2
    import edb.edgeql.parser.grammar.sdldocument as schema_spec

    base_path = pathlib.Path(__file__).parent.resolve()

    for spec in (edgeql_spec, edgeql_spec2, schema_spec):
        spec_path = pathlib.Path(spec.__file__).parent
        subpath = pathlib.Path(str(spec_path)[len(str(base_path)) + 1:])
        pickle_name = spec.__name__.rpartition('.')[2] + '.pickle'
        pickle_path = subpath / pickle_name
        cache = build_lib / pickle_path
        cache.parent.mkdir(parents=True, exist_ok=True)
        parsing.Spec(spec, pickleFile=str(cache), verbose=True)
        if inplace:
            shutil.copy2(cache, base_path / pickle_path)


def _compile_build_meta(build_lib, version, pg_config, runstatedir):
    import pkg_resources
    from edb.server import buildmeta

    parsed_version = buildmeta.parse_version(
        pkg_resources.parse_version(version))

    vertuple = list(parsed_version._asdict().values())
    vertuple[2] = int(vertuple[2])
    vertuple = tuple(vertuple)

    content = textwrap.dedent('''
        #
        # This source file is part of the EdgeDB open source project.
        #
        # Copyright 2008-present MagicStack Inc. and the EdgeDB authors.
        #
        # Licensed under the Apache License, Version 2.0 (the "License");
        #
        # THIS FILE HAS BEEN AUTOMATICALLY GENERATED.
        #

        PG_CONFIG_PATH = {pg_config!r}
        RUNSTATE_DIR = {runstatedir!r}
        VERSION = {version!r}
    ''').format(version=vertuple, pg_config=pg_config, runstatedir=runstatedir)

    directory = build_lib / 'edb' / 'server'
    if not directory.exists():
        directory.mkdir(parents=True)

    with open(directory / '_buildmeta.py', 'w+t') as f:
        f.write(content)


def _compile_postgres(build_base, *,
                      force_build=False, fresh_build=True,
                      run_configure=True, build_contrib=True):

    proc = subprocess.run(
        ['git', 'submodule', 'status', 'postgres'],
        stdout=subprocess.PIPE, universal_newlines=True, check=True)
    status = proc.stdout
    if status[0] == '-':
        print(
            'postgres submodule not initialized, '
            'run `git submodule init; git submodule update`')
        exit(1)

    proc = subprocess.run(
        ['git', 'submodule', 'status', 'postgres'],
        stdout=subprocess.PIPE, universal_newlines=True, check=True)
    revision, _, _ = proc.stdout[1:].partition(' ')
    source_stamp = proc.stdout[0] + revision

    postgres_build = (build_base / 'postgres').resolve()
    postgres_src = (pathlib.Path(__file__).parent / 'postgres').resolve()
    postgres_build_stamp = postgres_build / 'stamp'

    if postgres_build_stamp.exists():
        with open(postgres_build_stamp, 'r') as f:
            build_stamp = f.read()
    else:
        build_stamp = None

    is_outdated = source_stamp != build_stamp

    if is_outdated or force_build:
        system = platform.system()
        if system == 'Darwin':
            uuidlib = 'e2fs'
        elif system == 'Linux':
            uuidlib = 'e2fs'
        else:
            raise NotImplementedError('unsupported system: {}'.format(system))

        if fresh_build and postgres_build.exists():
            shutil.rmtree(postgres_build)
        build_dir = postgres_build / 'build'
        if not build_dir.exists():
            build_dir.mkdir(parents=True)

        if run_configure or fresh_build or is_outdated:
            subprocess.run([
                str(postgres_src / 'configure'),
                '--prefix=' + str(postgres_build / 'install'),
                '--with-uuid=' + uuidlib,
            ], check=True, cwd=str(build_dir))

        subprocess.run(
            ['make', 'MAKELEVEL=0', '-j', str(max(os.cpu_count() - 1, 1))],
            cwd=str(build_dir), check=True)

        if build_contrib or fresh_build or is_outdated:
            subprocess.run(
                [
                    'make', '-C', 'contrib', 'MAKELEVEL=0', '-j',
                    str(max(os.cpu_count() - 1, 1))
                ],
                cwd=str(build_dir), check=True)

        subprocess.run(
            ['make', 'MAKELEVEL=0', 'install'],
            cwd=str(build_dir), check=True)

        if build_contrib or fresh_build or is_outdated:
            subprocess.run(
                ['make', '-C', 'contrib', 'MAKELEVEL=0', 'install'],
                cwd=str(build_dir), check=True)

        with open(postgres_build_stamp, 'w') as f:
            f.write(source_stamp)


def _compile_postgres_extensions(build_base):
    postgres_build = (build_base / 'postgres').resolve()
    postgres_build_stamp_path = postgres_build / 'stamp'

    ext_build = (build_base / 'ext').resolve()
    ext_build_stamp_path = ext_build / 'stamp'

    if postgres_build_stamp_path.exists():
        with open(postgres_build_stamp_path, 'r') as f:
            postgres_build_stamp = f.read()
    else:
        raise RuntimeError('Postgres is not built, cannot build extensions')

    if ext_build_stamp_path.exists():
        with open(ext_build_stamp_path, 'r') as f:
            ext_build_stamp = f.read()
    else:
        ext_build_stamp = None

    ext_dir = (pathlib.Path(__file__).parent / 'ext').resolve()
    pg_config = (build_base / 'postgres' / 'install' /
                 'bin' / 'pg_config').resolve()

    if not ext_dir.exists():
        raise RuntimeError('missing Postgres extension directory')

    ext_make = ['make', '-C', str(ext_dir), 'PG_CONFIG=' + str(pg_config)]

    if ext_build_stamp != postgres_build_stamp:
        print('Extensions build stamp does not match Postgres build stamp. '
              'Rebuilding...')
        subprocess.run(ext_make + ['clean'], check=True)

    ext_build.mkdir(parents=True, exist_ok=True)

    subprocess.run(ext_make, check=True)
    subprocess.run(ext_make + ['install'], check=True)

    ext_build_stamp = postgres_build_stamp

    with open(ext_build_stamp_path, 'w') as f:
        f.write(ext_build_stamp)


class build(distutils_build.build):

    user_options = distutils_build.build.user_options + [
        ('pg-config=', None, 'path to pg_config to use with this build'),
        ('runstatedir=', None, 'directory to use for the runtime state'),
    ]

    def initialize_options(self):
        super().initialize_options()
        self.pg_config = None
        self.runstatedir = None

    def finalize_options(self):
        super().finalize_options()

    def run(self, *args, **kwargs):
        super().run(*args, **kwargs)
        build_lib = pathlib.Path(self.build_lib)
        _compile_parsers(build_lib)
        if self.pg_config:
            _compile_build_meta(
                build_lib,
                self.distribution.metadata.version,
                self.pg_config,
                self.runstatedir,
            )


class develop(setuptools_develop.develop):

    def run(self, *args, **kwargs):
        _compile_parsers(pathlib.Path('build/lib'), inplace=True)
        _compile_postgres(pathlib.Path('build').resolve())
        _compile_postgres_extensions(pathlib.Path('build').resolve())

        scripts = self.distribution.entry_points['console_scripts']
        patched_scripts = [s + '_dev' for s in scripts
                           if not s.startswith('edgedb-server')]
        patched_scripts.append('edb = edb.tools.edb:edbcommands')
        self.distribution.entry_points['console_scripts'] = patched_scripts

        super().run(*args, **kwargs)


class build_postgres(setuptools.Command):

    description = "build postgres"

    user_options = [
        ('configure', None, 'run ./configure'),
        ('build-contrib', None, 'build contrib'),
        ('fresh-build', None, 'rebuild from scratch'),
    ]

    def initialize_options(self):
        self.configure = False
        self.build_contrib = False
        self.fresh_build = False

    def finalize_options(self):
        pass

    def run(self, *args, **kwargs):
        _compile_postgres(
            pathlib.Path('build').resolve(),
            force_build=True,
            fresh_build=self.fresh_build,
            run_configure=self.configure,
            build_contrib=self.build_contrib)

        _compile_postgres_extensions(
            pathlib.Path('build').resolve())


class build_postgres_ext(setuptools.Command):

    description = "build postgres extensions"

    user_options = []

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self, *args, **kwargs):
        _compile_postgres_extensions(
            pathlib.Path('build').resolve())


class build_ext(distutils_build_ext.build_ext):

    user_options = distutils_build_ext.build_ext.user_options + [
        ('cython-annotate', None,
            'Produce a colorized HTML version of the Cython source.'),
        ('cython-directives=', None,
            'Cython compiler directives'),
    ]

    def initialize_options(self):
        # initialize_options() may be called multiple times on the
        # same command object, so make sure not to override previously
        # set options.
        if getattr(self, '_initialized', False):
            return

        super(build_ext, self).initialize_options()

        if os.environ.get('EDGEDB_DEBUG'):
            self.cython_always = True
            self.cython_annotate = True
            self.cython_directives = "linetrace=True"
            self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
            self.debug = True
        else:
            self.cython_always = False
            self.cython_annotate = None
            self.cython_directives = None
            self.debug = False

    def finalize_options(self):
        # finalize_options() may be called multiple times on the
        # same command object, so make sure not to override previously
        # set options.
        if getattr(self, '_initialized', False):
            return

        import pkg_resources

        # Double check Cython presence in case setup_requires
        # didn't go into effect (most likely because someone
        # imported Cython before setup_requires injected the
        # correct egg into sys.path.
        try:
            import Cython
        except ImportError:
            raise RuntimeError(
                'please install {} to compile edgedb from source'.format(
                    CYTHON_DEPENDENCY))

        cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
        if Cython.__version__ not in cython_dep:
            raise RuntimeError(
                'edgedb requires {}, got Cython=={}'.format(
                    CYTHON_DEPENDENCY, Cython.__version__
                ))

        from Cython.Build import cythonize

        directives = {
            'language_level': '3'
        }

        if self.cython_directives:
            for directive in self.cython_directives.split(','):
                k, _, v = directive.partition('=')
                if v.lower() == 'false':
                    v = False
                if v.lower() == 'true':
                    v = True

                directives[k] = v

        self.distribution.ext_modules[:] = cythonize(
            self.distribution.ext_modules,
            compiler_directives=directives,
            annotate=self.cython_annotate,
            include_path=["edb/server/pgproto/"])

        super(build_ext, self).finalize_options()


setuptools.setup(
    setup_requires=RUNTIME_DEPS + BUILD_DEPS,
    use_scm_version=True,
    name='edgedb-server',
    description='EdgeDB Server',
    author='MagicStack Inc.',
    author_email='hello@magic.io',
    packages=['edb'],
    include_package_data=True,
    cmdclass={
        'build': build,
        'build_ext': build_ext,
        'develop': develop,
        'build_postgres': build_postgres,
        'build_postgres_ext': build_postgres_ext,
    },
    entry_points={
        'console_scripts': [
            'edgedb = edb.cli:cli',
            'edgedb-server = edb.server.main:main',
        ]
    },
    ext_modules=[
        distutils_extension.Extension(
            "edb.server.pgproto.pgproto",
            ["edb/server/pgproto/pgproto.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.dbview.dbview",
            ["edb/server/dbview/dbview.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.mng_port.edgecon",
            ["edb/server/mng_port/edgecon.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.cache.stmt_cache",
            ["edb/server/cache/stmt_cache.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.pgcon.pgcon",
            ["edb/server/pgcon/pgcon.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.http.http",
            ["edb/server/http/http.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.http_edgeql_port.protocol",
            ["edb/server/http_edgeql_port/protocol.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.http_graphql_port.protocol",
            ["edb/server/http_graphql_port/protocol.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),
    ],
    install_requires=RUNTIME_DEPS,
    extras_require=EXTRA_DEPS,
    test_suite='tests.suite',
)

代码说明

核心代码还是setuptools.setup 中的内容，一种包含了依赖包以及扩展的配置

• 依赖

  依赖包含了运行时依赖以及构建依赖

运行时依赖，主要是一些核心库，包括了pg 连接库 http 操作的，graphql 依赖库，cli解析的
RUNTIME_DEPS = [
    'asyncpg~=0.18.2',
    'click~=6.7',
    'httptools>=0.0.13',
    'immutables>=0.9',
    'parsing~=1.6.1',
    'prompt_toolkit>=2.0.0',
    'psutil~=5.6.1',
    'Pygments~=2.3.0',
    'setproctitle~=1.1.10',
    'setuptools_scm~=3.2.0',
    'uvloop~=0.12.2',
    'graphql-core~=2.1.0',
    'promise~=2.2.0',
]
CYTHON_DEPENDENCY = 'Cython==0.29.6'
BUILD_DEPS = [
    CYTHON_DEPENDENCY,
]

• 包名称

  edgedb 基本上所有python相关的代码都在edb 中

    packages=['edb'],

• 自定义cmdclass 以及entry_points

  详细的下边细说

# 添加了关于（pg 以及edgedb）构建，（cython，pg）扩展构建
cmdclass={
        'build': build,
        'build_ext': build_ext,
        'develop': develop,
        'build_postgres': build_postgres,
        'build_postgres_ext': build_postgres_ext,
    },
    entry_points={
        'console_scripts': [
            'edgedb = edb.cli:cli',
            'edgedb-server = edb.server.main:main',
        ]
    }    

• 依赖的cython extension 配置

  主要是cython 配置，暴露的包名以及cython 路径，以及编译选项

ext_modules=[
        distutils_extension.Extension(
            "edb.server.pgproto.pgproto",
            ["edb/server/pgproto/pgproto.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.dbview.dbview",
            ["edb/server/dbview/dbview.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.mng_port.edgecon",
            ["edb/server/mng_port/edgecon.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.cache.stmt_cache",
            ["edb/server/cache/stmt_cache.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.pgcon.pgcon",
            ["edb/server/pgcon/pgcon.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.http.http",
            ["edb/server/http/http.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.http_edgeql_port.protocol",
            ["edb/server/http_edgeql_port/protocol.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),

        distutils_extension.Extension(
            "edb.server.http_graphql_port.protocol",
            ["edb/server/http_graphql_port/protocol.pyx"],
            extra_compile_args=EXT_CFLAGS,
            extra_link_args=EXT_LDFLAGS),
    ],
    install_requires=RUNTIME_DEPS,
    extras_require=EXTRA_DEPS,

cmdclass 说明

cmdclass 是edgedb 包的核心部分，包含了cython 构建以及pg 源码构建pg extension 扩展构建

• build_ext 说明

  主要是cython 相关的配置

class build_ext(distutils_build_ext.build_ext):

    user_options = distutils_build_ext.build_ext.user_options + [
        ('cython-annotate', None,
            'Produce a colorized HTML version of the Cython source.'),
        ('cython-directives=', None,
            'Cython compiler directives'),
    ]

    def initialize_options(self):
        # initialize_options() may be called multiple times on the
        # same command object, so make sure not to override previously
        # set options.
        if getattr(self, '_initialized', False):
            return

        super(build_ext, self).initialize_options()

        if os.environ.get('EDGEDB_DEBUG'):
            self.cython_always = True
            self.cython_annotate = True
            self.cython_directives = "linetrace=True"
            self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
            self.debug = True
        else:
            self.cython_always = False
            self.cython_annotate = None
            self.cython_directives = None
            self.debug = False

    def finalize_options(self):
        # finalize_options() may be called multiple times on the
        # same command object, so make sure not to override previously
        # set options.
        if getattr(self, '_initialized', False):
            return

        import pkg_resources

        # Double check Cython presence in case setup_requires
        # didn't go into effect (most likely because someone
        # imported Cython before setup_requires injected the
        # correct egg into sys.path.
        try:
            import Cython
        except ImportError:
            raise RuntimeError(
                'please install {} to compile edgedb from source'.format(
                    CYTHON_DEPENDENCY))

        cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
        if Cython.__version__ not in cython_dep:
            raise RuntimeError(
                'edgedb requires {}, got Cython=={}'.format(
                    CYTHON_DEPENDENCY, Cython.__version__
                ))

        from Cython.Build import cythonize

        directives = {
            'language_level': '3'
        }

        if self.cython_directives:
            for directive in self.cython_directives.split(','):
                k, _, v = directive.partition('=')
                if v.lower() == 'false':
                    v = False
                if v.lower() == 'true':
                    v = True

                directives[k] = v

        self.distribution.ext_modules[:] = cythonize(
            self.distribution.ext_modules,
            compiler_directives=directives,
            annotate=self.cython_annotate,
            include_path=["edb/server/pgproto/"])

        super(build_ext, self).finalize_options()

• build_postgres pg 构建配置

  pg 是edgedb 的核心，build_postgres 提供了pg 源码编译的处理

class build_postgres(setuptools.Command):

    description = "build postgres"

    user_options = [
        ('configure', None, 'run ./configure'),
        ('build-contrib', None, 'build contrib'),
        ('fresh-build', None, 'rebuild from scratch'),
    ]

    def initialize_options(self):
        self.configure = False
        self.build_contrib = False
        self.fresh_build = False

    def finalize_options(self):
        pass

    def run(self, *args, **kwargs):
        _compile_postgres(
            pathlib.Path('build').resolve(),
            force_build=True,
            fresh_build=self.fresh_build,
            run_configure=self.configure,
            build_contrib=self.build_contrib)

        _compile_postgres_extensions(
            pathlib.Path('build').resolve())

• build_postgres_ext 提供pg 扩展的构建处理

  目前主要的扩展是提供一些工具类的封装扩展名称为 edbsys，代码在ext 中

class build_postgres_ext(setuptools.Command):
    description = "build postgres extensions"
    user_options = []
    def initialize_options(self):
        pass
    def finalize_options(self):
        pass
    def run(self, *args, **kwargs):
        _compile_postgres_extensions(
            pathlib.Path('build').resolve())

• build 配置

  主要提供了pg_config 的配置，指定后边pg 以及pg 扩展构建的环境

class build(distutils_build.build):

    user_options = distutils_build.build.user_options + [
        ('pg-config=', None, 'path to pg_config to use with this build'),
        ('runstatedir=', None, 'directory to use for the runtime state'),
    ]

    def initialize_options(self):
        super().initialize_options()
        self.pg_config = None
        self.runstatedir = None

    def finalize_options(self):
        super().finalize_options()

    def run(self, *args, **kwargs):
        super().run(*args, **kwargs)
        build_lib = pathlib.Path(self.build_lib)
        _compile_parsers(build_lib)
        if self.pg_config:
            _compile_build_meta(
                build_lib,
                self.distribution.metadata.version,
                self.pg_config,
                self.runstatedir,
            )

说明

以上只是一个简单的分析，这个文件主要是为了进行edgedb 打包构建处理的，里面也使用了关于cython 的技术
还是值得学习的

参考资料

https://github.com/edgedb/edgedb
https://medium.com/@shamir.stav_83310/making-your-c-library-callable-from-python-by-wrapping-it-with-cython-b09db35012a3
https://github.com/stavshamir/cython-c-wrapper/
https://cython.readthedocs.io/en/latest/src/tutorial/external.html
https://cython.readthedocs.io/en/latest/src/tutorial/clibraries.html
http://pages.cs.wisc.edu/~yezheng/post/cython/
https://github.com/rongfengliang/cython-c-pip-demo/tree/local_source