一、一个closure使用的直观例子
tsecer@harry: cat closure.py
def d(x):
def e(y):
return x + y
return e
f = d(1)
print(f(2))
tsecer@harry: ../../Python-3.6.0/python
Python 3.6.0 (default, Nov 15 2018, 10:32:57)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-4)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import closure, dis
3
>>> dis.dis(closure)
Disassembly of d:
2 0 LOAD_CLOSURE 0 (x)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object e at 0x7f3933b96880, file "/data1/harry/work/python/namelookup/closure.py", line 2>)
6 LOAD_CONST 2 ('d.<locals>.e')
8 MAKE_FUNCTION 8
10 STORE_FAST 1 (e)
4 12 LOAD_FAST 1 (e)
14 RETURN_VALUE
Disassembly of f:
3 0 LOAD_DEREF 0 (x)
2 LOAD_FAST 0 (y)
4 BINARY_ADD
6 RETURN_VALUE
>>> dis.dis(closure.d.__code__.co_consts[1])
3 0 LOAD_DEREF 0 (x)
2 LOAD_FAST 0 (y)
4 BINARY_ADD
6 RETURN_VALUE
tsecer@harry: ../../Python-3.6.0/python
二、使用到的虚拟机指令简单分析
1、函数d的主要内容
>>> dis.dis(closure)
Disassembly of d:
2 0 LOAD_CLOSURE 0 (x)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object e at 0x7f3933b96880, file "/data1/harry/work/python/namelookup/closure.py", line 2>)
6 LOAD_CONST 2 ('d.<locals>.e')
8 MAKE_FUNCTION 8
10 STORE_FAST 1 (e)
4 12 LOAD_FAST 1 (e)
14 RETURN_VALUE
2、主要的虚拟机指令解析
在 MAKE_FUNCTION 8 指令中，参数8是一个flag标志位，如果有这个标志位，会额外从栈中取出栈顶元素，放入新创建函数的func ->func_closure变量中
static PyObject *
_PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
PyObject **args, Py_ssize_t argcount,
PyObject **kwnames, PyObject **kwargs,
Py_ssize_t kwcount, int kwstep,
PyObject **defs, Py_ssize_t defcount,
PyObject *kwdefs, PyObject *closure,
PyObject *name, PyObject *qualname)
{
……
PyObject **fastlocals, **freevars;
……
freevars = f->f_localsplus + co->co_nlocals;
……
TARGET(LOAD_CLOSURE) {
PyObject *cell = freevars[oparg];
Py_INCREF(cell);
PUSH(cell);
DISPATCH();
}
……
TARGET(BUILD_TUPLE) {
PyObject *tup = PyTuple_New(oparg);
if (tup == NULL)
goto error;
while (--oparg >= 0) {
PyObject *item = POP();
PyTuple_SET_ITEM(tup, oparg, item);
}
PUSH(tup);
DISPATCH();
}
……
TARGET(MAKE_FUNCTION) {
PyObject *qualname = POP();
PyObject *codeobj = POP();
PyFunctionObject *func = (PyFunctionObject *)
PyFunction_NewWithQualName(codeobj, f->f_globals, qualname);
Py_DECREF(codeobj);
Py_DECREF(qualname);
if (func == NULL) {
goto error;
}
if (oparg & 0x08) {
assert(PyTuple_CheckExact(TOP()));
func ->func_closure = POP();
}
……
TARGET(LOAD_DEREF) {
PyObject *cell = freevars[oparg];
PyObject *value = PyCell_GET(cell);
if (value == NULL) {
format_exc_unbound(co, oparg);
goto error;
}
Py_INCREF(value);
PUSH(value);
DISPATCH();
}
……
}
3、函数调用时对函数func_closure的使用
这些收集到这些额外参数放在新创建函数对象的f->f_localsplus + co->co_nlocals中，作为“自己定义”的locals之外的额外补充
static PyObject *
_PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals,
PyObject **args, Py_ssize_t argcount,
PyObject **kwnames, PyObject **kwargs,
Py_ssize_t kwcount, int kwstep,
PyObject **defs, Py_ssize_t defcount,
PyObject *kwdefs, PyObject *closure,
PyObject *name, PyObject *qualname)
{
……
fastlocals = f->f_localsplus;
freevars = f->f_localsplus + co->co_nlocals;
……
/* Copy closure variables to free variables */
for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
Py_INCREF(o);
freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o;
}
……
三、源代码中的一些注释
1、FREE变量的概念
排除掉DEF_GLOBAL、DEF_NONLOCAL、以及DEF_BOUND包含的 DEF_LOCAL | DEF_PARAM | DEF_IMPORT之外，在一个block中的变量就被设置为FREE类型
#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
static int
static int
analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
PyObject *bound, PyObject *local, PyObject *free,
PyObject *global)
{
……
if (flags & DEF_GLOBAL) {
…… return 1;
}
if (flags & DEF_NONLOCAL) {
……
return PySet_Add(free, name) >= 0;
}
if (flags & DEF_BOUND) {
……
return 1;
}
/* If an enclosing block has a binding for this name, it
is a free variable rather than a global variable.
Note that having a non-NULL bound implies that the block
is nested.
*/
if (bound && PySet_Contains(bound, name)) {
SET_SCOPE(scopes, name, FREE);
ste->ste_free = 1;
return PySet_Add(free, name) >= 0;
}
……
}
2、FREE变量向更上层扩散
static int
update_symbols(PyObject *symbols, PyObject *scopes,
PyObject *bound, PyObject *free, int classflag)
{
……
/* Propagate new free symbol up the lexical stack */
if (PyDict_SetItem(symbols, name, v_free) < 0) {
goto error;
}
Py_DECREF(name);
}
Py_DECREF(itr);
Py_DECREF(v_free);
return 1;
error:
Py_XDECREF(v_free);
Py_XDECREF(itr);
Py_XDECREF(name);
return 0;
}
3、和FREE对应的CELL变量
简单的说，FREE就是引用发生的地方，而CELL则是被FREE引用的地方，这两个是对应的。
以开始的这个例子说明
def d(x):
def e(y):
return x + y
return e
函数e中的x就是FREE类型，而函数d中参数x则为CELL类型。
/* If a name is defined in free and also in locals, then this block
provides the binding for the free variable. The name should be
marked CELL in this block and removed from the free list.
Note that the current block's free variables are included in free.
That's safe because no name can be free and local in the same scope.
*/
static int
analyze_cells(PyObject *scopes, PyObject *free)
{
……
}
4、编译器对于closure变量的处理
static int
compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, PyObject *qualname)
{
Py_ssize_t i, free = PyCode_GetNumFree(co);
if (qualname == NULL)
qualname = co->co_name;
if (free) {
for (i = 0; i < free; ++i) {
……
flags |= 0x08;//这里设置flags中包含0x80
ADDOP_I(c, BUILD_TUPLE, free);
}
ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts);
ADDOP_O(c, LOAD_CONST, qualname, consts);
ADDOP_I(c, MAKE_FUNCTION, flags);
return 1;
}
四、大致的流程
1、思路
lua的作者有一片介绍lua中closure实现的文章《Closures in Lua》，实现的思路和python应该类似：
a)、在词法分析阶段找到这些free变量，这个是整个流程的开始；
b)、从这些包含free变量的词法作用域向外层逐层查找，找到这些变量的定义，称为CELL
c)、再反过来，对于包含了CELL变量的block，在执行MAKE_FUNCTION(对应源代码的def)是，把自己的CELL通过LOAD_CLOSURE放入将要make的函数的func_closure成员中
d)、FREE变量的地方访问变量时通过LOAD_DEREF来找到这些变量
2、关键的思路
在词法分析阶段要识别出哪些是FREE变量，然后找到这些FREE变量的定义位置，在定义位置和使用位置都要插入代码进行这些变量的“层层传递”。
看下面的代码，x在最内层的h函数中使用，但是从定义它的f到中间的g都添加了LOAD_CLOSURE函数，这相当于是一个层层接力的过程
tsecer@harry: cat nestedfree.py
def f(x, y):
def g(z):
def h(v):
return x + z + v
return h;
return g
tsecer@harry: ../../Python-3.6.0/python
Python 3.6.0 (default, Nov 15 2018, 10:32:57)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-4)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import dis, nestedfree
>>> dis.dis(nestedfree)
Disassembly of f:
2 0 LOAD_CLOSURE 0 (x)
2 BUILD_TUPLE 1
4 LOAD_CONST 1 (<code object g at 0x7fbfc5289280, file "/data1/harry/work/python/namelookup/nestedfree.py", line 2>)
6 LOAD_CONST 2 ('f.<locals>.g')
8 MAKE_FUNCTION 8
10 STORE_FAST 2 (g)
6 12 LOAD_FAST 2 (g)
14 RETURN_VALUE
>>> dis.dis(nestedfree.f.__code__.co_consts[1])
3 0 LOAD_CLOSURE 1 (x)
2 LOAD_CLOSURE 0 (z)
4 BUILD_TUPLE 2
6 LOAD_CONST 1 (<code object h at 0x7fbfc53087c0, file "/data1/harry/work/python/namelookup/nestedfree.py", line 3>)
8 LOAD_CONST 2 ('f.<locals>.g.<locals>.h')
10 MAKE_FUNCTION 8
12 STORE_FAST 1 (h)
5 14 LOAD_FAST 1 (h)
16 RETURN_VALUE
>>>