【CPython3.6源码分析】Python 控制字节码执行

前言

本章将通过几个字节码指令,探究 Python 中iffor语句的实现及原理,
阅读本章前需了解PyCodeObject/PyFrameObject

IF控制流(JUMP_*)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
a = 0

if a > 0:
b = 3
elif a < 0:
b = -3
else:
b = 0
c = 1

co_consts: (0, 3, 1, None, -3)
co_names: ('a', 'b', 'c')

1 0 LOAD_CONST 0 (0)
2 STORE_NAME 0 (a)
3 4 LOAD_NAME 0 (a)
6 LOAD_CONST 0 (0)
8 COMPARE_OP 4 (>) // 执行比较操作
10 POP_JUMP_IF_FALSE 18 // if POP()== Py_False: JUMPTO(18)
4 12 LOAD_CONST 1 (3)
14 STORE_NAME 1 (b)
16 JUMP_FORWARD 18 (to 36) // JUMPBY(18)
5 >> 18 LOAD_NAME 0 (a)
20 LOAD_CONST 0 (0)
22 COMPARE_OP 0 (<)
24 POP_JUMP_IF_FALSE 32// if POP()== Py_False: JUMPTO(32)
6 26 LOAD_CONST 4 (-3)
28 STORE_NAME 1 (b)
30 JUMP_FORWARD 4 (to 36)
8 >> 32 LOAD_CONST 0 (0)
34 STORE_NAME 1 (b)
9 >> 36 LOAD_CONST 2 (1)
38 STORE_NAME 2 (c)
40 LOAD_CONST 3 (None)
42 RETURN_VALUE

typedef uint16_t _Py_CODEUNIT; // 2字节
#define JUMPTO(x) (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT))
#define JUMPBY(x) (next_instr += (x) / sizeof(_Py_CODEUNIT))

可以看出,比较操作,会调用 COMPARE_OP,然后根据结果进行跳转。而大于符号>,在数组 opcode.h/enum cmp_op{}中,索引为4。

我们知道 first_instr,始终指向字节码开始位置。因此,JUMPTO(18),最终的结果是,改变下一条 code 指针next_instr的值,实际上就是跳转到索引为 18 的字节码处。JUMPBY(18)相当于在下一条的基础上,跳转偏移量为18。

这里要说一下还是说一下,为什么要(x) / sizeof(_Py_CODEUNIT)。在前面我们也已经提到,STORE_NAME本身是一个数字,而这个数字是_Py_CODEUNIT形式。所以,每增加一条指令,指针就要移动相应的位置。

还需要说明的是,这里的JUMP*皆是 Python 用户层面的跳转,影响的是用户代码逻辑。而goto fast_block_end,是在 Python虚拟机层面的跳转,影响的是虚拟机的状态。

COMPARE_OP

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
// opcode.h
enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE,
PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, PyCmp_IN, PyCmp_NOT_IN,
PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};

TARGET(COMPARE_OP) {
PyObject *right = POP();
PyObject *left = TOP();
PyObject *res = cmp_outcome(oparg, left, right);
Py_DECREF(left);
Py_DECREF(right);
SET_TOP(res);
if (res == NULL)
goto error;
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_IF_TRUE);
DISPATCH();
}

static PyObject *
cmp_outcome(int op, PyObject *v, PyObject *w)
{
int res = 0;
switch (op) {
case PyCmp_IS:
res = (v == w); // 注意,此处是 PyObject 之间的比较
break;
case PyCmp_IS_NOT:
res = (v != w); // 而不是 C对象
break;
case PyCmp_IN:
...
}
v = res ? Py_True : Py_False;
Py_INCREF(v);
return v;
}

从定义中可以看见,in/is 操作也是走 COMPARE_OP 流程。并且返回的是 PyObject。

FOR控制流

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# demo.py
a = [1, 2, 3, 4]
for i in a:
print(i)

co_consts: (1, 2, 3, 4, None)
co_names: ('a', 'i', 'print')

1 0 LOAD_CONST 0 (1)
2 LOAD_CONST 1 (2)
4 LOAD_CONST 2 (3)
6 LOAD_CONST 3 (4)
8 BUILD_LIST 4
10 STORE_NAME 0 (a)

2 12 SETUP_LOOP 20 (to 34)
14 LOAD_NAME 0 (a)
16 GET_ITER
>> 18 FOR_ITER 12 (to 32)
20 STORE_NAME 1 (i)

3 22 LOAD_NAME 2 (print)
24 LOAD_NAME 1 (i)
26 CALL_FUNCTION 1
28 POP_TOP
30 JUMP_ABSOLUTE 18
>> 32 POP_BLOCK
>> 34 LOAD_CONST 4 (None)
36 RETURN_VALUE

SETUP_LOOP

1
2
3
4
5
6
7
8
9
10
11
12
opcode:: SETUP_LOOP (delta)

Pushes a block for a loop onto the block stack. The block spans from the
current instruction with a size of *delta* bytes.

TARGET(SETUP_LOOP) // 三种方式,都走该函数
TARGET(SETUP_EXCEPT) // loop/except/finally
TARGET(SETUP_FINALLY) {
PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg,
STACK_LEVEL());
DISPATCH();
}

下面是很重要的循环控制,关于 FrameObject 的操作,自然在frameobject.c

PyFrame_BlockSetup

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
void
PyFrame_BlockSetup(PyFrameObject *f, int type, int handler, int level)
{
PyTryBlock *b;
// index in f_blockstack,在 PyFrame_New 中,初始化为 0
if (f->f_iblock >= CO_MAXBLOCKS)
Py_FatalError("XXX block stack overflow");

/* 在 PyFrameObject 结构体中,被定义为
PyTryBlock f_blockstack[CO_MAXBLOCKS]
获取 PyTryBlock,指针 ++ */
b = &f->f_blockstack[f->f_iblock++];

/* 定义 code block 块类型 ,直接跟 opcode 一致。
SETUP_* -> loop/except/finally */
b->b_type = type; // block 类型
b->b_level = level; // 保存栈位置
b->b_handler = handler;
}

#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */

typedef struct {
int b_type; /* what kind of block this is */
int b_handler; /* where to jump to find handler */
int b_level; /* value stack level to pop to */
} PyTryBlock;

这里的 f 自然是,系统运行时的当前 FrameObject。在数组中获取一个新的 block,并存放相关信息。

GET_ITER

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
         16 GET_ITER
>> 18 FOR_ITER 12 (to 32)

TARGET(GET_ITER) {
/* before: [obj]; after [getiter(obj)] */
PyObject *iterable = TOP(); // 出栈
PyObject *iter = PyObject_GetIter(iterable);
Py_DECREF(iterable);
SET_TOP(iter); // 入栈
if (iter == NULL)
goto error;
PREDICT(FOR_ITER);
PREDICT(CALL_FUNCTION);
DISPATCH();
}

如上,先从栈顶获取到对象,处理后再推入栈中。TOS = iter(TOS)

PyObject_GetIter

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
// object.h
typedef PyObject *(*getiterfunc) (PyObject *);

// abstract.c.3127
PyObject * PyObject_GetIter(PyObject *o)
{
PyTypeObject *t = o->ob_type;
getiterfunc f = NULL;
f = t->tp_iter; // == list_iter
if (f == NULL) {
if (PySequence_Check(o))
return PySeqIter_New(o);
return type_error("'%.200s' object is not iterable", o);
}
else {
PyObject *res = (*f)(o);
if (res != NULL && !PyIter_Check(res)) {
PyErr_Format(PyExc_TypeError,
"iter() returned non-iterator "
"of type '%.100s'",
res->ob_type->tp_name);
Py_DECREF(res);
res = NULL;
}
return res;
}
}

这里传入的 object *o,是上一步获取到的 PyListObject,通过获取 ob_type,调用其类型对象定义的tp_iter方法,即 list_iter

list_iter

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
static PyObject * list_iter(PyObject *seq)
{
listiterobject *it;

if (!PyList_Check(seq)) {
PyErr_BadInternalCall();
return NULL;
}
it = PyObject_GC_New(listiterobject, &PyListIter_Type);
if (it == NULL)
return NULL;
it->it_index = 0;
Py_INCREF(seq);
it->it_seq = (PyListObject *)seq; // 赋值
_PyObject_GC_TRACK(it);
return (PyObject *)it;
}

typedef struct {
PyObject_HEAD
Py_ssize_t it_index;
PyListObject *it_seq; /* Set to NULL when iterator is exhausted */
} listiterobject;

如上,调用 iter 方法,返回的是一个新的对象,即我们常说的迭代器。

FOR_ITER

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
    >>   18 FOR_ITER                12 (to 32)
... // 执行 print(i)
30 JUMP_ABSOLUTE 18

opcode:: FOR_ITER (delta)

TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If
this yields a new value, push it on the stack (leaving the iterator below
it). If the iterator indicates it is exhausted TOS is popped, and the byte
code counter is incremented by *delta*.

PREDICTED(FOR_ITER);
TARGET(FOR_ITER) {
/* before: [iter]; after: [iter, iter()] *or* [] */
PyObject *iter = TOP();
PyObject *next = (*iter->ob_type->tp_iternext)(iter);
if (next != NULL) {
PUSH(next);
PREDICT(STORE_FAST);
PREDICT(UNPACK_SEQUENCE);
DISPATCH();
}
STACKADJ(-1);
Py_DECREF(iter);
JUMPBY(oparg);
PREDICT(POP_BLOCK);
DISPATCH();
}

通过 GET_ITER,将一个栈顶的对象转换成迭代器。再调用 PyTypeObject 的tp_iternext方法,获取 next 值,入栈。最终处理 next==NULL 情况。

  1. 当迭代未结束时,仅仅将 next() 值压入了栈中,栈顶依然保持为迭代器。
  2. 当迭代结束时,才弹出栈顶迭代器,下一条字节码计数增加,执行相对跳转。

JUMP_ABSOLUTE

1
2
3
4
5
6
7
8
9
10
    >>   18 FOR_ITER                12 (to 32)
30 JUMP_ABSOLUTE 18

#define JUMPTO(x) (next_instr = first_instr + (x) / sizeof(_Py_CODEUNIT))

PREDICTED(JUMP_ABSOLUTE);
TARGET(JUMP_ABSOLUTE) {
JUMPTO(oparg);
DISPATCH();
}

JUMP_ABSOLUTE 18 执行绝对跳转,跳转到字节码计数为18的指令处,继续迭代。

POP_BLOCK(结束循环)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    >>   18 FOR_ITER                12 (to 32)
>> 32 POP_BLOCK

opcode:: POP_BLOCK
Removes one block from the block stack. Per frame, there is a stack of
blocks, denoting nested loops, try statements, and such.

TARGET(FOR_ITER) {
/* before: [iter]; after: [iter, iter()] *or* [] */
PyObject *iter = TOP();
PyObject *next = (*iter->ob_type->tp_iternext)(iter);
if (next != NULL) {
...
}
if (PyErr_Occurred()) {
if (!PyErr_ExceptionMatches(PyExc_StopIteration))
goto error;
else if (tstate->c_tracefunc != NULL)
call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, f);
PyErr_Clear();
}
/* iterator ended normally */
STACKADJ(-1); // stack_pointer += n
Py_DECREF(iter);
JUMPBY(oparg); // (next_instr += (x) / sizeof(_Py_CODEUNIT))
PREDICT(POP_BLOCK);
DISPATCH();
}

PREDICTED(POP_BLOCK);
TARGET(POP_BLOCK) {
PyTryBlock *b = PyFrame_BlockPop(f);
UNWIND_BLOCK(b);
DISPATCH();
}
// frameobject.c.788
PyTryBlock * PyFrame_BlockPop(PyFrameObject *f)
{
PyTryBlock *b;
if (f->f_iblock <= 0)
Py_FatalError("XXX block stack underflow");

// 指针 --,返回的是 SETUP_LOOP(PyFrame_BlockSetup) 中获取 TryBlock
b = &f->f_blockstack[--f->f_iblock];
return b;
}

#define UNWIND_BLOCK(b) \
while (STACK_LEVEL() > (b)->b_level) { \
PyObject *v = POP(); \
Py_XDECREF(v); \
}

FOR_ITER在调用时,附带有参数 12,下一条为20,JUMPBY 跳转到32,执行POP_BLOCK。从 block 堆栈中删除一个 block,恢复 f_iblock 到循环之前。每个 Frame 都会有一个 block 堆栈,表示嵌套循环、try语句等。