Skip to content
Permalink
Browse files

Add bytecode and disassembly support for Python 3.8.

Also fixes running pymultic from outside of its source directory.
  • Loading branch information...
zrax committed Feb 9, 2019
1 parent de78e1b commit 428d11c4b57fb03f4c5a8567993b1e850ff9997f
Showing with 419 additions and 267 deletions.
  1. +1 −1 CMakeLists.txt
  2. +257 −252 PythonBytecode.txt
  3. +2 −0 bytecode.cpp
  4. +4 −0 bytecode_ops.inl
  5. +1 −1 bytes/comp_map.py
  6. +120 −0 bytes/python_38.map
  7. +5 −0 pyc_code.cpp
  8. +14 −6 pyc_code.h
  9. +6 −0 pyc_module.cpp
  10. +1 −2 pyc_module.h
  11. +2 −0 pycdas.cpp
  12. +6 −5 scripts/pymultic
@@ -17,7 +17,7 @@ endif()
set(PYTHON_VERSIONS
10 11 13 14 15 16 # Python 1.1 and 1.2 are marshal-identical
20 21 22 23 24 25 26 27
30 31 32 33 34 35 36 37
30 31 32 33 34 35 36 37 38
)

foreach(ver ${PYTHON_VERSIONS})

Large diffs are not rendered by default.

@@ -31,6 +31,7 @@ DECLARE_PYTHON(3, 4)
DECLARE_PYTHON(3, 5)
DECLARE_PYTHON(3, 6)
DECLARE_PYTHON(3, 7)
DECLARE_PYTHON(3, 8)

const char* Pyc::OpcodeName(int opcode)
{
@@ -95,6 +96,7 @@ int Pyc::ByteToOpcode(int maj, int min, int opcode)
case 5: return python_35_map(opcode);
case 6: return python_36_map(opcode);
case 7: return python_37_map(opcode);
case 8: return python_38_map(opcode);
}
break;
}
@@ -91,6 +91,8 @@ OPCODE(GET_ANEXT)
OPCODE(BEFORE_ASYNC_WITH)
OPCODE(GET_YIELD_FROM_ITER)
OPCODE(GET_AWAITABLE)
OPCODE(BEGIN_FINALLY)
OPCODE(END_ASYNC_FOR)

/* Has parameter word */
OPCODE_A_FIRST(STORE_NAME)
@@ -170,3 +172,5 @@ OPCODE_A(BUILD_STRING)
OPCODE_A(BUILD_TUPLE_UNPACK_WITH_CALL)
OPCODE_A(LOAD_METHOD)
OPCODE_A(CALL_METHOD)
OPCODE_A(CALL_FINALLY)
OPCODE_A(POP_FINALLY)
@@ -27,7 +27,7 @@

maplist = [ 10, 11, 13, 14, 15, 16,
20, 21, 22, 23, 24, 25, 26, 27,
30, 31, 32, 33, 34, 35, 36, 37 ]
30, 31, 32, 33, 34, 35, 36, 37, 38 ]

for mapver in maplist:
infile = open(os.path.join(sys.argv[1], 'python_%d.map' % mapver), 'rt')
@@ -0,0 +1,120 @@
1 POP_TOP
2 ROT_TWO
3 ROT_THREE
4 DUP_TOP
5 DUP_TOP_TWO
6 ROT_FOUR
9 NOP
10 UNARY_POSITIVE
11 UNARY_NEGATIVE
12 UNARY_NOT
15 UNARY_INVERT
16 BINARY_MATRIX_MULTIPLY
17 INPLACE_MATRIX_MULTIPLY
19 BINARY_POWER
20 BINARY_MULTIPLY
22 BINARY_MODULO
23 BINARY_ADD
24 BINARY_SUBTRACT
25 BINARY_SUBSCR
26 BINARY_FLOOR_DIVIDE
27 BINARY_TRUE_DIVIDE
28 INPLACE_FLOOR_DIVIDE
29 INPLACE_TRUE_DIVIDE
50 GET_AITER
51 GET_ANEXT
52 BEFORE_ASYNC_WITH
53 BEGIN_FINALLY
54 END_ASYNC_FOR
55 INPLACE_ADD
56 INPLACE_SUBTRACT
57 INPLACE_MULTIPLY
59 INPLACE_MODULO
60 STORE_SUBSCR
61 DELETE_SUBSCR
62 BINARY_LSHIFT
63 BINARY_RSHIFT
64 BINARY_AND
65 BINARY_XOR
66 BINARY_OR
67 INPLACE_POWER
68 GET_ITER
69 GET_YIELD_FROM_ITER
70 PRINT_EXPR
71 LOAD_BUILD_CLASS
72 YIELD_FROM
73 GET_AWAITABLE
75 INPLACE_LSHIFT
76 INPLACE_RSHIFT
77 INPLACE_AND
78 INPLACE_XOR
79 INPLACE_OR
81 WITH_CLEANUP_START
82 WITH_CLEANUP_FINISH
83 RETURN_VALUE
84 IMPORT_STAR
85 SETUP_ANNOTATIONS
86 YIELD_VALUE
87 POP_BLOCK
88 END_FINALLY
89 POP_EXCEPT
90 STORE_NAME_A
91 DELETE_NAME_A
92 UNPACK_SEQUENCE_A
93 FOR_ITER_A
94 UNPACK_EX_A
95 STORE_ATTR_A
96 DELETE_ATTR_A
97 STORE_GLOBAL_A
98 DELETE_GLOBAL_A
100 LOAD_CONST_A
101 LOAD_NAME_A
102 BUILD_TUPLE_A
103 BUILD_LIST_A
104 BUILD_SET_A
105 BUILD_MAP_A
106 LOAD_ATTR_A
107 COMPARE_OP_A
108 IMPORT_NAME_A
109 IMPORT_FROM_A
110 JUMP_FORWARD_A
111 JUMP_IF_FALSE_OR_POP_A
112 JUMP_IF_TRUE_OR_POP_A
113 JUMP_ABSOLUTE_A
114 POP_JUMP_IF_FALSE_A
115 POP_JUMP_IF_TRUE_A
116 LOAD_GLOBAL_A
122 SETUP_FINALLY_A
124 LOAD_FAST_A
125 STORE_FAST_A
126 DELETE_FAST_A
130 RAISE_VARARGS_A
131 CALL_FUNCTION_A
132 MAKE_FUNCTION_A
133 BUILD_SLICE_A
135 LOAD_CLOSURE_A
136 LOAD_DEREF_A
137 STORE_DEREF_A
138 DELETE_DEREF_A
141 CALL_FUNCTION_KW_A
142 CALL_FUNCTION_EX_A
143 SETUP_WITH_A
144 EXTENDED_ARG_A
145 LIST_APPEND_A
146 SET_ADD_A
147 MAP_ADD_A
148 LOAD_CLASSDEREF_A
149 BUILD_LIST_UNPACK_A
150 BUILD_MAP_UNPACK_A
151 BUILD_MAP_UNPACK_WITH_CALL_A
152 BUILD_TUPLE_UNPACK_A
153 BUILD_SET_UNPACK_A
154 SETUP_ASYNC_WITH_A
155 FORMAT_VALUE_A
156 BUILD_CONST_KEY_MAP_A
157 BUILD_STRING_A
158 BUILD_TUPLE_UNPACK_WITH_CALL_A
160 LOAD_METHOD_A
161 CALL_METHOD_A
162 CALL_FINALLY_A
163 POP_FINALLY_A
@@ -9,6 +9,11 @@ void PycCode::load(PycData* stream, PycModule* mod)
else if (mod->verCompare(2, 3) >= 0)
m_argCount = stream->get32();

if (mod->verCompare(3, 8) >= 0)
m_posOnlyArgCount = stream->get32();
else
m_posOnlyArgCount = 0;

if (mod->majorVer() >= 3)
m_kwOnlyArgCount = stream->get32();
else
@@ -28,12 +28,13 @@ class PycCode : public PycObject {
};

PycCode(int type = TYPE_CODE)
: PycObject(type), m_argCount(0), m_kwOnlyArgCount(0), m_numLocals(0),
m_stackSize(0), m_flags(0), m_firstLine(0) { }
: PycObject(type), m_argCount(), m_posOnlyArgCount(), m_kwOnlyArgCount(),
m_numLocals(), m_stackSize(), m_flags(), m_firstLine() { }

void load(class PycData* stream, class PycModule* mod) override;

int argCount() const { return m_argCount; }
int posOnlyArgCount() const { return m_posOnlyArgCount; }
int kwOnlyArgCount() const { return m_kwOnlyArgCount; }
int numLocals() const { return m_numLocals; }
int stackSize() const { return m_stackSize; }
@@ -50,13 +51,19 @@ class PycCode : public PycObject {
PycRef<PycString> lnTable() const { return m_lnTable; }

PycRef<PycObject> getConst(int idx) const
{ return m_consts->get(idx); }
{
return m_consts->get(idx);
}

PycRef<PycString> getName(int idx) const
{ return m_names->get(idx).require_cast<PycString>(); }
{
return m_names->get(idx).require_cast<PycString>();
}

PycRef<PycString> getVarName(int idx) const
{ return m_varNames->get(idx).require_cast<PycString>(); }
{
return m_varNames->get(idx).require_cast<PycString>();
}

PycRef<PycString> getCellVar(int idx) const
{
@@ -74,7 +81,8 @@ class PycCode : public PycObject {
}

private:
int m_argCount, m_kwOnlyArgCount, m_numLocals, m_stackSize, m_flags;
int m_argCount, m_posOnlyArgCount, m_kwOnlyArgCount, m_numLocals;
int m_stackSize, m_flags;
PycRef<PycString> m_code;
PycRef<PycSequence> m_consts;
PycRef<PycSequence> m_names;
@@ -146,6 +146,12 @@ void PycModule::setVersion(unsigned int magic)
m_unicode = true;
break;

case MAGIC_3_8:
m_maj = 3;
m_min = 8;
m_unicode = true;
break;

/* Bad Magic detected */
default:
m_maj = -1;
@@ -30,10 +30,9 @@ enum PycMagic {
MAGIC_3_5_3 = 0x0A0D0D17,
MAGIC_3_6 = 0x0A0D0D33,
MAGIC_3_7 = 0x0A0D0D42,
MAGIC_3_8 = 0x0A0D0D55,
};

#define PYC_VERSION(maj, min) MAGIC_##maj##_##min

class PycModule {
public:
PycModule() : m_maj(-1), m_min(-1), m_unicode(false) { }
@@ -86,6 +86,8 @@ void output_object(PycRef<PycObject> obj, PycModule* mod, int indent)
iprintf(indent + 1, "File Name: %s\n", codeObj->fileName()->value());
iprintf(indent + 1, "Object Name: %s\n", codeObj->name()->value());
iprintf(indent + 1, "Arg Count: %d\n", codeObj->argCount());
if (mod->verCompare(3, 8) >= 0)
iprintf(indent + 1, "Pos Only Arg Count: %d\n", codeObj->posOnlyArgCount());
if (mod->majorVer() >= 3)
iprintf(indent + 1, "KW Only Arg Count: %d\n", codeObj->kwOnlyArgCount());
iprintf(indent + 1, "Locals: %d\n", codeObj->numLocals());
@@ -31,6 +31,7 @@ PYVERS = {
'3.5': '3.5.7',
'3.6': '3.6.9',
'3.7': '3.7.4',
'3.8': '3.8.0',
}

OLD_PYTHONS = ('1.0', '1.1', '1.2', '1.3', '1.4', '1.5')
@@ -53,7 +54,7 @@ def fetch_python(snekdir, version):
tarball = 'Python-{}.tgz'.format(realver)
url = '{}/{}/{}'.format(PYURL, realver, tarball)

pyver_dir = 'Python-{}'.format(realver)
pyver_dir = os.path.join(snekdir, 'Python-{}'.format(realver))
if os.path.exists(pyver_dir):
return

@@ -72,16 +73,16 @@ def fetch_python(snekdir, version):
sys.exit(1)

print('Extracting Python {}...'.format(realver))
if subprocess.call(['tar', 'xaf', tarfile]) != 0:
if subprocess.call(['tar', 'xaf', tarfile], cwd=snekdir) != 0:
sys.exit(1)

if os.path.exists('python-{}'.format(realver)) \
if os.path.exists(os.path.join(snekdir, 'python-{}'.format(realver))) \
and not os.path.exists(pyver_dir):
# The dual check prevents useless renames on case-insensitive
# file systems
os.rename('python-{}'.format(realver), pyver_dir)
os.rename(os.path.join(snekdir, 'python-{}'.format(realver)), pyver_dir)

patch_file = '{}/python-builds/Python-{}.patch'.format(snekdir, realver)
patch_file = os.path.join(snekdir, 'python-builds', 'Python-{}.patch'.format(realver))
if os.path.exists(patch_file):
if subprocess.call(['patch', '-p1', '-i', patch_file], cwd=pyver_dir) != 0:
sys.exit(1)

0 comments on commit 428d11c

Please sign in to comment.
You can’t perform that action at this time.