Add bytecode and disassembly support for Python 3.8.

Also fixes running pymultic from outside of its source directory.

CMakeLists.txt

@@ -17,7 +17,7 @@ endif()
 set(PYTHON_VERSIONS
     10 11    13 14 15 16        # Python 1.1 and 1.2 are marshal-identical
     20 21 22 23 24 25 26 27
-    30 31 32 33 34 35 36 37
+    30 31 32 33 34 35 36 37 38
 )
 
 foreach(ver ${PYTHON_VERSIONS})

bytecode.cpp

@@ -31,6 +31,7 @@ DECLARE_PYTHON(3, 4)
 DECLARE_PYTHON(3, 5)
 DECLARE_PYTHON(3, 6)
 DECLARE_PYTHON(3, 7)
+DECLARE_PYTHON(3, 8)
 
 const char* Pyc::OpcodeName(int opcode)
 {
@@ -95,6 +96,7 @@ int Pyc::ByteToOpcode(int maj, int min, int opcode)
         case 5: return python_35_map(opcode);
         case 6: return python_36_map(opcode);
         case 7: return python_37_map(opcode);
+        case 8: return python_38_map(opcode);
         }
         break;
     }

bytecode_ops.inl

@@ -91,6 +91,8 @@ OPCODE(GET_ANEXT)
 OPCODE(BEFORE_ASYNC_WITH)
 OPCODE(GET_YIELD_FROM_ITER)
 OPCODE(GET_AWAITABLE)
+OPCODE(BEGIN_FINALLY)
+OPCODE(END_ASYNC_FOR)
 
 /* Has parameter word */
 OPCODE_A_FIRST(STORE_NAME)
@@ -170,3 +172,5 @@ OPCODE_A(BUILD_STRING)
 OPCODE_A(BUILD_TUPLE_UNPACK_WITH_CALL)
 OPCODE_A(LOAD_METHOD)
 OPCODE_A(CALL_METHOD)
+OPCODE_A(CALL_FINALLY)
+OPCODE_A(POP_FINALLY)

bytes/comp_map.py

@@ -27,7 +27,7 @@
 
 maplist = [ 10, 11,     13, 14, 15, 16,
             20, 21, 22, 23, 24, 25, 26, 27,
-            30, 31, 32, 33, 34, 35, 36, 37  ]
+            30, 31, 32, 33, 34, 35, 36, 37, 38 ]
 
 for mapver in maplist:
     infile = open(os.path.join(sys.argv[1], 'python_%d.map' % mapver), 'rt')

bytes/python_38.map

@@ -0,0 +1,120 @@
+1   POP_TOP
+2   ROT_TWO
+3   ROT_THREE
+4   DUP_TOP
+5   DUP_TOP_TWO
+6   ROT_FOUR
+9   NOP
+10  UNARY_POSITIVE
+11  UNARY_NEGATIVE
+12  UNARY_NOT
+15  UNARY_INVERT
+16  BINARY_MATRIX_MULTIPLY
+17  INPLACE_MATRIX_MULTIPLY
+19  BINARY_POWER
+20  BINARY_MULTIPLY
+22  BINARY_MODULO
+23  BINARY_ADD
+24  BINARY_SUBTRACT
+25  BINARY_SUBSCR
+26  BINARY_FLOOR_DIVIDE
+27  BINARY_TRUE_DIVIDE
+28  INPLACE_FLOOR_DIVIDE
+29  INPLACE_TRUE_DIVIDE
+50  GET_AITER
+51  GET_ANEXT
+52  BEFORE_ASYNC_WITH
+53  BEGIN_FINALLY
+54  END_ASYNC_FOR
+55  INPLACE_ADD
+56  INPLACE_SUBTRACT
+57  INPLACE_MULTIPLY
+59  INPLACE_MODULO
+60  STORE_SUBSCR
+61  DELETE_SUBSCR
+62  BINARY_LSHIFT
+63  BINARY_RSHIFT
+64  BINARY_AND
+65  BINARY_XOR
+66  BINARY_OR
+67  INPLACE_POWER
+68  GET_ITER
+69  GET_YIELD_FROM_ITER
+70  PRINT_EXPR
+71  LOAD_BUILD_CLASS
+72  YIELD_FROM
+73  GET_AWAITABLE
+75  INPLACE_LSHIFT
+76  INPLACE_RSHIFT
+77  INPLACE_AND
+78  INPLACE_XOR
+79  INPLACE_OR
+81  WITH_CLEANUP_START
+82  WITH_CLEANUP_FINISH
+83  RETURN_VALUE
+84  IMPORT_STAR
+85  SETUP_ANNOTATIONS
+86  YIELD_VALUE
+87  POP_BLOCK
+88  END_FINALLY
+89  POP_EXCEPT
+90  STORE_NAME_A
+91  DELETE_NAME_A
+92  UNPACK_SEQUENCE_A
+93  FOR_ITER_A
+94  UNPACK_EX_A
+95  STORE_ATTR_A
+96  DELETE_ATTR_A
+97  STORE_GLOBAL_A
+98  DELETE_GLOBAL_A
+100 LOAD_CONST_A
+101 LOAD_NAME_A
+102 BUILD_TUPLE_A
+103 BUILD_LIST_A
+104 BUILD_SET_A
+105 BUILD_MAP_A
+106 LOAD_ATTR_A
+107 COMPARE_OP_A
+108 IMPORT_NAME_A
+109 IMPORT_FROM_A
+110 JUMP_FORWARD_A
+111 JUMP_IF_FALSE_OR_POP_A
+112 JUMP_IF_TRUE_OR_POP_A
+113 JUMP_ABSOLUTE_A
+114 POP_JUMP_IF_FALSE_A
+115 POP_JUMP_IF_TRUE_A
+116 LOAD_GLOBAL_A
+122 SETUP_FINALLY_A
+124 LOAD_FAST_A
+125 STORE_FAST_A
+126 DELETE_FAST_A
+130 RAISE_VARARGS_A
+131 CALL_FUNCTION_A
+132 MAKE_FUNCTION_A
+133 BUILD_SLICE_A
+135 LOAD_CLOSURE_A
+136 LOAD_DEREF_A
+137 STORE_DEREF_A
+138 DELETE_DEREF_A
+141 CALL_FUNCTION_KW_A
+142 CALL_FUNCTION_EX_A
+143 SETUP_WITH_A
+144 EXTENDED_ARG_A
+145 LIST_APPEND_A
+146 SET_ADD_A
+147 MAP_ADD_A
+148 LOAD_CLASSDEREF_A
+149 BUILD_LIST_UNPACK_A
+150 BUILD_MAP_UNPACK_A
+151 BUILD_MAP_UNPACK_WITH_CALL_A
+152 BUILD_TUPLE_UNPACK_A
+153 BUILD_SET_UNPACK_A
+154 SETUP_ASYNC_WITH_A
+155 FORMAT_VALUE_A
+156 BUILD_CONST_KEY_MAP_A
+157 BUILD_STRING_A
+158 BUILD_TUPLE_UNPACK_WITH_CALL_A
+160 LOAD_METHOD_A
+161 CALL_METHOD_A
+162 CALL_FINALLY_A
+163 POP_FINALLY_A

pyc_code.cpp

@@ -9,6 +9,11 @@ void PycCode::load(PycData* stream, PycModule* mod)
     else if (mod->verCompare(2, 3) >= 0)
         m_argCount = stream->get32();
 
+    if (mod->verCompare(3, 8) >= 0)
+        m_posOnlyArgCount = stream->get32();
+    else
+        m_posOnlyArgCount = 0;
+
     if (mod->majorVer() >= 3)
         m_kwOnlyArgCount = stream->get32();
     else

pyc_code.h

@@ -28,12 +28,13 @@ class PycCode : public PycObject {
     };
 
     PycCode(int type = TYPE_CODE)
-        : PycObject(type), m_argCount(0), m_kwOnlyArgCount(0), m_numLocals(0),
-          m_stackSize(0), m_flags(0), m_firstLine(0) { }
+        : PycObject(type), m_argCount(), m_posOnlyArgCount(), m_kwOnlyArgCount(),
+          m_numLocals(), m_stackSize(), m_flags(), m_firstLine() { }
 
     void load(class PycData* stream, class PycModule* mod) override;
 
     int argCount() const { return m_argCount; }
+    int posOnlyArgCount() const { return m_posOnlyArgCount; }
     int kwOnlyArgCount() const { return m_kwOnlyArgCount; }
     int numLocals() const { return m_numLocals; }
     int stackSize() const { return m_stackSize; }
@@ -50,13 +51,19 @@ class PycCode : public PycObject {
     PycRef<PycString> lnTable() const { return m_lnTable; }
 
     PycRef<PycObject> getConst(int idx) const
-    { return m_consts->get(idx); }
+    {
+        return m_consts->get(idx);
+    }
 
     PycRef<PycString> getName(int idx) const
-    { return m_names->get(idx).require_cast<PycString>(); }
+    {
+        return m_names->get(idx).require_cast<PycString>();
+    }
 
     PycRef<PycString> getVarName(int idx) const
-    { return m_varNames->get(idx).require_cast<PycString>(); }
+    {
+        return m_varNames->get(idx).require_cast<PycString>();
+    }
 
     PycRef<PycString> getCellVar(int idx) const
     {
@@ -74,7 +81,8 @@ class PycCode : public PycObject {
     }
 
 private:
-    int m_argCount, m_kwOnlyArgCount, m_numLocals, m_stackSize, m_flags;
+    int m_argCount, m_posOnlyArgCount, m_kwOnlyArgCount, m_numLocals;
+    int m_stackSize, m_flags;
     PycRef<PycString> m_code;
     PycRef<PycSequence> m_consts;
     PycRef<PycSequence> m_names;

pyc_module.cpp

@@ -146,6 +146,12 @@ void PycModule::setVersion(unsigned int magic)
         m_unicode = true;
         break;
 
+    case MAGIC_3_8:
+        m_maj = 3;
+        m_min = 8;
+        m_unicode = true;
+        break;
+
     /* Bad Magic detected */
     default:
         m_maj = -1;

pyc_module.h

@@ -30,10 +30,9 @@ enum PycMagic {
     MAGIC_3_5_3 = 0x0A0D0D17,
     MAGIC_3_6 = 0x0A0D0D33,
     MAGIC_3_7 = 0x0A0D0D42,
+    MAGIC_3_8 = 0x0A0D0D55,
 };
 
-#define PYC_VERSION(maj, min)  MAGIC_##maj##_##min
-
 class PycModule {
 public:
     PycModule() : m_maj(-1), m_min(-1), m_unicode(false) { }

pycdas.cpp

@@ -86,6 +86,8 @@ void output_object(PycRef<PycObject> obj, PycModule* mod, int indent)
             iprintf(indent + 1, "File Name: %s\n", codeObj->fileName()->value());
             iprintf(indent + 1, "Object Name: %s\n", codeObj->name()->value());
             iprintf(indent + 1, "Arg Count: %d\n", codeObj->argCount());
+            if (mod->verCompare(3, 8) >= 0)
+                iprintf(indent + 1, "Pos Only Arg Count: %d\n", codeObj->posOnlyArgCount());
             if (mod->majorVer() >= 3)
                 iprintf(indent + 1, "KW Only Arg Count: %d\n", codeObj->kwOnlyArgCount());
             iprintf(indent + 1, "Locals: %d\n", codeObj->numLocals());

scripts/pymultic

@@ -31,6 +31,7 @@ PYVERS = {
     '3.5': '3.5.7',
     '3.6': '3.6.9',
     '3.7': '3.7.4',
+    '3.8': '3.8.0',
 }
 
 OLD_PYTHONS = ('1.0', '1.1', '1.2', '1.3', '1.4', '1.5')
@@ -53,7 +54,7 @@ def fetch_python(snekdir, version):
         tarball = 'Python-{}.tgz'.format(realver)
         url = '{}/{}/{}'.format(PYURL, realver, tarball)
 
-    pyver_dir = 'Python-{}'.format(realver)
+    pyver_dir = os.path.join(snekdir, 'Python-{}'.format(realver))
     if os.path.exists(pyver_dir):
         return
 
@@ -72,16 +73,16 @@ def fetch_python(snekdir, version):
             sys.exit(1)
 
     print('Extracting Python {}...'.format(realver))
-    if subprocess.call(['tar', 'xaf', tarfile]) != 0:
+    if subprocess.call(['tar', 'xaf', tarfile], cwd=snekdir) != 0:
         sys.exit(1)
 
-    if os.path.exists('python-{}'.format(realver)) \
+    if os.path.exists(os.path.join(snekdir, 'python-{}'.format(realver))) \
             and not os.path.exists(pyver_dir):
         # The dual check prevents useless renames on case-insensitive
         # file systems
-        os.rename('python-{}'.format(realver), pyver_dir)
+        os.rename(os.path.join(snekdir, 'python-{}'.format(realver)), pyver_dir)
 
-    patch_file = '{}/python-builds/Python-{}.patch'.format(snekdir, realver)
+    patch_file = os.path.join(snekdir, 'python-builds', 'Python-{}.patch'.format(realver))
     if os.path.exists(patch_file):
         if subprocess.call(['patch', '-p1', '-i', patch_file], cwd=pyver_dir) != 0:
             sys.exit(1)