From 881ec1e27941c41c5084c15a42165559b7f002f6 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 19 Sep 2016 16:13:07 -0700 Subject: [PATCH] Fixes #1648 Fix debug-attach and profiling for Python 3.6 --- Python/Product/Core/Core.csproj | 16 ++++++ .../Product/PyDebugAttach/PyDebugAttach.cpp | 25 ++++---- Python/Product/VsPyProf/PythonApi.cpp | 16 +++++- Python/Product/VsPyProf/python.h | 57 +++++++++++++++---- Python/Tests/Core/DebugReplEvaluatorTests.cs | 17 +++++- Python/Tests/DebuggerTests/AttachTests.cs | 31 +++++++++- Python/Tests/DebuggerTests/DebuggerTests.cs | 18 ++++++ .../Tests/ProfilingUITests/ProfilingTests.cs | 19 +++++++ 8 files changed, 167 insertions(+), 32 deletions(-) diff --git a/Python/Product/Core/Core.csproj b/Python/Product/Core/Core.csproj index 11629c61ae..8a30bb6056 100644 --- a/Python/Product/Core/Core.csproj +++ b/Python/Product/Core/Core.csproj @@ -77,6 +77,14 @@ {89d51398-a003-44ba-b1b2-cfc6f8396d7e} Microsoft.PythonTools.BuildTasks + + {25956dfa-17a2-4109-b9e5-d46cce1ed52f} + DebuggerHelper + + + {a2a795f7-27d0-4801-88da-95b368f070ad} + DebuggerHelperX86 + {DECC7971-FA58-4DB0-9561-BFFADD393BBD} Microsoft.PythonTools.Debugger @@ -93,6 +101,14 @@ {3814d9db-10e6-4478-bd98-6c5840612af8} Microsoft.PythonTools.ProjectWizards + + {ac19caa0-5c69-4b20-8a18-d4b6b65f22b8} + PyDebugAttach + + + {70e7eb43-81d3-4aa0-9870-0b304732aff2} + PyDebugAttachX86 + {fa7be5f5-e04f-4613-b7ac-70ce10d1bb68} Microsoft.PythonTools diff --git a/Python/Product/PyDebugAttach/PyDebugAttach.cpp b/Python/Product/PyDebugAttach/PyDebugAttach.cpp index dea09ba4ae..6aab7eaa21 100644 --- a/Python/Product/PyDebugAttach/PyDebugAttach.cpp +++ b/Python/Product/PyDebugAttach/PyDebugAttach.cpp @@ -722,8 +722,8 @@ long GetPythonThreadId(PythonVersion version, PyThreadState* curThread) { threadId = ((PyThreadState_25_27*)curThread)->thread_id; } else if (PyThreadState_30_33::IsFor(version)) { threadId = ((PyThreadState_30_33*)curThread)->thread_id; - } else if (PyThreadState_34_35::IsFor(version)) { - threadId = ((PyThreadState_34_35*)curThread)->thread_id; + } else if (PyThreadState_34_36::IsFor(version)) { + threadId = ((PyThreadState_34_36*)curThread)->thread_id; } return threadId; } @@ -855,8 +855,6 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) { auto getThreadTls = (PyThread_get_key_value*)GetProcAddress(module, "PyThread_get_key_value"); auto setThreadTls = (PyThread_set_key_value*)GetProcAddress(module, "PyThread_set_key_value"); auto delThreadTls = (PyThread_delete_key_value*)GetProcAddress(module, "PyThread_delete_key_value"); - auto pyGilStateEnsure = (PyGILState_EnsureFunc*)GetProcAddress(module, "PyGILState_Ensure"); - auto pyGilStateRelease = (PyGILState_ReleaseFunc*)GetProcAddress(module, "PyGILState_Release"); auto PyCFrame_Type = (PyTypeObject*)GetProcAddress(module, "PyCFrame_Type"); if (addPendingCall == nullptr || curPythonThread == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr || @@ -864,8 +862,7 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) { pyDictNew == nullptr || pyCompileString == nullptr || pyEvalCode == nullptr || getDictItem == nullptr || call == nullptr || getBuiltins == nullptr || dictSetItem == nullptr || intFromLong == nullptr || pyErrRestore == nullptr || pyErrFetch == nullptr || errOccurred == nullptr || pyImportMod == nullptr || pyGetAttr == nullptr || pyNone == nullptr || pySetAttr == nullptr || boolFromLong == nullptr || - getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr || - pyGilStateEnsure == nullptr || pyGilStateRelease == nullptr) { + getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr) { // we're missing some APIs, we cannot attach. connInfo.ReportError(ConnError_PythonNotFound); return false; @@ -975,13 +972,13 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) { // Py_InitThreads to bring up multi-threading. // Some context here: http://bugs.python.org/issue11329 // http://pytools.codeplex.com/workitem/834 - gilState = pyGilStateEnsure(); + gilState = gilEnsure(); } initThreads(); if (version >= PythonVersion_32) { // we will release the GIL here - pyGilStateRelease(gilState); + gilRelease(gilState); } else { releaseLock(); } @@ -1149,8 +1146,8 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) { frame = ((PyThreadState_25_27*)curThread)->frame; } else if (PyThreadState_30_33::IsFor(version)) { frame = ((PyThreadState_30_33*)curThread)->frame; - } else if (PyThreadState_34_35::IsFor(version)) { - frame = ((PyThreadState_34_35*)curThread)->frame; + } else if (PyThreadState_34_36::IsFor(version)) { + frame = ((PyThreadState_34_36*)curThread)->frame; } auto threadObj = PyObjectHolder(isDebug, call(new_thread.ToPython(), pyThreadId.ToPython(), pyTrue, frame, NULL)); @@ -1342,8 +1339,8 @@ int TraceGeneral(int interpreterId, PyObject *obj, PyFrameObject *frame, int wha ((PyThreadState_25_27*)curThread)->c_tracefunc(((PyThreadState_25_27*)curThread)->c_traceobj, frame, what, arg); } else if (PyThreadState_30_33::IsFor(version)) { ((PyThreadState_30_33*)curThread)->c_tracefunc(((PyThreadState_30_33*)curThread)->c_traceobj, frame, what, arg); - } else if (PyThreadState_34_35::IsFor(version)) { - ((PyThreadState_34_35*)curThread)->c_tracefunc(((PyThreadState_34_35*)curThread)->c_traceobj, frame, what, arg); + } else if (PyThreadState_34_36::IsFor(version)) { + ((PyThreadState_34_36*)curThread)->c_tracefunc(((PyThreadState_34_36*)curThread)->c_traceobj, frame, what, arg); } } return 0; @@ -1387,8 +1384,8 @@ void SetInitialTraceFunc(DWORD interpreterId, PyThreadState *thread) { gilstate_counter = ((PyThreadState_25_27*)thread)->gilstate_counter; } else if (PyThreadState_30_33::IsFor(version)) { gilstate_counter = ((PyThreadState_30_33*)thread)->gilstate_counter; - } else if (PyThreadState_34_35::IsFor(version)) { - gilstate_counter = ((PyThreadState_34_35*)thread)->gilstate_counter; + } else if (PyThreadState_34_36::IsFor(version)) { + gilstate_counter = ((PyThreadState_34_36*)thread)->gilstate_counter; } if (gilstate_counter == 1) { diff --git a/Python/Product/VsPyProf/PythonApi.cpp b/Python/Product/VsPyProf/PythonApi.cpp index 5e4f7e9b67..c994ae3bf5 100644 --- a/Python/Product/VsPyProf/PythonApi.cpp +++ b/Python/Product/VsPyProf/PythonApi.cpp @@ -88,7 +88,7 @@ VsPyProf* VsPyProf::Create(HMODULE pythonModule) { } if ((major == 2 && (minor >= 4 && minor <= 7)) || - (major == 3 && (minor >= 0 && minor <= 5))) { + (major == 3 && (minor >= 0 && minor <= 6))) { return new VsPyProf(pythonModule, major, minor, @@ -134,6 +134,8 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR& filename = ((PyCodeObject30_32*)codeObj)->co_filename; } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) { filename = ((PyCodeObject33_35*)codeObj)->co_filename; + } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) { + filename = ((PyCodeObject36*)codeObj)->co_filename; } module = (DWORD_PTR)filename; @@ -210,6 +212,9 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR& } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) { RegisterName(func, ((PyCodeObject33_35*)codeObj)->co_name, &moduleName); lineno = ((PyCodeObject33_35*)codeObj)->co_firstlineno; + } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) { + RegisterName(func, ((PyCodeObject36*)codeObj)->co_name, &moduleName); + lineno = ((PyCodeObject36*)codeObj)->co_firstlineno; } // give the profiler the line number of this function @@ -242,6 +247,9 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) { argCount = ((PyCodeObject33_35*)codeObj)->co_argcount; argNames = (PyTupleObject*)((PyCodeObject33_35*)codeObj)->co_varnames; + } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) { + argCount = ((PyCodeObject36*)codeObj)->co_argcount; + argNames = (PyTupleObject*)((PyCodeObject36*)codeObj)->co_varnames; } if (argCount != 0 && argNames->ob_type == PyTuple_Type) { @@ -252,8 +260,8 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO PyObject* self = nullptr; if (PyFrameObject25_33::IsFor(MajorVersion, MinorVersion)) { self = ((PyFrameObject25_33*)frameObj)->f_localsplus[0]; - } else if (PyFrameObject34_35::IsFor(MajorVersion, MinorVersion)) { - self = ((PyFrameObject34_35*)frameObj)->f_localsplus[0]; + } else if (PyFrameObject34_36::IsFor(MajorVersion, MinorVersion)) { + self = ((PyFrameObject34_36*)frameObj)->f_localsplus[0]; } return GetClassNameFromSelf(self, codeObj); } @@ -281,6 +289,8 @@ wstring VsPyProf::GetClassNameFromSelf(PyObject* self, PyObject *codeObj) { nameObj = ((PyCodeObject30_32*)codeObj)->co_name; } else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) { nameObj = ((PyCodeObject33_35*)codeObj)->co_name; + } else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) { + nameObj = ((PyCodeObject36*)codeObj)->co_name; } GetNameAscii(nameObj, codeName); diff --git a/Python/Product/VsPyProf/python.h b/Python/Product/VsPyProf/python.h index 0ac32ad22e..372e060b34 100644 --- a/Python/Product/VsPyProf/python.h +++ b/Python/Product/VsPyProf/python.h @@ -28,7 +28,8 @@ enum PythonVersion { PythonVersion_32 = 0x0302, PythonVersion_33 = 0x0303, PythonVersion_34 = 0x0304, - PythonVersion_35 = 0x0305 + PythonVersion_35 = 0x0305, + PythonVersion_36 = 0x0306 }; @@ -144,7 +145,38 @@ class PyCodeObject33_35 : public PyObject { } }; -// 2.5 - 3.1 +// 3.6 +class PyCodeObject36 : public PyObject { +public: + int co_argcount; /* #arguments, except *args */ + int co_kwonlyargcount; /* #keyword only arguments */ + int co_nlocals; /* #local variables */ + int co_stacksize; /* #entries needed for evaluation stack */ + int co_flags; /* CO_..., see below */ + int co_firstlineno; /* first source line number */ + PyObject *co_code; /* instruction opcodes */ + PyObject *co_consts; /* list (constants used) */ + PyObject *co_names; /* list of strings (names used) */ + PyObject *co_varnames; /* tuple of strings (local variable names) */ + PyObject *co_freevars; /* tuple of strings (free variable names) */ + PyObject *co_cellvars; /* tuple of strings (cell variable names) */ + /* The rest doesn't count for hash or comparisons */ + unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */ + PyObject *co_filename; /* unicode (where it was loaded from) */ + PyObject *co_name; /* unicode (name, for reference) */ + PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */ + void *co_zombieframe; /* for optimization only (see frameobject.c) */ + + static bool IsFor(int majorVersion, int minorVersion) { + return majorVersion == 3 && minorVersion >= 6; + } + + static bool IsFor(PythonVersion version) { + return version >= PythonVersion_36; + } +}; + +// 2.5 - 3.6 class PyFunctionObject : public PyObject { public: PyObject *func_code; /* A code object */ @@ -175,7 +207,7 @@ typedef struct { long hash; /* Hash value; -1 if not set */ } PyUnicodeObject; -// 2.4 - 3.5 compatible +// 2.4 - 3.6 compatible class PyFrameObject : public PyVarObject { public: PyFrameObject *f_back; /* previous frame, or NULL */ @@ -216,7 +248,7 @@ class PyFrameObject25_33 : public PyFrameObject { } }; -class PyFrameObject34_35 : public PyFrameObject { +class PyFrameObject34_36 : public PyFrameObject { public: /* Borrowed reference to a generator, or NULL */ PyObject *f_gen; @@ -231,14 +263,14 @@ class PyFrameObject34_35 : public PyFrameObject { PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ static bool IsFor(int majorVersion, int minorVersion) { - return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5; + return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6; } }; typedef void (*destructor)(PyObject *); -// 2.4 - 3.5 +// 2.4 - 3.6 class PyMethodDef { public: char *ml_name; /* The name of the built-in function/method */ @@ -261,7 +293,7 @@ class PyTypeObject : public PyVarObject { void* tp_setattr; union { void* tp_compare; /* 2.4 - 3.4 */ - void* tp_as_async; /* 3.5 */ + void* tp_as_async; /* 3.5 - 3.6 */ }; void* tp_repr; @@ -331,7 +363,7 @@ class PyTypeObject : public PyVarObject { unsigned int tp_version_tag; }; -// 2.4 - 3.5 +// 2.4 - 3.6 class PyTupleObject : public PyVarObject { public: PyObject *ob_item[1]; @@ -342,7 +374,7 @@ class PyTupleObject : public PyVarObject { */ }; -// 2.4 - 3.5 +// 2.4 - 3.6 class PyCFunctionObject : public PyObject { public: PyMethodDef *m_ml; /* Description of the C function to call */ @@ -473,7 +505,7 @@ class PyThreadState_30_33 : public PyThreadState { } }; -class PyThreadState_34_35 : public PyThreadState { +class PyThreadState_34_36 : public PyThreadState { public: PyThreadState *prev; PyThreadState *next; @@ -513,11 +545,11 @@ class PyThreadState_34_35 : public PyThreadState { /* XXX signal handlers should also be here */ static bool IsFor(int majorVersion, int minorVersion) { - return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5; + return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6; } static bool IsFor(PythonVersion version) { - return version >= PythonVersion_34 && version <= PythonVersion_35; + return version >= PythonVersion_34 && version <= PythonVersion_36; } }; @@ -570,6 +602,7 @@ static PythonVersion GetPythonVersion(HMODULE hMod) { case '3': return PythonVersion_33; case '4': return PythonVersion_34; case '5': return PythonVersion_35; + case '6': return PythonVersion_36; } } } diff --git a/Python/Tests/Core/DebugReplEvaluatorTests.cs b/Python/Tests/Core/DebugReplEvaluatorTests.cs index 698ab617a6..00667ac18a 100644 --- a/Python/Tests/Core/DebugReplEvaluatorTests.cs +++ b/Python/Tests/Core/DebugReplEvaluatorTests.cs @@ -116,7 +116,7 @@ public void ErrorInInput() { Assert.AreEqual(@"Traceback (most recent call last): File """", line 1, in NameError: name 'does_not_exist' is not defined -", _window.Error); +".Replace("\r\n", "\n"), _window.Error.Replace("\r\n", "\n")); } [TestMethod, Priority(3)] @@ -434,6 +434,21 @@ internal override PythonVersion Version { } } + [TestClass] + public class DebugReplEvaluatorTests36 : DebugReplEvaluatorTests { + [ClassInitialize] + public static new void DoDeployment(TestContext context) { + AssertListener.Initialize(); + PythonTestData.Deploy(); + } + + internal override PythonVersion Version { + get { + return PythonPaths.Python36 ?? PythonPaths.Python36_x64; + } + } + } + [TestClass] public class DebugReplEvaluatorTests27 : DebugReplEvaluatorTests { [ClassInitialize] diff --git a/Python/Tests/DebuggerTests/AttachTests.cs b/Python/Tests/DebuggerTests/AttachTests.cs index 0f8eb0a315..b14f2eb59a 100644 --- a/Python/Tests/DebuggerTests/AttachTests.cs +++ b/Python/Tests/DebuggerTests/AttachTests.cs @@ -807,7 +807,10 @@ public void AttachPtvsd() { string script = TestData.GetPath(@"TestData\DebuggerProject\AttachPtvsd.py"); var psi = new ProcessStartInfo(Version.InterpreterPath, PtvsdInterpreterArguments + " \"" + script + "\"") { WorkingDirectory = TestData.GetPath(), - UseShellExecute = false + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true }; var p = Process.Start(psi); @@ -822,7 +825,7 @@ public void AttachPtvsd() { break; } catch (SocketException) { // Failed to connect - the process might have not started yet, so keep trying a few more times. - if (i >= 5) { + if (i >= 5 || p.HasExited) { throw; } } @@ -862,6 +865,8 @@ public void AttachPtvsd() { DetachProcess(proc); } } finally { + Console.WriteLine(p.StandardOutput.ReadToEnd()); + Console.WriteLine(p.StandardError.ReadToEnd()); DisposeProcess(p); } } @@ -1046,6 +1051,28 @@ internal override PythonVersion Version { } } + [TestClass] + public class AttachTests36 : AttachTests { + internal override PythonVersion Version { + get { + return PythonPaths.Python36; + } + } + + public override void AttachNewThread_PyThreadState_New() { + // PyEval_AcquireLock deprecated in 3.2 + } + } + + [TestClass] + public class AttachTests36_x64 : AttachTests35 { + internal override PythonVersion Version { + get { + return PythonPaths.Python36_x64; + } + } + } + [TestClass] public class AttachTests25 : AttachTests { internal override PythonVersion Version { diff --git a/Python/Tests/DebuggerTests/DebuggerTests.cs b/Python/Tests/DebuggerTests/DebuggerTests.cs index cfa2ab28ad..50047b8459 100644 --- a/Python/Tests/DebuggerTests/DebuggerTests.cs +++ b/Python/Tests/DebuggerTests/DebuggerTests.cs @@ -2115,6 +2115,24 @@ internal override PythonVersion Version { } } + [TestClass] + public class DebuggerTests36 : DebuggerTests3x { + internal override PythonVersion Version { + get { + return PythonPaths.Python36; + } + } + } + + [TestClass] + public class DebuggerTests36_x64 : DebuggerTests36 { + internal override PythonVersion Version { + get { + return PythonPaths.Python36_x64; + } + } + } + [TestClass] public class DebuggerTests25 : DebuggerTests { internal override PythonVersion Version { diff --git a/Python/Tests/ProfilingUITests/ProfilingTests.cs b/Python/Tests/ProfilingUITests/ProfilingTests.cs index 4aeae6155e..fb44709c8c 100644 --- a/Python/Tests/ProfilingUITests/ProfilingTests.cs +++ b/Python/Tests/ProfilingUITests/ProfilingTests.cs @@ -1571,6 +1571,25 @@ public void BuiltinsProfilePython35x64() { ); } + [TestMethod, Priority(1)] + [HostType("VSTestHost"), TestCategory("Installed")] + public void BuiltinsProfilePython36() { + BuiltinsProfile( + PythonPaths.Python36, + new[] { "BuiltinsProfile.f", "str.startswith", "isinstance", "marshal.dumps", "array.array.tostring" }, + new[] { "compile", "exec", "execfile", "_io.TextIOWrapper.read" } + ); + } + + [TestMethod, Priority(1)] + [HostType("VSTestHost"), TestCategory("Installed")] + public void BuiltinsProfilePython36x64() { + BuiltinsProfile( + PythonPaths.Python36_x64, + new[] { "BuiltinsProfile.f", "str.startswith", "isinstance", "marshal.dumps", "array.array.tostring" }, + new[] { "compile", "exec", "execfile", "_io.TextIOWrapper.read" } + ); + } [TestMethod, Priority(1)] [HostType("VSTestHost"), TestCategory("Installed")] public void Python64Bit() {