Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixes #1648 Fix debug-attach and profiling for Python 3.6 #1650

Merged
merged 1 commit into from
Sep 22, 2016
Merged

Fixes #1648 Fix debug-attach and profiling for Python 3.6 #1650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions Python/Product/Core/Core.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,14 @@
<Project>{89d51398-a003-44ba-b1b2-cfc6f8396d7e}</Project>
<Name>Microsoft.PythonTools.BuildTasks</Name>
</ProjectReference>
<ProjectReference Include="..\DebuggerHelper\DebuggerHelper.vcxproj">
<Project>{25956dfa-17a2-4109-b9e5-d46cce1ed52f}</Project>
<Name>DebuggerHelper</Name>
</ProjectReference>
<ProjectReference Include="..\DebuggerHelper\DebuggerHelperX86.vcxproj">
<Project>{a2a795f7-27d0-4801-88da-95b368f070ad}</Project>
<Name>DebuggerHelperX86</Name>
</ProjectReference>
<ProjectReference Include="..\Debugger\Debugger.csproj">
<Project>{DECC7971-FA58-4DB0-9561-BFFADD393BBD}</Project>
<Name>Microsoft.PythonTools.Debugger</Name>
Expand All @@ -93,6 +101,14 @@
<Project>{3814d9db-10e6-4478-bd98-6c5840612af8}</Project>
<Name>Microsoft.PythonTools.ProjectWizards</Name>
</ProjectReference>
<ProjectReference Include="..\PyDebugAttach\PyDebugAttach.vcxproj">
<Project>{ac19caa0-5c69-4b20-8a18-d4b6b65f22b8}</Project>
<Name>PyDebugAttach</Name>
</ProjectReference>
<ProjectReference Include="..\PyDebugAttach\PyDebugAttachX86.vcxproj">
<Project>{70e7eb43-81d3-4aa0-9870-0b304732aff2}</Project>
<Name>PyDebugAttachX86</Name>
</ProjectReference>
<ProjectReference Include="..\PythonTools\PythonTools.csproj">
<Project>{fa7be5f5-e04f-4613-b7ac-70ce10d1bb68}</Project>
<Name>Microsoft.PythonTools</Name>
Expand Down
25 changes: 11 additions & 14 deletions Python/Product/PyDebugAttach/PyDebugAttach.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,8 +722,8 @@ long GetPythonThreadId(PythonVersion version, PyThreadState* curThread) {
threadId = ((PyThreadState_25_27*)curThread)->thread_id;
} else if (PyThreadState_30_33::IsFor(version)) {
threadId = ((PyThreadState_30_33*)curThread)->thread_id;
} else if (PyThreadState_34_35::IsFor(version)) {
threadId = ((PyThreadState_34_35*)curThread)->thread_id;
} else if (PyThreadState_34_36::IsFor(version)) {
threadId = ((PyThreadState_34_36*)curThread)->thread_id;
}
return threadId;
}
Expand Down Expand Up @@ -855,17 +855,14 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
auto getThreadTls = (PyThread_get_key_value*)GetProcAddress(module, "PyThread_get_key_value");
auto setThreadTls = (PyThread_set_key_value*)GetProcAddress(module, "PyThread_set_key_value");
auto delThreadTls = (PyThread_delete_key_value*)GetProcAddress(module, "PyThread_delete_key_value");
auto pyGilStateEnsure = (PyGILState_EnsureFunc*)GetProcAddress(module, "PyGILState_Ensure");
auto pyGilStateRelease = (PyGILState_ReleaseFunc*)GetProcAddress(module, "PyGILState_Release");
auto PyCFrame_Type = (PyTypeObject*)GetProcAddress(module, "PyCFrame_Type");

if (addPendingCall == nullptr || curPythonThread == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr ||
initThreads == nullptr || releaseLock == nullptr || threadsInited == nullptr || threadNext == nullptr || threadSwap == nullptr ||
pyDictNew == nullptr || pyCompileString == nullptr || pyEvalCode == nullptr || getDictItem == nullptr || call == nullptr ||
getBuiltins == nullptr || dictSetItem == nullptr || intFromLong == nullptr || pyErrRestore == nullptr || pyErrFetch == nullptr ||
errOccurred == nullptr || pyImportMod == nullptr || pyGetAttr == nullptr || pyNone == nullptr || pySetAttr == nullptr || boolFromLong == nullptr ||
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr ||
pyGilStateEnsure == nullptr || pyGilStateRelease == nullptr) {
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr) {
// we're missing some APIs, we cannot attach.
connInfo.ReportError(ConnError_PythonNotFound);
return false;
Expand Down Expand Up @@ -975,13 +972,13 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
// Py_InitThreads to bring up multi-threading.
// Some context here: http://bugs.python.org/issue11329
// http://pytools.codeplex.com/workitem/834
gilState = pyGilStateEnsure();
gilState = gilEnsure();
}
initThreads();

if (version >= PythonVersion_32) {
// we will release the GIL here
pyGilStateRelease(gilState);
gilRelease(gilState);
} else {
releaseLock();
}
Expand Down Expand Up @@ -1149,8 +1146,8 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
frame = ((PyThreadState_25_27*)curThread)->frame;
} else if (PyThreadState_30_33::IsFor(version)) {
frame = ((PyThreadState_30_33*)curThread)->frame;
} else if (PyThreadState_34_35::IsFor(version)) {
frame = ((PyThreadState_34_35*)curThread)->frame;
} else if (PyThreadState_34_36::IsFor(version)) {
frame = ((PyThreadState_34_36*)curThread)->frame;
}

auto threadObj = PyObjectHolder(isDebug, call(new_thread.ToPython(), pyThreadId.ToPython(), pyTrue, frame, NULL));
Expand Down Expand Up @@ -1342,8 +1339,8 @@ int TraceGeneral(int interpreterId, PyObject *obj, PyFrameObject *frame, int wha
((PyThreadState_25_27*)curThread)->c_tracefunc(((PyThreadState_25_27*)curThread)->c_traceobj, frame, what, arg);
} else if (PyThreadState_30_33::IsFor(version)) {
((PyThreadState_30_33*)curThread)->c_tracefunc(((PyThreadState_30_33*)curThread)->c_traceobj, frame, what, arg);
} else if (PyThreadState_34_35::IsFor(version)) {
((PyThreadState_34_35*)curThread)->c_tracefunc(((PyThreadState_34_35*)curThread)->c_traceobj, frame, what, arg);
} else if (PyThreadState_34_36::IsFor(version)) {
((PyThreadState_34_36*)curThread)->c_tracefunc(((PyThreadState_34_36*)curThread)->c_traceobj, frame, what, arg);
}
}
return 0;
Expand Down Expand Up @@ -1387,8 +1384,8 @@ void SetInitialTraceFunc(DWORD interpreterId, PyThreadState *thread) {
gilstate_counter = ((PyThreadState_25_27*)thread)->gilstate_counter;
} else if (PyThreadState_30_33::IsFor(version)) {
gilstate_counter = ((PyThreadState_30_33*)thread)->gilstate_counter;
} else if (PyThreadState_34_35::IsFor(version)) {
gilstate_counter = ((PyThreadState_34_35*)thread)->gilstate_counter;
} else if (PyThreadState_34_36::IsFor(version)) {
gilstate_counter = ((PyThreadState_34_36*)thread)->gilstate_counter;
}

if (gilstate_counter == 1) {
Expand Down
16 changes: 13 additions & 3 deletions Python/Product/VsPyProf/PythonApi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ VsPyProf* VsPyProf::Create(HMODULE pythonModule) {
}

if ((major == 2 && (minor >= 4 && minor <= 7)) ||
(major == 3 && (minor >= 0 && minor <= 5))) {
(major == 3 && (minor >= 0 && minor <= 6))) {
return new VsPyProf(pythonModule,
major,
minor,
Expand Down Expand Up @@ -134,6 +134,8 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
filename = ((PyCodeObject30_32*)codeObj)->co_filename;
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
filename = ((PyCodeObject33_35*)codeObj)->co_filename;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
filename = ((PyCodeObject36*)codeObj)->co_filename;
}
module = (DWORD_PTR)filename;

Expand Down Expand Up @@ -210,6 +212,9 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
RegisterName(func, ((PyCodeObject33_35*)codeObj)->co_name, &moduleName);
lineno = ((PyCodeObject33_35*)codeObj)->co_firstlineno;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
RegisterName(func, ((PyCodeObject36*)codeObj)->co_name, &moduleName);
lineno = ((PyCodeObject36*)codeObj)->co_firstlineno;
}

// give the profiler the line number of this function
Expand Down Expand Up @@ -242,6 +247,9 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
argCount = ((PyCodeObject33_35*)codeObj)->co_argcount;
argNames = (PyTupleObject*)((PyCodeObject33_35*)codeObj)->co_varnames;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
argCount = ((PyCodeObject36*)codeObj)->co_argcount;
argNames = (PyTupleObject*)((PyCodeObject36*)codeObj)->co_varnames;
}

if (argCount != 0 && argNames->ob_type == PyTuple_Type) {
Expand All @@ -252,8 +260,8 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
PyObject* self = nullptr;
if (PyFrameObject25_33::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject25_33*)frameObj)->f_localsplus[0];
} else if (PyFrameObject34_35::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject34_35*)frameObj)->f_localsplus[0];
} else if (PyFrameObject34_36::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject34_36*)frameObj)->f_localsplus[0];
}
return GetClassNameFromSelf(self, codeObj);
}
Expand Down Expand Up @@ -281,6 +289,8 @@ wstring VsPyProf::GetClassNameFromSelf(PyObject* self, PyObject *codeObj) {
nameObj = ((PyCodeObject30_32*)codeObj)->co_name;
} else if (PyCodeObject33_35::IsFor(MajorVersion, MinorVersion)) {
nameObj = ((PyCodeObject33_35*)codeObj)->co_name;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
nameObj = ((PyCodeObject36*)codeObj)->co_name;
}
GetNameAscii(nameObj, codeName);

Expand Down
57 changes: 45 additions & 12 deletions Python/Product/VsPyProf/python.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ enum PythonVersion {
PythonVersion_32 = 0x0302,
PythonVersion_33 = 0x0303,
PythonVersion_34 = 0x0304,
PythonVersion_35 = 0x0305
PythonVersion_35 = 0x0305,
PythonVersion_36 = 0x0306
};


Expand Down Expand Up @@ -144,7 +145,38 @@ class PyCodeObject33_35 : public PyObject {
}
};

// 2.5 - 3.1
// 3.6
class PyCodeObject36 : public PyObject {
public:
int co_argcount; /* #arguments, except *args */
int co_kwonlyargcount; /* #keyword only arguments */
int co_nlocals; /* #local variables */
int co_stacksize; /* #entries needed for evaluation stack */
int co_flags; /* CO_..., see below */
int co_firstlineno; /* first source line number */
PyObject *co_code; /* instruction opcodes */
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
/* The rest doesn't count for hash or comparisons */
unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
void *co_zombieframe; /* for optimization only (see frameobject.c) */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 6;
}

static bool IsFor(PythonVersion version) {
return version >= PythonVersion_36;
}
};

// 2.5 - 3.6
class PyFunctionObject : public PyObject {
public:
PyObject *func_code; /* A code object */
Expand Down Expand Up @@ -175,7 +207,7 @@ typedef struct {
long hash; /* Hash value; -1 if not set */
} PyUnicodeObject;

// 2.4 - 3.5 compatible
// 2.4 - 3.6 compatible
class PyFrameObject : public PyVarObject {
public:
PyFrameObject *f_back; /* previous frame, or NULL */
Expand Down Expand Up @@ -216,7 +248,7 @@ class PyFrameObject25_33 : public PyFrameObject {
}
};

class PyFrameObject34_35 : public PyFrameObject {
class PyFrameObject34_36 : public PyFrameObject {
public:
/* Borrowed reference to a generator, or NULL */
PyObject *f_gen;
Expand All @@ -231,14 +263,14 @@ class PyFrameObject34_35 : public PyFrameObject {
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5;
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
}
};


typedef void (*destructor)(PyObject *);

// 2.4 - 3.5
// 2.4 - 3.6
class PyMethodDef {
public:
char *ml_name; /* The name of the built-in function/method */
Expand All @@ -261,7 +293,7 @@ class PyTypeObject : public PyVarObject {
void* tp_setattr;
union {
void* tp_compare; /* 2.4 - 3.4 */
void* tp_as_async; /* 3.5 */
void* tp_as_async; /* 3.5 - 3.6 */
};
void* tp_repr;

Expand Down Expand Up @@ -331,7 +363,7 @@ class PyTypeObject : public PyVarObject {
unsigned int tp_version_tag;
};

// 2.4 - 3.5
// 2.4 - 3.6
class PyTupleObject : public PyVarObject {
public:
PyObject *ob_item[1];
Expand All @@ -342,7 +374,7 @@ class PyTupleObject : public PyVarObject {
*/
};

// 2.4 - 3.5
// 2.4 - 3.6
class PyCFunctionObject : public PyObject {
public:
PyMethodDef *m_ml; /* Description of the C function to call */
Expand Down Expand Up @@ -473,7 +505,7 @@ class PyThreadState_30_33 : public PyThreadState {
}
};

class PyThreadState_34_35 : public PyThreadState {
class PyThreadState_34_36 : public PyThreadState {
public:
PyThreadState *prev;
PyThreadState *next;
Expand Down Expand Up @@ -513,11 +545,11 @@ class PyThreadState_34_35 : public PyThreadState {

/* XXX signal handlers should also be here */
static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 5;
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
}

static bool IsFor(PythonVersion version) {
return version >= PythonVersion_34 && version <= PythonVersion_35;
return version >= PythonVersion_34 && version <= PythonVersion_36;
}
};

Expand Down Expand Up @@ -570,6 +602,7 @@ static PythonVersion GetPythonVersion(HMODULE hMod) {
case '3': return PythonVersion_33;
case '4': return PythonVersion_34;
case '5': return PythonVersion_35;
case '6': return PythonVersion_36;
}
}
}
Expand Down
17 changes: 16 additions & 1 deletion Python/Tests/Core/DebugReplEvaluatorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public class DebugReplEvaluatorTests {
Assert.AreEqual(@"Traceback (most recent call last):
File ""<debug input>"", line 1, in <module>
NameError: name 'does_not_exist' is not defined
", _window.Error);
".Replace("\r\n", "\n"), _window.Error.Replace("\r\n", "\n"));
}

[TestMethod, Priority(3)]
Expand Down Expand Up @@ -434,6 +434,21 @@ public class DebugReplEvaluatorTests35 : DebugReplEvaluatorTests {
}
}

[TestClass]
public class DebugReplEvaluatorTests36 : DebugReplEvaluatorTests {
[ClassInitialize]
public static new void DoDeployment(TestContext context) {
AssertListener.Initialize();
PythonTestData.Deploy();
}

internal override PythonVersion Version {
get {
return PythonPaths.Python36 ?? PythonPaths.Python36_x64;
}
}
}

[TestClass]
public class DebugReplEvaluatorTests27 : DebugReplEvaluatorTests {
[ClassInitialize]
Expand Down
31 changes: 29 additions & 2 deletions Python/Tests/DebuggerTests/AttachTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,10 @@ void main()
string script = TestData.GetPath(@"TestData\DebuggerProject\AttachPtvsd.py");
var psi = new ProcessStartInfo(Version.InterpreterPath, PtvsdInterpreterArguments + " \"" + script + "\"") {
WorkingDirectory = TestData.GetPath(),
UseShellExecute = false
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
};

var p = Process.Start(psi);
Expand All @@ -822,7 +825,7 @@ void main()
break;
} catch (SocketException) {
// Failed to connect - the process might have not started yet, so keep trying a few more times.
if (i >= 5) {
if (i >= 5 || p.HasExited) {
throw;
}
}
Expand Down Expand Up @@ -862,6 +865,8 @@ void main()
DetachProcess(proc);
}
} finally {
Console.WriteLine(p.StandardOutput.ReadToEnd());
Console.WriteLine(p.StandardError.ReadToEnd());
DisposeProcess(p);
}
}
Expand Down Expand Up @@ -1046,6 +1051,28 @@ public class AttachTests35_x64 : AttachTests35 {
}
}

[TestClass]
public class AttachTests36 : AttachTests {
internal override PythonVersion Version {
get {
return PythonPaths.Python36;
}
}

public override void AttachNewThread_PyThreadState_New() {
// PyEval_AcquireLock deprecated in 3.2
}
}

[TestClass]
public class AttachTests36_x64 : AttachTests35 {
internal override PythonVersion Version {
get {
return PythonPaths.Python36_x64;
}
}
}

[TestClass]
public class AttachTests25 : AttachTests {
internal override PythonVersion Version {
Expand Down
Loading