This commit represents the including of the new external python for w…

…in32. credit goes to mostly WiSo and blinkseb.

[WIN32] added include/python to the include path and removed lib/python/PC
[WIN32] added python27.lib to the linker
[WIN32] removed Py_NO_ENABLE_SHARED
Still need the python directory that was under system because linux internal python build still put things there.
There is a bug in the Windows external python because on Windows, the DLL that python is linked against may not be the DLL that xbmc is linked against. So passing a FILE* to python from an fopen has the potential to crash. This resolves that problem. Credit to blinkseb.
added: download python dependencies with our script.
changed: use Python 2.6.6 on Windows. Debug build is included in the archive, but not copied by default. Set DEBUG to true to copy these.
[WIN32] changed: in case of debug python is used, be sure to exclude it from our installer
[WIN32] fixed: get debug python working. The following has to be defined:
 - Py_NO_ENABLE_SHARED
 - Py_TRACE_REFS
[WIN32] fixed: oups...
[WIN32] changed: got external python work. It needs more testing to be sure. We don't link against a .lib to be able to load python from where we want (that's why Py_NO_ENABLE_SHARED is defined). We don't define USE_EXTERNAL_PYTHON because we need to use our dll loader.
Fixed a couple of problems: 1) The Makefile.in file was merged incorrectly. 2) We cannot call PyEval_AcquireLock prior to the threads being initialized in linux.
OSX shouldn't build the internal python ... ever ... but especially when USE_EXTERNAL_PYTHON is set.

Makefile.in

@@ -461,17 +461,11 @@ imagelib: dllloader
 	$(MAKE) -C lib/cximage-6.0
 
 codecs: papcodecs dvdpcodecs
-libs: cmyth libhdhomerun libid3tag imagelib libexif python system/libcpluff-@ARCH@.so
-externals: codecs libs python visualizations screensavers
-
-ifeq ($(findstring osx,@ARCH@), osx)
-libs: libhdhomerun libid3tag imagelib libexif $(PYTHON_TARGET) system/libcpluff-@ARCH@.so system/players/paplayer/libmodplug-@ARCH@.so
-else
-libs: libhdhomerun libid3tag imagelib libexif $(PYTHON_TARGET) system/libcpluff-@ARCH@.so
-endif
+libs: cmyth libhdhomerun libid3tag imagelib libexif  $(PYTHON_TARGET) system/libcpluff-@ARCH@.so
+externals: codecs libs $(PYTHON_TARGET) visualizations screensavers
 
 xcode_depends: \
-	codecs libs python visualizations screensavers eventclients skins \
+	codecs libs $(PYTHON_TARGET) visualizations screensavers eventclients skins \
 	lib/libsquish/libsquish.a \
 	lib/libapetag/.libs/libapetag.a \
 	lib/libXBMS/libxbms.a \

project/BuildDependencies/scripts/copy_deps_d.bat

@@ -13,4 +13,4 @@ xcopy "%XBMC_PATH%\lib\Python\Lib" "%XBMC_PATH%\system\python\Lib" /E /Q /I /Y /
 del py_exclude.txt
 
 rem create directories
-IF NOT EXIST "%XBMC_PATH%\system\players\paplayer" md "%XBMC_PATH%\system\players\paplayer"
+IF NOT EXIST "%XBMC_PATH%\system\players\paplayer" md "%XBMC_PATH%\system\players\paplayer"

project/BuildDependencies/scripts/python2.6_d.bat

@@ -0,0 +1,32 @@
+@ECHO ON
+
+SET LOC_PATH=%CD%
+SET FILES=%LOC_PATH%\python2.6_d.txt
+
+CALL dlextract.bat python2.6 %FILES%
+
+cd %TMP_PATH%
+
+set DEBUG=false
+
+echo \test\ > py_exclude.txt
+
+if "%DEBUG%" == "false" (
+  echo _d.dll >> py_exclude.txt
+  echo _d.pyd >> py_exclude.txt
+  echo _d.lib >> py_exclude.txt
+  echo .pdb >> py_exclude.txt
+  echo tcl85g. >> py_exclude.txt
+  echo tclpip85g. >> py_exclude.txt
+  echo tk85g. >> py_exclude.txt
+)
+
+xcopy python2.6.6\include\* "%CUR_PATH%\include\python" /E /Q /I /Y /EXCLUDE:py_exclude.txt
+xcopy python2.6.6\python\DLLs "%XBMC_PATH%\system\python\DLLs" /E /Q /I /Y /EXCLUDE:py_exclude.txt
+xcopy python2.6.6\python\Lib "%XBMC_PATH%\system\python\Lib" /E /Q /I /Y /EXCLUDE:py_exclude.txt
+xcopy python2.6.6\python26.dll "%XBMC_PATH%\system\python\" /Q /I /Y /EXCLUDE:py_exclude.txt
+xcopy python2.6.6\python26_d.dll "%XBMC_PATH%\system\python\" /Q /I /Y /EXCLUDE:py_exclude.txt
+xcopy python2.6.6\libs\*.lib "%CUR_PATH%\lib\" /E /Q /I /Y /EXCLUDE:py_exclude.txt
+xcopy python2.6.6\libs\*.pdb "%CUR_PATH%\lib\" /E /Q /I /Y /EXCLUDE:py_exclude.txt
+
+cd %LOC_PATH%

project/BuildDependencies/scripts/python2.6_d.txt

@@ -0,0 +1,2 @@
+; filename                        source of the file
+python2.6.6.7z                    http://xbmcwindeps.googlecode.com/files

project/VS2010Express/XBMC for Windows.props

@@ -3,7 +3,7 @@
   <ImportGroup Label="PropertySheets" />
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
-    <IncludePath>$(SolutionDir)\..\BuildDependencies\include;$(IncludePath)</IncludePath>
+    <IncludePath>$(SolutionDir)\..\BuildDependencies\include;$(SolutionDir)\..\BuildDependencies\include\python;$(IncludePath)</IncludePath>
   </PropertyGroup>
   <PropertyGroup>
     <LibraryPath>$(SolutionDir)\..\BuildDependencies\lib;$(LibraryPath)</LibraryPath>

project/VS2010Express/XBMC.vcxproj

@@ -146,7 +146,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug (DirectX)|Win32'">
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\;..\..\xbmc\;..\..\xbmc\win32\;..\..\xbmc\cores\dvdplayer;..\..\lib;..\..\lib\ffmpeg;..\..\lib\ffmpeg\include-xbmc-win32;..\..\lib\freetype\include;..\..\lib\jsoncpp\jsoncpp\include;..\..\lib\liblame\include;..\..\lib\libUPnP\Platinum\Source\Devices\MediaRenderer;..\..\lib\libUPnP\Platinum\Source\Devices\MediaConnect;..\..\lib\libUPnP\Platinum\Source\Devices\MediaServer;..\..\lib\libUPnP\Platinum\Source\Platinum;..\..\lib\libUPnP\Platinum\Source\Core;..\..\lib\libUPnP\Neptune\Source\Core;..\..\lib\libUPnP\Neptune\Source\System\Win32;..\..\lib\Python\PC;..\..\lib\win32\boost;..\..\lib\win32\libbluray_win32;..\..\lib\win32\libcdio\include;..\..\lib\win32\libiconv\include;..\..\lib\win32\libmicrohttpd_win32\include;..\..\lib\win32\libwavpack;..\..\lib\win32\pcre;..\..\lib\win32\vorbisfile\libvorbis\include;..\..\lib\win32\vorbisfile\ogg\include;..\..\lib\jsoncpp\include</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\;..\..\xbmc\;..\..\xbmc\cores;..\..\xbmc\cores\dvdplayer;..\..\xbmc\dialogs;..\..\xbmc\FileSystem;..\..\xbmc\guilib\;..\..\xbmc\utils;..\..\xbmc\win32;..\..\xbmc\windowing;..\..\lib;..\..\lib\ffmpeg;..\..\lib\ffmpeg\include-xbmc-win32;..\..\lib\freetype\include;..\..\lib\jsoncpp\jsoncpp\include;..\..\lib\libflac\flac-1.2.1\include;..\..\lib\liblame\include;..\..\lib\libUPnP\Platinum\Source\Devices\MediaRenderer;..\..\lib\libUPnP\Platinum\Source\Devices\MediaConnect;..\..\lib\libUPnP\Platinum\Source\Devices\MediaServer;..\..\lib\libUPnP\Platinum\Source\Platinum;..\..\lib\libUPnP\Platinum\Source\Core;..\..\lib\libUPnP\Neptune\Source\Core;..\..\lib\libUPnP\Neptune\Source\System\Win32;..\..\lib\win32\boost;..\..\lib\win32\libbluray_win32;..\..\lib\win32\libcdio\include;..\..\lib\win32\libiconv\include;..\..\lib\win32\libmicrohttpd_win32\include;..\..\lib\win32\libsamplerate\src;..\..\lib\win32\libssh_win32\include;..\..\lib\win32\libwavpack;..\..\lib\win32\pcre;..\..\lib\win32\sqlite;..\..\lib\win32\vorbisfile\libvorbis\include;..\..\lib\win32\vorbisfile\ogg\include;..\..\lib\jsoncpp\include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_WINDOWS;_MSVC;WIN32;_DEBUG;_WIN32_WINNT=0x0501;WINVER=0x0500;NOMINMAX;_USE_32BIT_TIME_T;HAS_DX;Py_NO_ENABLE_SHARED;USE_EXTERNAL_PYTHON;D3D_DEBUG_INFO;__STDC_CONSTANT_MACROS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <ExceptionHandling>Async</ExceptionHandling>
@@ -738,6 +738,7 @@
     <ClCompile Include="..\..\xbmc\win32\strverscmp.cpp" />
     <ClCompile Include="..\..\xbmc\win32\Win32DelayedDllLoad.cpp" />
     <ClCompile Include="..\..\xbmc\win32\WIN32Util.cpp" />
+    <ClCompile Include="..\..\xbmc\win32\WIN32XBPythonDll.cpp" />
     <ClCompile Include="..\..\xbmc\win32\WINDirectSound.cpp" />
     <ClCompile Include="..\..\xbmc\win32\WindowHelper.cpp" />
     <ClCompile Include="..\..\xbmc\win32\WINFileSMB.cpp" />

project/VS2010Express/XBMC.vcxproj.filters

@@ -2454,6 +2454,9 @@
     <ClCompile Include="..\..\xbmc\XBDateTime.cpp">
       <Filter>utils</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\xbmc\win32\WIN32XBPythonDll.cpp">
+      <Filter>interfaces\python</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\xbmc\win32\pch.h">

project/Win32BuildSetup/XBMC for Windows.nsi

@@ -102,7 +102,7 @@ Section "XBMC" SecXBMC
   SetOutPath "$INSTDIR\sounds"
   File /r /x *.so "${xbmc_root}\Xbmc\sounds\*.*"
   SetOutPath "$INSTDIR\system"
-  File /r /x *.so /x mplayer "${xbmc_root}\Xbmc\system\*.*"
+  File /r /x *.so /x mplayer /x *_d.* /x tcl85g.dll /x tclpip85g.dll /x tk85g.dll "${xbmc_root}\Xbmc\system\*.*"
 
   ; delete  msvc?90.dll's in INSTDIR, we use the vcredist installer later
   Delete "$INSTDIR\msvcr90.dll"
@@ -165,6 +165,7 @@ Section "XBMC" SecXBMC
                  "HelpLink" "http://xbmc.org/support"
   WriteRegStr HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\XBMC" \
                  "URLInfoAbout" "http://xbmc.org"
+
 SectionEnd
 
 SectionGroup "Language" SecLanguages

system/python/readme.txt

@@ -0,0 +1,9 @@
+python directory structure
+
+DLLs dir:
+  Library directory used by python.
+  python exensions (.pyd) from cvs can be placed here
+
+Lib dir:
+  Library directory used by python.
+  Here can new packages or modules be installed by the user.

xbmc/interfaces/python/XBPyThread.cpp

@@ -229,15 +229,39 @@ void XBPyThread::Process()
   {
     if (m_type == 'F')
     {
+#ifdef USE_EXTERNAL_PYTHON
       // run script from file
-      FILE *fp = fopen_utf8(_P(m_source).c_str(), "r");
+      // We need to have python open the file because on Windows the DLL that python
+      //  is linked against may not be the DLL that xbmc is linked against so 
+      //  passing a FILE* to python from an fopen has the potential to crash.
+      PyObject* file = PyFile_FromString((char *) _P(m_source).c_str(), (char*)"r");
+      FILE *fp = PyFile_AsFile(file);
+#else
+      FILE *fp = fopen_utf8(_P(m_source).c_str(), "r");      
+#endif
+
       if (fp)
       {
         PyObject *f = PyString_FromString(_P(m_source).c_str());
         PyDict_SetItemString(moduleDict, "__file__", f);
         Py_DECREF(f);
         PyRun_File(fp, _P(m_source).c_str(), m_Py_file_input, moduleDict, moduleDict);
+
+#ifdef USE_EXTERNAL_PYTHON
+        // Get a reference to the main module
+        // and global dictionary
+        PyObject* main_module = PyImport_AddModule((char*)"__main__");
+        PyObject* global_dict = PyModule_GetDict(main_module);
+
+        // Extract a reference to the function "func_name"
+        // from the global dictionary
+        PyObject* expression = PyDict_GetItemString(global_dict, "xbmcclosefilehack");
+
+        if (!PyObject_CallFunction(expression,(char*)"(O)",file))
+          CLog::Log(LOGERROR,"Failed to close the script file %s",_P(m_source).c_str());
+#else
         fclose(fp);
+#endif
       }
       else
         CLog::Log(LOGERROR, "%s not found!", m_source);

xbmc/interfaces/python/XBPython.cpp

@@ -41,7 +41,7 @@
 #include "Util.h"
 
 #ifndef _LINUX
-#if (defined USE_EXTERNAL_PYTHON) && (defined HAVE_LIBPYTHON2_6)
+#if defined HAVE_LIBPYTHON2_6
 #define PYTHON_DLL "special://xbmcbin/system/python/python26.dll"
 #else
 #define PYTHON_DLL "special://xbmcbin/system/python/python24.dll"
@@ -324,10 +324,13 @@ void XBPython::InitializeInterpreter()
         "\t\txbmc.output('.')\n"
         "\tdef flush(self):\n"
         "\t\txbmc.output('.')\n"
-        "\n"
         "import sys\n"
         "sys.stdout = xbmcout()\n"
         "sys.stderr = xbmcout()\n"
+        "def xbmcclosefilehack(f):\n"
+//        "\txbmc.output(\"Closing Script File.\")\n"
+        "\tf.close()\n"
+        "\n"
         "print '-->Python Interpreter Initialized<--'\n"
         "") == -1)
   {
@@ -420,7 +423,14 @@ void XBPython::Initialize()
       CLog::Log(LOGDEBUG, "Python wrapper library linked with system Python library");
 #endif /* USE_EXTERNAL_PYTHON */
 
+      if (PyEval_ThreadsInitialized())
+        PyEval_AcquireLock();
+      else
+        PyEval_InitThreads();
+
       Py_Initialize();
+      PyEval_ReleaseLock();
+
       // If this is not the first time we initialize Python, the interpreter
       // lock already exists and we need to lock it as PyEval_InitThreads
       // would not do that in that case.

xbmc/interfaces/python/XBPythonDll.h

@@ -49,6 +49,10 @@
 #define PyExc_RuntimeError ((PyObject*)(*(long*)pointer_PyExc_RuntimeError))
 #define PyExc_ReferenceError ((PyObject*)(*(long*)pointer_PyExc_ReferenceError))
 
+#if defined(_WIN32) && defined(Py_TRACE_REFS)
+#define _Py_RefTotal (*((Py_ssize_t*)pointer__Py_RefTotal))
+#endif
+
 #if (defined USE_EXTERNAL_PYTHON) && (!defined HAVE_LIBPYTHON2_4)
   /* Upstream Python rename Py_InitModule4 for 64-bit systems for Python
    versions higher than 2.4 */
@@ -92,6 +96,10 @@ class LibraryLoader;
   extern DATA_OBJECT(PyExc_RuntimeError)
   extern DATA_OBJECT(PyExc_ReferenceError)
 
+#if defined(_WIN32) && defined(Py_TRACE_REFS)
+  extern DATA_OBJECT(_Py_RefTotal)
+#endif
+
   bool python_load_dll(LibraryLoader& dll);
 
 #ifdef __cplusplus

xbmc/win32/WIN32XBPythonDll.cpp

@@ -214,7 +214,11 @@ extern "C"
   FUNCTION4(PyLong_FromLong)
   FUNCTION12(PyModule_AddStringConstant)
   FUNCTION12(PyModule_AddObject)
+#ifndef Py_TRACE_REFS
   FUNCTION20(Py_InitModule4)
+#else
+  FUNCTION20(Py_InitModule4TraceRefs)
+#endif
   FUNCTION4(PyInt_AsLong)
   FUNCTION4(PyFloat_AsDouble)
   FUNCTION4(PyString_FromString)
@@ -271,7 +275,12 @@ extern "C"
   FUNCTION(PyErr_Clear)
   FUNCTION12(PyObject_SetAttrString)
 
-#if (defined USE_EXTERNAL_PYTHON) && (defined HAVE_LIBPYTHON2_6)
+#ifdef Py_TRACE_REFS
+  FUNCTION12(_Py_NegativeRefcount)
+  FUNCTION4(_Py_Dealloc)
+#endif
+
+#if (defined HAVE_LIBPYTHON2_6)
   FUNCTION8(PyRun_SimpleStringFlags)
   FUNCTION20(PyRun_StringFlags)
   FUNCTION28(PyRun_FileExFlags)
@@ -302,6 +311,10 @@ extern "C"
   DATA_OBJECT(PyTuple_Type)
   DATA_OBJECT(PyDict_Type)
 
+#ifdef Py_TRACE_REFS
+  DATA_OBJECT(_Py_RefTotal)
+#endif
+
   bool python_load_dll(LibraryLoader& dll)
   {
     bool bResult;
@@ -355,7 +368,11 @@ extern "C"
       dll.ResolveExport(DLL_FUNCTION(PyLong_FromLong)) &&
       dll.ResolveExport(DLL_FUNCTION(PyModule_AddStringConstant)) &&
       dll.ResolveExport(DLL_FUNCTION(PyModule_AddObject)) &&
+#ifndef Py_TRACE_REFS
       dll.ResolveExport(DLL_FUNCTION(Py_InitModule4)) &&
+#else
+      dll.ResolveExport(DLL_FUNCTION(Py_InitModule4TraceRefs)) &&
+#endif
       dll.ResolveExport(DLL_FUNCTION(PyInt_AsLong)) &&
       dll.ResolveExport(DLL_FUNCTION(PyFloat_AsDouble)) &&
       dll.ResolveExport(DLL_FUNCTION(PyString_FromString)) &&
@@ -402,6 +419,11 @@ extern "C"
       dll.ResolveExport(DLL_OBJECT_DATA(PyUnicode_Type)) &&
       dll.ResolveExport(DLL_OBJECT_DATA(PyTuple_Type)) &&
       dll.ResolveExport(DLL_OBJECT_DATA(PyDict_Type)) &&
+#ifdef Py_TRACE_REFS
+      dll.ResolveExport(DLL_OBJECT_DATA(_Py_RefTotal)) &&
+      dll.ResolveExport(DLL_FUNCTION(_Py_NegativeRefcount)) &&
+      dll.ResolveExport(DLL_FUNCTION(_Py_Dealloc)) &&
+#endif
       dll.ResolveExport(DLL_FUNCTION(PyErr_Fetch)) &&
       dll.ResolveExport(DLL_FUNCTION(PyImport_AddModule)) &&
       dll.ResolveExport(DLL_FUNCTION(PyImport_ImportModule)) &&
@@ -410,7 +432,7 @@ extern "C"
       dll.ResolveExport(DLL_FUNCTION(PyErr_Clear)) &&
       dll.ResolveExport(DLL_FUNCTION(PyObject_SetAttrString)) &&
       dll.ResolveExport(DLL_FUNCTION(PyErr_ExceptionMatches)) &&
-#if (defined USE_EXTERNAL_PYTHON) && (defined HAVE_LIBPYTHON2_6)
+#if (defined HAVE_LIBPYTHON2_6)
       dll.ResolveExport(DLL_FUNCTION(PyRun_SimpleStringFlags)) &&
       dll.ResolveExport(DLL_FUNCTION(PyRun_StringFlags)) &&
       dll.ResolveExport(DLL_FUNCTION(PyRun_FileExFlags)) &&