Issue #1847: Byte-compile all startup scripts

CArchiveWriter compiles all typecode 's' entries (originally
'PYSOURCE') to bytecode using `get_code_object`, `strip_paths_in_code`,
and `marshal.dumps`.

pyi_launch_run_scripts unmarshals the compiled code objects and
executes each one as the __main__ module. This is a bit sloppy since
the namespace leaks between modules, but that can be fixed separately

PyInstaller/archive/writers.py

@@ -28,6 +28,7 @@
 import marshal
 import zlib
 
+from PyInstaller.building.utils import get_code_object, strip_paths_in_code
 from .readers import PYZ_TYPE_MODULE, PYZ_TYPE_PKG, PYZ_TYPE_DATA
 from ..compat import BYTECODE_MAGIC
 
@@ -356,52 +357,56 @@ def add(self, entry):
         (nm, pathnm, flag, typcd) = entry[:4]
         # FIXME Could we make the version 5 the default one?
         # Version 5 - allow type 'o' = runtime option.
+        code_data = None
+        fh = None
         try:
             if typcd in ('o', 'd'):
-                fh = None
                 ulen = 0
-                postfix = b''
                 flag = 0
             elif typcd == 's':
-                # If it's a source code file, add \0 terminator as it will be
-                # executed as-is by the bootloader.
-                # Must read this in binary-mode, too, because
-                # compression only accepts a binary stream. Further we
-                # do not process it here, so why decode?
-                fh = open(pathnm, 'rb')
-                postfix = b'\n\0'
-                ulen = os.fstat(fh.fileno()).st_size + len(postfix)
+                # If it's a source code file, compile it to a code object and marshall
+                # the object so it can be unmarshalled by the bootloader.
+
+                code = get_code_object(nm, pathnm)
+                code = strip_paths_in_code(code)
+
+                code_data = marshal.dumps(code)
+                ulen = len(code_data)
             else:
                 fh = open(pathnm, 'rb')
-                postfix = b''
                 ulen = os.fstat(fh.fileno()).st_size
         except IOError:
             print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd))
             raise
 
         where = self.lib.tell()
         assert flag in range(3)
-        if not fh:
+        if not fh and not code_data:
             # no need to write anything
             pass
         elif flag == 1:
-            assert fh
             comprobj = zlib.compressobj(self.LEVEL)
-            while 1:
-                buf = fh.read(16*1024)
-                if not buf:
-                    break
-                self.lib.write(comprobj.compress(buf))
-            self.lib.write(comprobj.compress(postfix))
+            if code_data is not None:
+                self.lib.write(comprobj.compress(code_data))
+            else:
+                assert fh
+                while 1:
+                    buf = fh.read(16*1024)
+                    if not buf:
+                        break
+                    self.lib.write(comprobj.compress(buf))
             self.lib.write(comprobj.flush())
+
         else:
-            assert fh
-            while 1:
-                buf = fh.read(16*1024)
-                if not buf:
-                    break
-                self.lib.write(buf)
-            self.lib.write(postfix)
+            if code_data is not None:
+                self.lib.write(code_data)
+            else:
+                assert fh
+                while 1:
+                    buf = fh.read(16*1024)
+                    if not buf:
+                        break
+                    self.lib.write(buf)
 
         dlen = self.lib.tell() - where
         if typcd == 'm':

bootloader/src/pyi_launch.c

@@ -20,10 +20,16 @@
     #include <Carbon/Carbon.h>  // TransformProcessType
 #endif
 
-
 #ifdef _WIN32
     #include <windows.h>
+    #include <winsock.h>  // ntohl
 #else
+    #ifdef __FreeBSD__
+    	// freebsd issue #188316
+    	#include <arpa/inet.h>  // ntohl
+    #else
+    	#include <netinet/in.h>  // ntohl
+    #endif
     #include <langinfo.h>  // CODESET, nl_langinfo
     #include <limits.h>  // PATH_MAX
     #include <stdlib.h>  // malloc
@@ -322,10 +328,23 @@ int pyi_launch_run_scripts(ARCHIVE_STATUS *status)
 {
 	unsigned char *data;
 	char buf[PATH_MAX];
-	int rc = 0;
 	TOC * ptoc = status->tocbuff;
-	PyObject *__main__ = PI_PyImport_AddModule("__main__");
+	PyObject *__main__;
 	PyObject *__file__;
+	PyObject *main_dict;
+	PyObject *code, *retval;
+
+	__main__ = PI_PyImport_AddModule("__main__");
+	if (!__main__) {
+	    FATALERROR("Could not get __main__ module.");
+	    return -1;
+	}
+
+	main_dict = PI_PyModule_GetDict(__main__);
+	if (!main_dict) {
+	    FATALERROR("Could not get __main__ module's dict.");
+	    return -1;
+	}
 
 	/* Iterate through toc looking for scripts (type 's') */
 	while (ptoc < status->tocend) {
@@ -338,18 +357,27 @@ int pyi_launch_run_scripts(ARCHIVE_STATUS *status)
 			strcat(buf, ".py");
 			VS("LOADER: Running %s\n", buf);
             if (is_py2) {
-	      __file__ = PI_PyString_FromString(buf);
-	    } else {
-	      __file__ = PI_PyUnicode_FromString(buf);
-	    };
+              __file__ = PI_PyString_FromString(buf);
+            } else {
+              __file__ = PI_PyUnicode_FromString(buf);
+            };
             PI_PyObject_SetAttrString(__main__, "__file__", __file__);
             Py_DECREF(__file__);
+
+            /* Unmarshall code object */
+            code = PI_PyMarshal_ReadObjectFromString(data, ntohl(ptoc->ulen));
+            if(!code) {
+                FATALERROR("Failed to unmarshal code object for %s\n", ptoc->name);
+			    PI_PyErr_Print();
+                return -1;
+            }
 			/* Run it */
-			rc = PI_PyRun_SimpleString((char *) data);
-			/* log errors and abort */
-			if (rc != 0) {
-				FATALERROR("%s returned %d\n", ptoc->name, rc);
-				return rc;
+			retval = PI_PyEval_EvalCode(code, main_dict, main_dict);
+			/* If retval is NULL, an error occured. Otherwise, it is a Python object */
+			if (!retval) {
+                FATALERROR("Failed to execute script %s\n", ptoc->name);
+                PI_PyErr_Print();
+				return -1;
 			}
 			free(data);
 		}

bootloader/src/pyi_python.c

@@ -82,6 +82,8 @@ DECLPROC(PyUnicode_FromFormat);
 DECLPROC(PyUnicode_DecodeFSDefault);
 DECLPROC(PyUnicode_Decode);
 
+DECLPROC(PyEval_EvalCode);
+DECLPROC(PyMarshal_ReadObjectFromString);
 
 /*
  * Get all of the entry points from libpython
@@ -135,6 +137,8 @@ int pyi_python_map_names(HMODULE dll, int pyvers)
     GETPROC(dll, PySys_GetObject);
     GETPROC(dll, PySys_SetObject);
     GETPROC(dll, PySys_SetPath);
+    GETPROC(dll, PyEval_EvalCode);
+    GETPROC(dll, PyMarshal_ReadObjectFromString);
     if (pyvers >= 30) {
       // new in Python 2.6, but not reliable available in all Linux distros
       GETPROC(dll, PyUnicode_FromString);

bootloader/src/pyi_python.h

@@ -147,6 +147,9 @@ EXTDECLPROC(PyObject *, PyUnicode_FromFormat, (const char *, ...));
 EXTDECLPROC(PyObject *, PyUnicode_DecodeFSDefault, (const char *));
 EXTDECLPROC(PyObject *, PyUnicode_Decode, (const char *, size_t, const char *, const char *)); // Py_ssize_t
 
+/* Used to load and execute marshalled code objects */
+EXTDECLPROC(PyObject *, PyEval_EvalCode, (PyObject *, PyObject *, PyObject *));
+EXTDECLPROC(PyObject *, PyMarshal_ReadObjectFromString, (const char *, size_t)); // Py_ssize_t
 
 
 /*