Declutter onedir bundles

PyInstaller/building/api.py

@@ -387,6 +387,8 @@ def __init__(self, *args, **kwargs):
             entitlements_file
                 macOS only. Optional path to entitlements file to use with code signing of collected binaries
                 (--entitlements option to codesign utility).
+            contents_directory
+                Onedir mode only. Specifies the name of the directory where all files par the executable will be placed.
         """
         from PyInstaller.config import CONF
 
@@ -407,6 +409,7 @@ def __init__(self, *args, **kwargs):
         self.strip = kwargs.get('strip', False)
         self.upx_exclude = kwargs.get("upx_exclude", [])
         self.runtime_tmpdir = kwargs.get('runtime_tmpdir', None)
+        self.contents_directory = kwargs.get("contents_directory", "_internal")
         # If ``append_pkg`` is false, the archive will not be appended to the exe, but copied beside it.
         self.append_pkg = kwargs.get('append_pkg', True)
 
@@ -456,6 +459,13 @@ def __init__(self, *args, **kwargs):
             logger.warning('Ignoring icon; supported only on Windows and macOS!')
             self.icon = None
 
+        if self.contents_directory in ("", ".", "..") \
+                or "/" in self.contents_directory or "\\" in self.contents_directory:
+            raise SystemExit(
+                f'Invalid value "{self.contents_directory}" passed to `--contents-directory` or `contents_directory`. '
+                'Exactly one directory level is required.'
+            )
+
         # Old .spec format included in 'name' the path where to put created app. New format includes only exename.
         #
         # Ignore fullpath in the 'name' and prepend DISTPATH or WORKPATH.
@@ -512,6 +522,8 @@ def __init__(self, *args, **kwargs):
             # no value; presence means "true"
             self.toc.append(("pyi-macos-argv-emulation", "", "OPTION"))
 
+        self.toc.append(("pyi-contents-directory " + self.contents_directory, "", "OPTION"))
+
         # If the icon path is relative, make it relative to the .spec file.
         def makeabs(path):
             if os.path.isabs(path):
@@ -898,6 +910,13 @@ def __init__(self, *args, **kwargs):
         # DISTPATH). Old .spec formats included parent path, so strip it away.
         self.name = os.path.join(CONF['distpath'], os.path.basename(kwargs.get('name')))
 
+        for arg in args:
+            if isinstance(arg, EXE):
+                self.contents_directory = arg.contents_directory
+                break
+        else:
+            raise ValueError("No EXE() instance was passed to COLLECT()")
+
         self.toc = []
         for arg in args:
             # Valid arguments: EXE object and TOC-like iterables
@@ -960,7 +979,10 @@ def assemble(self):
                     'Security-Alert: attempting to store file outside of the dist directory: %r. Aborting.' % dest_name
                 )
             # Create parent directory structure, if necessary
-            dest_path = os.path.join(self.name, dest_name)  # Absolute destination path
+            if typecode in ("EXECUTABLE", "PKG"):
+                dest_path = os.path.join(self.name, dest_name)
+            else:
+                dest_path = os.path.join(self.name, self.contents_directory, dest_name)
             dest_dir = os.path.dirname(dest_path)
             try:
                 os.makedirs(dest_dir, exist_ok=True)

PyInstaller/building/makespec.py

@@ -227,6 +227,11 @@ def __add_options(parser):
         "--name",
         help="Name to assign to the bundled app and spec file (default: first script's basename)",
     )
+    g.add_argument(
+        "--contents-directory",
+        help="For onedir builds only, specify the name of the directory in which all supporting files (i.e. everything "
+        "except the executable itself) will be placed in.",
+    )
 
     g = parser.add_argument_group('What to bundle, where to search')
     g.add_argument(
@@ -615,6 +620,7 @@ def main(
     noupx=False,
     upx_exclude=None,
     runtime_tmpdir=None,
+    contents_directory=None,
     pathex=[],
     version_file=None,
     specpath=None,
@@ -695,6 +701,8 @@ def main(
         # On Mac OS, the default icon has to be copied into the .app bundle.
         # The the text value 'None' means - use default icon.
         icon_file = 'None'
+    if contents_directory:
+        exe_options += "\n    contents_directory='%s'," % (contents_directory or "_internal")
 
     if bundle_identifier:
         # We need to encapsulate it into apostrofes.

PyInstaller/utils/conftest.py

@@ -13,6 +13,7 @@
 import glob
 import logging
 import os
+import platform
 import re
 import shutil
 import subprocess
@@ -541,7 +542,7 @@ def compiled_dylib(tmpdir, request):
         elif is_darwin:
             tmp_data_dir = tmp_data_dir.join('ctypes_dylib.dylib')
             # On Mac OS X we need to detect architecture - 32 bit or 64 bit.
-            arch = 'i386' if architecture == '32bit' else 'x86_64'
+            arch = 'arm64' if platform.machine() == 'arm64' else 'i386' if architecture == '32bit' else 'x86_64'
             cmd = (
                 'gcc -arch ' + arch + ' -Wall -dynamiclib '
                 'ctypes_dylib.c -o ctypes_dylib.dylib -headerpad_max_install_names'

bootloader/src/pyi_archive.c

@@ -511,40 +511,50 @@ pyi_arch_setup(ARCHIVE_STATUS *status, char const * archive_path, char const * e
     if (snprintf(status->executablename, PATH_MAX, "%s", executable_path) >= PATH_MAX) {
         return false;
     }
-    /* Set homepath to where the archive is */
-#if defined(__APPLE__)
-    char executable_dir[PATH_MAX];
-    size_t executable_dir_len;
 
+    /* Open the archive */
+    if (pyi_arch_open(status)) {
+        /* If this is not an archive, we MUST close the file, */
+        /* otherwise the open file-handle will be reused when */
+        /* testing the next file. */
+        pyi_arch_close_fp(status);
+        return false;
+    }
+
+    /* Set homepath (a.k.a. sys._MEIPASS) */
+    char executable_dir[PATH_MAX];
     pyi_path_dirname(executable_dir, executable_path);
+#if defined(__APPLE__)
+    size_t executable_dir_len;
     executable_dir_len = strnlen(executable_dir, PATH_MAX);
-    if (executable_dir_len > 19 && strncmp(executable_dir + executable_dir_len - 19, ".app/Contents/MacOS", 19) == 0) {
+    bool is_macos_app_bundle = executable_dir_len > 19 && strncmp(executable_dir + executable_dir_len - 19, ".app/Contents/MacOS", 19) == 0;
+#else
+    bool is_macos_app_bundle = false;
+#endif
+    if (is_macos_app_bundle) {
         /* macOS .app bundle; relocate homepath from Contents/MacOS
          * directory to Contents/Frameworks */
         char contents_dir[PATH_MAX];
         pyi_path_dirname(contents_dir, executable_dir);
         pyi_path_join(status->homepath, contents_dir, "Frameworks");
     } else {
-        pyi_path_dirname(status->homepath, archive_path);
+        char * contents_directory = pyi_arch_get_option(status, "pyi-contents-directory");
+        if (!contents_directory) {
+            FATALERROR("pyi-contents-directory option not found in onedir bundle archive!");
+            return false;
+        }
+        char root_path[PATH_MAX];
+        pyi_path_dirname(root_path, archive_path);
+        pyi_path_join(status->homepath, root_path, contents_directory);
     }
-#else
-    pyi_path_dirname(status->homepath, archive_path);
-#endif
+
     /*
      * Initial value of mainpath is homepath. It might be overridden
      * by temppath if it is available.
      */
     status->has_temp_directory = false;
     strcpy(status->mainpath, status->homepath);
 
-    /* Open the archive */
-    if (pyi_arch_open(status)) {
-        /* If this is not an archive, we MUST close the file, */
-        /* otherwise the open file-handle will be reused when */
-        /* testing the next file. */
-        pyi_arch_close_fp(status);
-        return false;
-    }
     return true;
 }
 

bootloader/src/pyi_launch.c

@@ -66,6 +66,7 @@ _format_and_check_path(char *buf, const char *fmt, ...)
     };
     va_end(args);
 
+    VS("Checking for file %s\n", buf);
     return stat(buf, &tmp);
 }
 
@@ -196,6 +197,7 @@ _extract_dependency(ARCHIVE_STATUS *archive_pool[], const char *item)
     char archive_path[PATH_MAX];
 
     char dirname[PATH_MAX];
+    char homepath_parent[PATH_MAX];
 
     VS("LOADER: Extracting dependency/reference %s\n", item);
 
@@ -204,16 +206,21 @@ _extract_dependency(ARCHIVE_STATUS *archive_pool[], const char *item)
     }
 
     pyi_path_dirname(dirname, path);
+    pyi_path_dirname(homepath_parent, archive_status->homepath);
+    char * contents_directory = pyi_arch_get_option(archive_pool[0], "pyi-contents-directory");
+    if (!contents_directory) {
+        FATALERROR("pyi-contents-directory option not found in onedir bundle archive!");
+        return -1;
+    }
 
     /* We need to identify and handle three situations:
      *  1) dependencies are in a onedir archive next to the current onefile archive,
      *  2) dependencies are in a onedir/onefile archive next to the current onedir archive,
      *  3) dependencies are in a onefile archive next to the current onefile archive.
      */
-    VS("LOADER: Checking if file exists\n");
-
+    VS("LOADER: homepath is %s\n", archive_status->homepath);
     /* TODO implement pyi_path_join to accept variable length of arguments for this case. */
-    if (_format_and_check_path(srcpath, "%s%c%s%c%s", archive_status->homepath, PYI_SEP, dirname, PYI_SEP, filename) == 0) {
+    if (_format_and_check_path(srcpath, "%s%c%s%c%s%c%s", homepath_parent, PYI_SEP, dirname, PYI_SEP, contents_directory, PYI_SEP, filename) == 0) {
         VS("LOADER: File %s found, assuming onedir reference\n", srcpath);
 
         if (_copy_dependency_from_dir(archive_status, srcpath, filename) == -1) {
@@ -222,7 +229,7 @@ _extract_dependency(ARCHIVE_STATUS *archive_pool[], const char *item)
         }
         /* TODO implement pyi_path_join to accept variable length of arguments for this case. */
     }
-    else if (_format_and_check_path(srcpath, "%s%c%s%c%s%c%s", archive_status->homepath, PYI_SEP, "..", PYI_SEP, dirname, PYI_SEP, filename) == 0) {
+    else if (_format_and_check_path(srcpath, "%s%c%s%c%s%c%s", homepath_parent, PYI_SEP, dirname, PYI_SEP, contents_directory, PYI_SEP, filename) == 0) {
         VS("LOADER: File %s found, assuming onedir reference\n", srcpath);
 
         if (_copy_dependency_from_dir(archive_status, srcpath, filename) == -1) {
@@ -234,9 +241,9 @@ _extract_dependency(ARCHIVE_STATUS *archive_pool[], const char *item)
         VS("LOADER: File %s not found, assuming onefile reference.\n", srcpath);
 
         /* TODO implement pyi_path_join to accept variable length of arguments for this case. */
-        if ((_format_and_check_path(archive_path, "%s%c%s.pkg", archive_status->homepath, PYI_SEP, path) != 0) &&
-            (_format_and_check_path(archive_path, "%s%c%s.exe", archive_status->homepath, PYI_SEP, path) != 0) &&
-            (_format_and_check_path(archive_path, "%s%c%s", archive_status->homepath, PYI_SEP, path) != 0)) {
+        if ((_format_and_check_path(archive_path, "%s%c%s%c%s.pkg", homepath_parent, PYI_SEP, contents_directory, PYI_SEP, path) != 0) &&
+            (_format_and_check_path(archive_path, "%s%c%s.exe", homepath_parent, PYI_SEP, path) != 0) &&
+            (_format_and_check_path(archive_path, "%s%c%s", homepath_parent, PYI_SEP, path) != 0)) {
             FATALERROR("Archive not found: %s\n", archive_path);
             return -1;
         }

bootloader/src/pyi_main.c

@@ -84,7 +84,6 @@ pyi_main(int argc, char * argv[])
     ARCHIVE_STATUS *archive_status = NULL;
     SPLASH_STATUS *splash_status = NULL;
     char executable[PATH_MAX];
-    char homepath[PATH_MAX];
     char archivefile[PATH_MAX];
     int rc = 0;
     int in_child = 0;
@@ -102,8 +101,7 @@ pyi_main(int argc, char * argv[])
         return -1;
     }
     if ((! pyi_path_executable(executable, argv[0])) ||
-        (! pyi_path_archivefile(archivefile, executable)) ||
-        (! pyi_path_homepath(homepath, executable))) {
+        (! pyi_path_archivefile(archivefile, executable))) {
         return -1;
     }
 
@@ -196,7 +194,7 @@ pyi_main(int argc, char * argv[])
     /* On Windows and Mac use single-process for --onedir mode. */
     if (!extractionpath && !pyi_launch_need_to_extract_binaries(archive_status)) {
         VS("LOADER: No need to extract files to run; setting extractionpath to homepath\n");
-        extractionpath = homepath;
+        extractionpath = archive_status->homepath;
     }
 
 #else
@@ -211,7 +209,7 @@ pyi_main(int argc, char * argv[])
 
         /* Set _MEIPASS2, so that the restarted bootloader process will enter
          * the codepath that corresponds to child process. */
-        pyi_setenv("_MEIPASS2", homepath);
+        pyi_setenv("_MEIPASS2", archive_status->homepath);
 
         /* Set _PYI_ONEDIR_MODE to signal to restarted bootloader that it
          * should reset in_child variable even though it is operating in
@@ -296,7 +294,7 @@ pyi_main(int argc, char * argv[])
         /*  If binaries were extracted to temppath,
          *  we pass it through status variable
          */
-        if (strcmp(homepath, extractionpath) != 0) {
+        if (strcmp(archive_status->homepath, extractionpath) != 0) {
             if (snprintf(archive_status->temppath, PATH_MAX,
                          "%s", extractionpath) >= PATH_MAX) {
                 VS("LOADER: temppath exceeds PATH_MAX\n");
@@ -372,7 +370,7 @@ pyi_main(int argc, char * argv[])
         VS("LOADER: Executing self as child\n");
         pyi_setenv("_MEIPASS2",
                    archive_status->temppath[0] !=
-                   0 ? archive_status->temppath : homepath);
+                   0 ? archive_status->temppath : archive_status->homepath);
 
         VS("LOADER: set _MEIPASS2 to %s\n", pyi_getenv("_MEIPASS2"));
 

bootloader/src/pyi_path.c

@@ -411,18 +411,6 @@ pyi_path_executable(char *execfile, const char *appname)
     return true;
 }
 
-/*
- * Return absolute path to homepath. It is the directory containing executable.
- */
-bool
-pyi_path_homepath(char *homepath, const char *thisfile)
-{
-    /* Fill in here (directory of thisfile). */
-    bool rc = pyi_path_dirname(homepath, thisfile);
-    VS("LOADER: homepath is %s\n", homepath);
-    return rc;
-}
-
 /*
  * Return full path to an external PYZ-archive.
  * The name is based on the excutable's name: path/myappname.pkg

doc/operating-mode.rst

@@ -133,12 +133,6 @@ than the entire folder.
 or different dependencies, or if the dependencies
 are upgraded, you must redistribute the whole bundle.)
 
-A small disadvantage of the one-folder format is that the one folder contains
-a large number of files.
-Your user must find the :file:`myscript` executable
-in a long list of names or among a big array of icons.
-Also your user can create
-a problem by accidentally dragging files out of the folder.
 
 .. _how the one-folder program works:
 

doc/runtime-information.rst

@@ -16,7 +16,8 @@ check "are we bundled?"::
 
 When a bundled app starts up, the bootloader sets the ``sys.frozen``
 attribute and stores the absolute path to the bundle folder in
-``sys._MEIPASS``. For a one-folder bundle, this is the path to that folder. For
+``sys._MEIPASS``. For a one-folder bundle, this is the path to the
+``_internal`` folder within the bundle. For
 a one-file bundle, this is the path to the temporary folder created by the
 bootloader (see :ref:`How the One-File Program Works`).
 

news/7713.breaking.rst

@@ -0,0 +1,13 @@
+All of onedir build's contents except for the executable are now moved into a
+sub-directory (called ``_internal`` by default). ``sys._MEIPASS`` is adjusted to
+point to this ``_internal`` directory. The breaking implications for this are:
+
+* Assumptions that ``os.path.dirname(sys.executable) == sys._MEIPASS`` will
+  break. Code locating application resources using
+  ``os.path.dirname(sys.executable)`` should be adjusted to use ``__file__``
+  or ``sys._MEIPASS`` and any code locating the original executable using
+  ``sys._MEIPASS`` should use :data:`sys.executable` directly.
+
+* Any custom post processing steps (either in the ``.spec`` file or
+  externally) which modify the bundle will likely need adjusting to
+  accommodate the new directory.

news/7713.feature.rst

@@ -0,0 +1,5 @@
+Restructure onedir mode builds so that everything except the executable (and
+``.pkg`` if you're using external PYZ archive mode) are hidden inside a
+sub-directory. This sub-directory's name defaults to ``_internal`` but may be
+configured with the new :option:`--internals-prefix` option. Onefile
+applications and macOS ``.app`` bundles are unaffected.

tests/functional/test_basic.py

@@ -771,3 +771,23 @@ def test_package_entry_point_name_collision(pyi_builder):
     exe, = pyi_builder._find_executables("matching_name")
     p = subprocess.run([exe], stdout=subprocess.PIPE, universal_newlines=True)
     assert re.findall("Running (.*) as (.*)", p.stdout) == expected
+
+
+def test_contents_directory(pyi_builder):
+    """
+    Test the --contents-directory option, including changing it without --clean.
+    """
+    if pyi_builder._mode != 'onedir':
+        pytest.skip('--contents-directory does not affect onefile builds.')
+
+    pyi_builder.test_source("", pyi_args=["--contents-directory=foo"])
+    exe, = pyi_builder._find_executables("test_source")
+    bundle = Path(exe).parent
+    assert (bundle / "foo").is_dir()
+
+    pyi_builder.test_source("", pyi_args=["--contents-directory=é³þ³źć🚀", "--noconfirm"])
+    assert not (bundle / "foo").exists()
+    assert (bundle / "é³þ³źć🚀").is_dir()
+
+    with pytest.raises(SystemExit, match='Invalid value "\\." passed'):
+        pyi_builder.test_source("", pyi_args=["--contents-directory=.", "--noconfirm"])