bpo-45020: Add more test cases for frozen modules. (gh-28664)

I've added a number of test-only modules. Some of those cases are covered by the recently frozen stdlib modules (and some will be once we add encodings back in). However, I figured we'd play it safe by having a set of modules guaranteed to be there during tests.

https://bugs.python.org/issue45020

Lib/__phello__/__init__.py

@@ -0,0 +1,7 @@
+initialized = True
+
+def main():
+    print("Hello world!")
+
+if __name__ == '__main__':
+    main()

Lib/__phello__/spam.py

@@ -0,0 +1,7 @@
+initialized = True
+
+def main():
+    print("Hello world!")
+
+if __name__ == '__main__':
+    main()

Lib/test/test_frozen.py

@@ -10,6 +10,7 @@
 # Invalid marshalled data in frozen.c could case the interpreter to
 # crash when __hello__ is imported.
 
+import importlib.machinery
 import sys
 import unittest
 from test.support import captured_stdout, import_helper
@@ -26,6 +27,33 @@ def test_frozen(self):
             __hello__.main()
         self.assertEqual(out.getvalue(), 'Hello world!\n')
 
+    def test_frozen_submodule_in_unfrozen_package(self):
+        with import_helper.CleanImport('__phello__', '__phello__.spam'):
+            with import_helper.frozen_modules(enabled=False):
+                import __phello__
+            with import_helper.frozen_modules(enabled=True):
+                import __phello__.spam as spam
+        self.assertIs(spam, __phello__.spam)
+        self.assertIsNot(__phello__.__spec__.loader,
+                         importlib.machinery.FrozenImporter)
+        self.assertIs(spam.__spec__.loader,
+                      importlib.machinery.FrozenImporter)
+
+    # This is not possible until frozen packages have __path__ set properly.
+    # See https://bugs.python.org/issue21736.
+    @unittest.expectedFailure
+    def test_unfrozen_submodule_in_frozen_package(self):
+        with import_helper.CleanImport('__phello__', '__phello__.spam'):
+            with import_helper.frozen_modules(enabled=True):
+                import __phello__
+            with import_helper.frozen_modules(enabled=False):
+                import __phello__.spam as spam
+        self.assertIs(spam, __phello__.spam)
+        self.assertIs(__phello__.__spec__.loader,
+                      importlib.machinery.FrozenImporter)
+        self.assertIsNot(spam.__spec__.loader,
+                         importlib.machinery.FrozenImporter)
+
 
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_importlib/frozen/test_finder.py

@@ -1,4 +1,5 @@
 from .. import abc
+import os.path
 from .. import util
 
 machinery = util.import_importlib('importlib.machinery')
@@ -13,34 +14,86 @@ class FindSpecTests(abc.FinderTests):
 
     """Test finding frozen modules."""
 
-    def find(self, name, path=None):
+    def find(self, name, **kwargs):
         finder = self.machinery.FrozenImporter
         with import_helper.frozen_modules():
-            return finder.find_spec(name, path)
+            return finder.find_spec(name, **kwargs)
 
-    def test_module(self):
-        name = '__hello__'
-        spec = self.find(name)
+    def check(self, spec, name):
+        self.assertEqual(spec.name, name)
+        self.assertIs(spec.loader, self.machinery.FrozenImporter)
         self.assertEqual(spec.origin, 'frozen')
+        self.assertFalse(spec.has_location)
 
-    def test_package(self):
-        spec = self.find('__phello__')
-        self.assertIsNotNone(spec)
-
-    def test_module_in_package(self):
-        spec = self.find('__phello__.spam', ['__phello__'])
-        self.assertIsNotNone(spec)
+    def test_module(self):
+        names = [
+            '__hello__',
+            '__hello_alias__',
+            '__hello_only__',
+            '__phello__.__init__',
+            '__phello__.spam',
+            '__phello__.ham.__init__',
+            '__phello__.ham.eggs',
+        ]
+        for name in names:
+            with self.subTest(name):
+                spec = self.find(name)
+                self.check(spec, name)
+                self.assertEqual(spec.submodule_search_locations, None)
 
-    # No frozen package within another package to test with.
+    def test_package(self):
+        names = [
+            '__phello__',
+            '__phello__.ham',
+            '__phello_alias__',
+        ]
+        for name in names:
+            with self.subTest(name):
+                spec = self.find(name)
+                self.check(spec, name)
+                self.assertEqual(spec.submodule_search_locations, [])
+
+    # These are covered by test_module() and test_package().
+    test_module_in_package = None
     test_package_in_package = None
 
     # No easy way to test.
     test_package_over_module = None
 
+    def test_path_ignored(self):
+        for name in ('__hello__', '__phello__', '__phello__.spam'):
+            actual = self.find(name)
+            for path in (None, object(), '', 'eggs', [], [''], ['eggs']):
+                with self.subTest((name, path)):
+                    spec = self.find(name, path=path)
+                    self.assertEqual(spec, actual)
+
+    def test_target_ignored(self):
+        imported = ('__hello__', '__phello__')
+        with import_helper.CleanImport(*imported, usefrozen=True):
+            import __hello__ as match
+            import __phello__ as nonmatch
+        name = '__hello__'
+        actual = self.find(name)
+        for target in (None, match, nonmatch, object(), 'not-a-module-object'):
+            with self.subTest(target):
+                spec = self.find(name, target=target)
+                self.assertEqual(spec, actual)
+
     def test_failure(self):
         spec = self.find('<not real>')
         self.assertIsNone(spec)
 
+    def test_not_using_frozen(self):
+        finder = self.machinery.FrozenImporter
+        with import_helper.frozen_modules(enabled=False):
+            # both frozen and not frozen
+            spec1 = finder.find_spec('__hello__')
+            # only frozen
+            spec2 = finder.find_spec('__hello_only__')
+        self.assertIsNone(spec1)
+        self.assertIsNone(spec2)
+
 
 (Frozen_FindSpecTests,
  Source_FindSpecTests

Makefile.pre.in

@@ -752,7 +752,12 @@ FROZEN_FILES_IN = \
 		Lib/os.py \
 		Lib/site.py \
 		Lib/stat.py \
-		Lib/__hello__.py
+		Lib/__hello__.py \
+		Lib/__phello__/__init__.py \
+		Lib/__phello__/ham/__init__.py \
+		Lib/__phello__/ham/eggs.py \
+		Lib/__phello__/spam.py \
+		Tools/freeze/flag.py
 # End FROZEN_FILES_IN
 FROZEN_FILES_OUT = \
 		Python/frozen_modules/importlib._bootstrap.h \
@@ -769,7 +774,12 @@ FROZEN_FILES_OUT = \
 		Python/frozen_modules/os.h \
 		Python/frozen_modules/site.h \
 		Python/frozen_modules/stat.h \
-		Python/frozen_modules/__hello__.h
+		Python/frozen_modules/__hello__.h \
+		Python/frozen_modules/__phello__.h \
+		Python/frozen_modules/__phello__.ham.h \
+		Python/frozen_modules/__phello__.ham.eggs.h \
+		Python/frozen_modules/__phello__.spam.h \
+		Python/frozen_modules/frozen_only.h
 # End FROZEN_FILES_OUT
 
 Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
@@ -824,6 +834,21 @@ Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
 Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py
 	Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py $(srcdir)/Python/frozen_modules/__hello__.h
 
+Python/frozen_modules/__phello__.h: Programs/_freeze_module Lib/__phello__/__init__.py
+	Programs/_freeze_module __phello__ $(srcdir)/Lib/__phello__/__init__.py $(srcdir)/Python/frozen_modules/__phello__.h
+
+Python/frozen_modules/__phello__.ham.h: Programs/_freeze_module Lib/__phello__/ham/__init__.py
+	Programs/_freeze_module __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py $(srcdir)/Python/frozen_modules/__phello__.ham.h
+
+Python/frozen_modules/__phello__.ham.eggs.h: Programs/_freeze_module Lib/__phello__/ham/eggs.py
+	Programs/_freeze_module __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h
+
+Python/frozen_modules/__phello__.spam.h: Programs/_freeze_module Lib/__phello__/spam.py
+	Programs/_freeze_module __phello__.spam $(srcdir)/Lib/__phello__/spam.py $(srcdir)/Python/frozen_modules/__phello__.spam.h
+
+Python/frozen_modules/frozen_only.h: Programs/_freeze_module Tools/freeze/flag.py
+	Programs/_freeze_module frozen_only $(srcdir)/Tools/freeze/flag.py $(srcdir)/Python/frozen_modules/frozen_only.h
+
 # END: freezing modules
 
 Tools/scripts/freeze_modules.py: Programs/_freeze_module

PCbuild/_freeze_module.vcxproj

@@ -305,6 +305,31 @@
       <IntFile>$(IntDir)__hello__.g.h</IntFile>
       <OutFile>$(PySourcePath)Python\frozen_modules\__hello__.h</OutFile>
     </None>
+    <None Include="..\Lib\__phello__\__init__.py">
+      <ModName>__phello__</ModName>
+      <IntFile>$(IntDir)__phello__.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.h</OutFile>
+    </None>
+    <None Include="..\Lib\__phello__\ham\__init__.py">
+      <ModName>__phello__.ham</ModName>
+      <IntFile>$(IntDir)__phello__.ham.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.h</OutFile>
+    </None>
+    <None Include="..\Lib\__phello__\ham\eggs.py">
+      <ModName>__phello__.ham.eggs</ModName>
+      <IntFile>$(IntDir)__phello__.ham.eggs.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h</OutFile>
+    </None>
+    <None Include="..\Lib\__phello__\spam.py">
+      <ModName>__phello__.spam</ModName>
+      <IntFile>$(IntDir)__phello__.spam.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.spam.h</OutFile>
+    </None>
+    <None Include="..\Tools\freeze\flag.py">
+      <ModName>frozen_only</ModName>
+      <IntFile>$(IntDir)frozen_only.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\frozen_only.h</OutFile>
+    </None>
     <!-- END frozen modules -->
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

PCbuild/_freeze_module.vcxproj.filters

@@ -61,6 +61,21 @@
     <None Include="..\Lib\__hello__.py">
       <Filter>Python Files</Filter>
     </None>
+    <None Include="..\Lib\__phello__\__init__.py">
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Lib\__phello__\ham\__init__.py">
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Lib\__phello__\ham\eggs.py">
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Lib\__phello__\spam.py">
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Tools\freeze\flag.py">
+      <Filter>Python Files</Filter>
+    </None>
     <!-- END frozen modules -->
   </ItemGroup>
 </Project>

Python/frozen.c

@@ -53,6 +53,11 @@
 #include "frozen_modules/site.h"
 #include "frozen_modules/stat.h"
 #include "frozen_modules/__hello__.h"
+#include "frozen_modules/__phello__.h"
+#include "frozen_modules/__phello__.ham.h"
+#include "frozen_modules/__phello__.ham.eggs.h"
+#include "frozen_modules/__phello__.spam.h"
+#include "frozen_modules/frozen_only.h"
 /* End includes */
 
 /* Note that a negative size indicates a package. */
@@ -84,8 +89,19 @@ static const struct _frozen _PyImport_FrozenModules[] = {
 
     /* Test module */
     {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
-    {"__phello__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__)},
-    {"__phello__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
+    {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
+    {"__phello_alias__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__)},
+    {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
+    {"__phello__", _Py_M____phello__, -(int)sizeof(_Py_M____phello__)},
+    {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__)},
+    {"__phello__.ham", _Py_M____phello___ham, -(int)sizeof(_Py_M____phello___ham)},
+    {"__phello__.ham.__init__", _Py_M____phello___ham,
+        (int)sizeof(_Py_M____phello___ham)},
+    {"__phello__.ham.eggs", _Py_M____phello___ham_eggs,
+        (int)sizeof(_Py_M____phello___ham_eggs)},
+    {"__phello__.spam", _Py_M____phello___spam,
+        (int)sizeof(_Py_M____phello___spam)},
+    {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
     {0, 0, 0} /* sentinel */
 };
 

Tools/scripts/freeze_modules.py

@@ -19,6 +19,7 @@
 
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 ROOT_DIR = os.path.abspath(ROOT_DIR)
+FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')
 
 STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
 # If MODULES_DIR is changed then the .gitattributes and .gitignore files
@@ -53,7 +54,6 @@ def find_tool():
 MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
 PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
 PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters')
-TEST_CTYPES = os.path.join(STDLIB_DIR, 'ctypes', 'test', 'test_values.py')
 
 
 OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath'
@@ -95,8 +95,11 @@ def find_tool():
         ]),
     ('Test module', [
         '__hello__',
-        '__hello__ : <__phello__>',
-        '__hello__ : __phello__.spam',
+        '__hello__ : __hello_alias__',
+        '__hello__ : <__phello_alias__>',
+        '__hello__ : __phello_alias__.spam',
+        '<__phello__.**.*>',
+        f'frozen_only : __hello_only__ = {FROZEN_ONLY}',
         ]),
 ]
 ESSENTIAL = {
@@ -135,14 +138,15 @@ def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):
     seen = {}
     for section, specs in sectionalspecs:
         parsed = _parse_specs(specs, section, seen)
-        for frozenid, pyfile, modname, ispkg, section in parsed:
+        for item in parsed:
+            frozenid, pyfile, modname, ispkg, section = item
             try:
                 source = seen[frozenid]
             except KeyError:
                 source = FrozenSource.from_id(frozenid, pyfile, destdir)
                 seen[frozenid] = source
             else:
-                assert not pyfile
+                assert not pyfile or pyfile == source.pyfile, item
             yield FrozenModule(modname, ispkg, section, source)
 
 
@@ -224,7 +228,6 @@ def _parse_spec(spec, knownids=None, section=None):
             pkgfiles = {pyfile: pkgid}
             def iter_subs():
                 for frozenid, pyfile, ispkg in resolved:
-                    assert not knownids or frozenid not in knownids, (frozenid, spec)
                     if pkgname:
                         modname = frozenid.replace(pkgid, pkgname, 1)
                     else:

Tools/scripts/generate_stdlib_module_names.py

@@ -21,6 +21,9 @@
     # Test modules and packages
     '__hello__',
     '__phello__',
+    '__hello_alias__',
+    '__phello_alias__',
+    '__hello_only__',
     '_ctypes_test',
     '_testbuffer',
     '_testcapi',