Separate Basilisp namespaces from python module hierarchy (#957)

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
  * Added several missing functions to `basilisp.core` (#956)
 
+### Changed
+ * Separate basilisp namespaces from python module hierarchy (#957)
+
 ## [v0.1.0]
 ### Added
  * Added `:end-line` and `:end-col` metadata to forms during compilation (#903)

docs/gettingstarted.rst

@@ -145,10 +145,10 @@ Specifically, any file with a ``.pth`` extension located in any of the known ``s
    $ python
    Python 3.12.1 (main, Jan  3 2024, 10:01:43) [GCC 11.4.0] on linux
    Type "help", "copyright", "credits" or "license" for more information.
-   >>> import importlib; importlib.import_module("basilisp.core")
-   <module 'basilisp.core' (/home/chris/Projects/basilisp/src/basilisp/core.lpy)>
+   >>> import importlib; importlib.import_module("basilisp.core.basilisp_namespace")
+   <module 'basilisp.core.basilisp_namespace' (/home/chris/Projects/basilisp/src/basilisp/core.lpy)>
 
-This method also enables you to directly execute Basilisp scripts as Python modules using ``python -m {namespace}``.
+This method also enables you to directly execute Basilisp scripts as Python modules using ``python -m {namespace}.basilisp_namespace``.
 Basilisp namespaces run as a Python module directly via ``python -m`` are resolved within the context of the current ``sys.path`` of the active Python interpreter.
 
 .. code-block:: bash

src/basilisp/cli.py

@@ -18,7 +18,6 @@
 from basilisp.lang import symbol as sym
 from basilisp.lang import vector as vec
 from basilisp.lang.exception import print_exception
-from basilisp.lang.util import munge
 from basilisp.prompt import get_prompter
 
 CLI_INPUT_FILE_PATH = "<CLI Input>"
@@ -82,7 +81,7 @@ def bootstrap_repl(ctx: compiler.CompilerContext, which_ns: str) -> types.Module
         ctx,
         ns,
     )
-    return importlib.import_module(REPL_NS)
+    return runtime.import_namespace(REPL_NS)
 
 
 def _to_bool(v: Optional[str]) -> Optional[bool]:
@@ -357,7 +356,7 @@ def nrepl_server(
 ) -> None:
     opts = compiler.compiler_opts()
     basilisp.init(opts)
-    nrepl_server_mod = importlib.import_module(munge(NREPL_SERVER_NS))
+    nrepl_server_mod = runtime.import_namespace(NREPL_SERVER_NS)
     nrepl_server_mod.start_server__BANG__(
         lmap.map(
             {

src/basilisp/contrib/pytest/testrunner.py

@@ -1,4 +1,3 @@
-import importlib.util
 import inspect
 import os
 import traceback
@@ -219,7 +218,7 @@ def teardown(self) -> None:
 
     def _import_module(self) -> runtime.BasilispModule:
         modname = _get_fully_qualified_module_name(self.path)
-        module = importlib.import_module(modname)
+        module = runtime.import_namespace(modname)
         assert isinstance(module, runtime.BasilispModule)
         return module
 

src/basilisp/importer.py

@@ -116,6 +116,22 @@ def _is_package(path: str) -> bool:
     return False
 
 
+def _is_namespace(fullname: str) -> bool:
+    """Return True if module fullname represents a loadable basilisp
+    namespace file and not a package module.
+    """
+    return fullname.endswith(runtime.NS_MODULE_SUFFIX)
+
+
+def _namespace_symbol(name: str) -> sym.Symbol:
+    """return the namespace name for basilisp module name"""
+    if _is_namespace(name):
+        name = name[: -len(runtime.NS_MODULE_SUFFIX)]
+    else:
+        name = name + runtime.PACKAGE_NS_SUFFIX
+    return sym.symbol(demunge(name))
+
+
 @lru_cache()
 def _is_namespace_package(path: str) -> bool:
     """Return True if the current directory is a namespace Basilisp package.
@@ -150,7 +166,9 @@ def find_spec(
 
         Returns None if the module is not a Basilisp module to allow import processing to continue.
         """
+        logger.debug(f"Searching for module {fullname}")
         package_components = fullname.split(".")
+        is_namespace = _is_namespace(fullname)
         if not path:
             path = sys.path
             module_name = package_components
@@ -159,45 +177,49 @@ def find_spec(
 
         for entry in path:
             root_path = os.path.join(entry, *module_name)
-            filenames = [
-                f"{os.path.join(root_path, '__init__')}.lpy",
-                f"{root_path}.lpy",
-            ]
-            for filename in filenames:
-                if os.path.isfile(filename):
-                    state = {
-                        "fullname": fullname,
-                        "filename": filename,
-                        "path": entry,
-                        "target": target,
-                        "cache_filename": _cache_from_source(filename),
-                    }
-                    logger.debug(
-                        f"Found potential Basilisp module '{fullname}' in file '{filename}'"
-                    )
-                    is_package = filename.endswith("__init__.lpy") or _is_package(
-                        root_path
-                    )
-                    spec = ModuleSpec(
-                        fullname,
-                        self,
-                        origin=filename,
-                        loader_state=state,
-                        is_package=is_package,
-                    )
-                    # The Basilisp loader can find packages regardless of
-                    # submodule_search_locations, but the Python loader cannot.
-                    # Set this to the root path to allow the Python loader to
-                    # load submodules of Basilisp "packages".
-                    if is_package:
-                        assert (
-                            spec.submodule_search_locations is not None
-                        ), "Package module spec must have submodule_search_locations list"
-                        spec.submodule_search_locations.append(root_path)
-                    return spec
+            if is_namespace:
+                root_path = root_path[: -len(runtime.NS_MODULE_SUFFIX)]
+                filename = f"{root_path}.lpy"
+            else:
+                filename = f"{os.path.join(root_path, '__init__')}.lpy"
+            is_package = _is_package(root_path)
+
+            if os.path.isfile(filename):
+                state = {
+                    "fullname": fullname,
+                    "filename": filename,
+                    "path": entry,
+                    "target": target,
+                    "cache_filename": _cache_from_source(filename),
+                }
+                logger.debug(
+                    f"Found potential Basilisp module '{fullname}' in file '{filename}'"
+                )
+                spec = ModuleSpec(
+                    fullname,
+                    self,
+                    origin=filename,
+                    loader_state=state,
+                    is_package=is_package,
+                )
+                # The Basilisp loader can find packages regardless of
+                # submodule_search_locations, but the Python loader cannot.
+                # Set this to the root path to allow the Python loader to
+                # load submodules of Basilisp "packages".
+                if is_package:
+                    assert (
+                        spec.submodule_search_locations is not None
+                    ), "Package module spec must have submodule_search_locations list"
+                    spec.submodule_search_locations.append(root_path)
+                return spec
+
             if os.path.isdir(root_path):
                 if _is_namespace_package(root_path):
+                    logger.debug(f"Found namespace package {fullname}")
                     return ModuleSpec(fullname, None, is_package=True)
+            elif os.path.isfile(f"{root_path}.lpy"):
+                logger.debug(f"Found psuedo-package {fullname}")
+                return ModuleSpec(fullname, None, is_package=True)
         return None
 
     def invalidate_caches(self) -> None:
@@ -246,7 +268,9 @@ def get_code(self, fullname: str) -> Optional[types.CodeType]:
             # Set the *main-ns* variable to the current namespace.
             main_ns_var = core_ns.find(sym.symbol(runtime.MAIN_NS_VAR_NAME))
             assert main_ns_var is not None
-            main_ns_var.bind_root(sym.symbol(demunge(fullname)))
+
+            ns_sym = _namespace_symbol(fullname)
+            main_ns_var.bind_root(ns_sym)
 
             # Set command line args passed to the module
             if pyargs := sys.argv[1:]:
@@ -264,7 +288,7 @@ def get_code(self, fullname: str) -> Optional[types.CodeType]:
             #
             # The target namespace is free to interpret
             code: List[types.CodeType] = []
-            path = "/" + "/".join(fullname.split("."))
+            path = "/" + "/".join(ns_sym.name.split("."))
             try:
                 compiler.load(
                     path,
@@ -307,7 +331,9 @@ def _exec_cached_module(
                 f"Loaded cached Basilisp module '{fullname}' in {duration / 1000000}ms"
             )
         ):
-            logger.debug(f"Checking for cached Basilisp module '{fullname}''")
+            logger.debug(
+                f"Checking for cached Basilisp module '{fullname}' for namespace '{ns}' in file '{filename}'"
+            )
             cache_data = self.get_data(cache_filename)
             cached_code = _get_basilisp_bytecode(
                 fullname, path_stats["mtime"], path_stats["size"], cache_data
@@ -385,8 +411,9 @@ def exec_module(self, module: types.ModuleType) -> None:
         # a blank module. If we do not replace the module here with the module we are
         # generating, then we will not be able to use advanced compilation features such
         # as direct Python variable access to functions and other def'ed values.
-        ns_name = demunge(fullname)
-        ns: runtime.Namespace = runtime.Namespace.get_or_create(sym.symbol(ns_name))
+        ns: runtime.Namespace = runtime.Namespace.get_or_create(
+            _namespace_symbol(fullname)
+        )
         ns.module = module
         module.__basilisp_namespace__ = ns
 

src/basilisp/lang/runtime.py

@@ -82,6 +82,13 @@
     (int(s) if s.isdigit() else s) for s in BASILISP_VERSION_STRING.split(".")
 )
 
+# Modules for a namespace are given this suffix. it will be used for
+# calling namespaces as python modules
+NS_MODULE_SUFFIX = ".basilisp_namespace"
+
+# Namespaces given to package modules e.g: `__init__.lpy`
+PACKAGE_NS_SUFFIX = ".--basilisp-package--"
+
 # Public basilisp.core symbol names
 COMPILER_OPTIONS_VAR_NAME = "*compiler-options*"
 COMMAND_LINE_ARGS_VAR_NAME = "*command-line-args*"
@@ -155,6 +162,10 @@
 )
 
 
+def import_namespace(namespace: str) -> types.ModuleType:
+    return importlib.import_module(munge(namespace) + NS_MODULE_SUFFIX)
+
+
 # Reader Conditional default features
 def _supported_python_versions_features() -> Iterable[kw.Keyword]:
     """Yield successive reader features corresponding to the various Python
@@ -204,7 +215,7 @@ class BasilispModule(types.ModuleType):
 def _new_module(name: str, doc=None) -> BasilispModule:
     """Create a new empty Basilisp Python module.
     Modules are created for each Namespace when it is created."""
-    mod = BasilispModule(name, doc=doc)
+    mod = BasilispModule(name + NS_MODULE_SUFFIX, doc=doc)
     mod.__loader__ = None
     mod.__package__ = None
     mod.__spec__ = None
@@ -661,7 +672,7 @@ def require(self, ns_name: str, *aliases: sym.Symbol) -> BasilispModule:
 
         This method is called in code generated for the `require*` special form."""
         try:
-            ns_module = importlib.import_module(munge(ns_name))
+            ns_module = import_namespace(ns_name)
         except ModuleNotFoundError as e:
             raise ImportError(
                 f"Basilisp namespace '{ns_name}' not found",

src/basilisp/main.py

@@ -1,4 +1,3 @@
-import importlib
 import logging
 import site
 from pathlib import Path
@@ -17,32 +16,36 @@
 
 
 def init(opts: Optional[CompilerOpts] = None) -> None:
-    """
-    Initialize the runtime environment for Basilisp code evaluation.
+    """Initialize the runtime environment for Basilisp code evaluation.
 
-    Basilisp only needs to be initialized once per Python VM invocation. Subsequent
-    imports of Basilisp namespaces will work using Python's standard ``import``
-    statement and :external:py:func:`importlib.import_module` function.
+    Basilisp only needs to be initialized once per Python VM
+    invocation. Subsequent imports of Basilisp namespaces will work using
+    Python's standard ``import`` statement and
+    :external:py:func:`importlib.import_module` function. (with appropriate
+    namespace module suffix)
 
     If you want to execute a Basilisp file which is stored in a well-formed package
     or module structure, you probably want to use :py:func:`bootstrap`.
+
     """
     runtime.init_ns_var()
     runtime.bootstrap_core(opts if opts is not None else compiler_opts())
     importer.hook_imports()
-    importlib.import_module("basilisp.core")
+    runtime.import_namespace("basilisp.core")
 
 
 def bootstrap(
     target: str, opts: Optional[CompilerOpts] = None
 ) -> None:  # pragma: no cover
-    """
-    Import a Basilisp namespace or function identified by ``target``. If a function
+    """Import a Basilisp namespace or function identified by ``target``. If a function
     reference is given, the function will be called with no arguments.
 
-    Basilisp only needs to be initialized once per Python VM invocation. Subsequent
-    imports of Basilisp namespaces will work using Python's standard ``import``
-    statement and :external:py:func:`importlib.import_module` function.
+    Basilisp only needs to be initialized once per Python VM
+    invocation. Subsequent imports of Basilisp namespaces will work using
+    Python's standard ``import`` statement and
+    :external:py:func:`importlib.import_module` function. (with appropriate
+    namespace module suffix)
+
 
     ``target`` must be a string naming a Basilisp namespace. Namespace references may
     be given exactly as they are found in Basilisp code. ``target`` may optionally
@@ -51,10 +54,11 @@ def bootstrap(
 
     ``opts`` is a mapping of compiler options that may be supplied for bootstrapping.
     This setting should be left alone unless you know what you are doing.
+
     """
     init(opts=opts)
     pkg_name, *rest = target.split(":", maxsplit=1)
-    mod = importlib.import_module(munge(pkg_name))
+    mod = runtime.import_namespace(pkg_name)
     if rest:
         fn_name = munge(rest[0])
         getattr(mod, fn_name)()

tests/basilisp/compiler_test.py

@@ -1363,7 +1363,7 @@ def test_deftype_can_have_classmethod(
                 (cls x y z))
               (__eq__ [this other]
                 (operator/eq
-                 [x y z] 
+                 [x y z]
                  [(.-x other) (.-y other) (.-z other)])))"""
             )
             assert Point(1, 2, 3) == Point.create(1, 2, 3)
@@ -5212,7 +5212,7 @@ def test_require_namespace_must_exist(self, lcompile: CompileFn):
     @pytest.fixture
     def _import_ns(self, ns: runtime.Namespace):
         def _import_ns_module(name: str):
-            ns_module = importlib.import_module(name)
+            ns_module = runtime.import_namespace(name)
             runtime.set_current_ns(ns.name)
             return ns_module
 

tests/basilisp/core_test.py

@@ -29,7 +29,7 @@ def setup_module():
     runtime.print_generated_python = orig
 
 
-import basilisp.core as core  # isort:skip
+import basilisp.core.basilisp_namespace as core  # isort:skip
 
 TRUTHY_VALUES = [
     True,