diff --git a/CHANGELOG.md b/CHANGELOG.md index fdd35575b..02bdea20c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added * Added several missing functions to `basilisp.core` (#956) +### Changed + * Separate basilisp namespaces from python module hierarchy (#957) + ## [v0.1.0] ### Added * Added `:end-line` and `:end-col` metadata to forms during compilation (#903) diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 4218a0516..7e92cd20a 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -145,10 +145,10 @@ Specifically, any file with a ``.pth`` extension located in any of the known ``s $ python Python 3.12.1 (main, Jan 3 2024, 10:01:43) [GCC 11.4.0] on linux Type "help", "copyright", "credits" or "license" for more information. - >>> import importlib; importlib.import_module("basilisp.core") - + >>> import importlib; importlib.import_module("basilisp.core.basilisp_namespace") + -This method also enables you to directly execute Basilisp scripts as Python modules using ``python -m {namespace}``. +This method also enables you to directly execute Basilisp scripts as Python modules using ``python -m {namespace}.basilisp_namespace``. Basilisp namespaces run as a Python module directly via ``python -m`` are resolved within the context of the current ``sys.path`` of the active Python interpreter. .. code-block:: bash diff --git a/src/basilisp/cli.py b/src/basilisp/cli.py index 8491f715e..5b27e4f53 100644 --- a/src/basilisp/cli.py +++ b/src/basilisp/cli.py @@ -18,7 +18,6 @@ from basilisp.lang import symbol as sym from basilisp.lang import vector as vec from basilisp.lang.exception import print_exception -from basilisp.lang.util import munge from basilisp.prompt import get_prompter CLI_INPUT_FILE_PATH = "" @@ -82,7 +81,7 @@ def bootstrap_repl(ctx: compiler.CompilerContext, which_ns: str) -> types.Module ctx, ns, ) - return importlib.import_module(REPL_NS) + return runtime.import_namespace(REPL_NS) def _to_bool(v: Optional[str]) -> Optional[bool]: @@ -357,7 +356,7 @@ def nrepl_server( ) -> None: opts = compiler.compiler_opts() basilisp.init(opts) - nrepl_server_mod = importlib.import_module(munge(NREPL_SERVER_NS)) + nrepl_server_mod = runtime.import_namespace(NREPL_SERVER_NS) nrepl_server_mod.start_server__BANG__( lmap.map( { diff --git a/src/basilisp/contrib/pytest/testrunner.py b/src/basilisp/contrib/pytest/testrunner.py index d3582aae0..4bf197e8b 100644 --- a/src/basilisp/contrib/pytest/testrunner.py +++ b/src/basilisp/contrib/pytest/testrunner.py @@ -1,4 +1,3 @@ -import importlib.util import inspect import os import traceback @@ -219,7 +218,7 @@ def teardown(self) -> None: def _import_module(self) -> runtime.BasilispModule: modname = _get_fully_qualified_module_name(self.path) - module = importlib.import_module(modname) + module = runtime.import_namespace(modname) assert isinstance(module, runtime.BasilispModule) return module diff --git a/src/basilisp/importer.py b/src/basilisp/importer.py index 47566a461..6007e8d99 100644 --- a/src/basilisp/importer.py +++ b/src/basilisp/importer.py @@ -116,6 +116,22 @@ def _is_package(path: str) -> bool: return False +def _is_namespace(fullname: str) -> bool: + """Return True if module fullname represents a loadable basilisp + namespace file and not a package module. + """ + return fullname.endswith(runtime.NS_MODULE_SUFFIX) + + +def _namespace_symbol(name: str) -> sym.Symbol: + """return the namespace name for basilisp module name""" + if _is_namespace(name): + name = name[: -len(runtime.NS_MODULE_SUFFIX)] + else: + name = name + runtime.PACKAGE_NS_SUFFIX + return sym.symbol(demunge(name)) + + @lru_cache() def _is_namespace_package(path: str) -> bool: """Return True if the current directory is a namespace Basilisp package. @@ -150,7 +166,9 @@ def find_spec( Returns None if the module is not a Basilisp module to allow import processing to continue. """ + logger.debug(f"Searching for module {fullname}") package_components = fullname.split(".") + is_namespace = _is_namespace(fullname) if not path: path = sys.path module_name = package_components @@ -159,45 +177,49 @@ def find_spec( for entry in path: root_path = os.path.join(entry, *module_name) - filenames = [ - f"{os.path.join(root_path, '__init__')}.lpy", - f"{root_path}.lpy", - ] - for filename in filenames: - if os.path.isfile(filename): - state = { - "fullname": fullname, - "filename": filename, - "path": entry, - "target": target, - "cache_filename": _cache_from_source(filename), - } - logger.debug( - f"Found potential Basilisp module '{fullname}' in file '{filename}'" - ) - is_package = filename.endswith("__init__.lpy") or _is_package( - root_path - ) - spec = ModuleSpec( - fullname, - self, - origin=filename, - loader_state=state, - is_package=is_package, - ) - # The Basilisp loader can find packages regardless of - # submodule_search_locations, but the Python loader cannot. - # Set this to the root path to allow the Python loader to - # load submodules of Basilisp "packages". - if is_package: - assert ( - spec.submodule_search_locations is not None - ), "Package module spec must have submodule_search_locations list" - spec.submodule_search_locations.append(root_path) - return spec + if is_namespace: + root_path = root_path[: -len(runtime.NS_MODULE_SUFFIX)] + filename = f"{root_path}.lpy" + else: + filename = f"{os.path.join(root_path, '__init__')}.lpy" + is_package = _is_package(root_path) + + if os.path.isfile(filename): + state = { + "fullname": fullname, + "filename": filename, + "path": entry, + "target": target, + "cache_filename": _cache_from_source(filename), + } + logger.debug( + f"Found potential Basilisp module '{fullname}' in file '{filename}'" + ) + spec = ModuleSpec( + fullname, + self, + origin=filename, + loader_state=state, + is_package=is_package, + ) + # The Basilisp loader can find packages regardless of + # submodule_search_locations, but the Python loader cannot. + # Set this to the root path to allow the Python loader to + # load submodules of Basilisp "packages". + if is_package: + assert ( + spec.submodule_search_locations is not None + ), "Package module spec must have submodule_search_locations list" + spec.submodule_search_locations.append(root_path) + return spec + if os.path.isdir(root_path): if _is_namespace_package(root_path): + logger.debug(f"Found namespace package {fullname}") return ModuleSpec(fullname, None, is_package=True) + elif os.path.isfile(f"{root_path}.lpy"): + logger.debug(f"Found psuedo-package {fullname}") + return ModuleSpec(fullname, None, is_package=True) return None def invalidate_caches(self) -> None: @@ -246,7 +268,9 @@ def get_code(self, fullname: str) -> Optional[types.CodeType]: # Set the *main-ns* variable to the current namespace. main_ns_var = core_ns.find(sym.symbol(runtime.MAIN_NS_VAR_NAME)) assert main_ns_var is not None - main_ns_var.bind_root(sym.symbol(demunge(fullname))) + + ns_sym = _namespace_symbol(fullname) + main_ns_var.bind_root(ns_sym) # Set command line args passed to the module if pyargs := sys.argv[1:]: @@ -264,7 +288,7 @@ def get_code(self, fullname: str) -> Optional[types.CodeType]: # # The target namespace is free to interpret code: List[types.CodeType] = [] - path = "/" + "/".join(fullname.split(".")) + path = "/" + "/".join(ns_sym.name.split(".")) try: compiler.load( path, @@ -307,7 +331,9 @@ def _exec_cached_module( f"Loaded cached Basilisp module '{fullname}' in {duration / 1000000}ms" ) ): - logger.debug(f"Checking for cached Basilisp module '{fullname}''") + logger.debug( + f"Checking for cached Basilisp module '{fullname}' for namespace '{ns}' in file '{filename}'" + ) cache_data = self.get_data(cache_filename) cached_code = _get_basilisp_bytecode( fullname, path_stats["mtime"], path_stats["size"], cache_data @@ -385,8 +411,9 @@ def exec_module(self, module: types.ModuleType) -> None: # a blank module. If we do not replace the module here with the module we are # generating, then we will not be able to use advanced compilation features such # as direct Python variable access to functions and other def'ed values. - ns_name = demunge(fullname) - ns: runtime.Namespace = runtime.Namespace.get_or_create(sym.symbol(ns_name)) + ns: runtime.Namespace = runtime.Namespace.get_or_create( + _namespace_symbol(fullname) + ) ns.module = module module.__basilisp_namespace__ = ns diff --git a/src/basilisp/lang/runtime.py b/src/basilisp/lang/runtime.py index 4f58d93fc..c483e2ea9 100644 --- a/src/basilisp/lang/runtime.py +++ b/src/basilisp/lang/runtime.py @@ -82,6 +82,13 @@ (int(s) if s.isdigit() else s) for s in BASILISP_VERSION_STRING.split(".") ) +# Modules for a namespace are given this suffix. it will be used for +# calling namespaces as python modules +NS_MODULE_SUFFIX = ".basilisp_namespace" + +# Namespaces given to package modules e.g: `__init__.lpy` +PACKAGE_NS_SUFFIX = ".--basilisp-package--" + # Public basilisp.core symbol names COMPILER_OPTIONS_VAR_NAME = "*compiler-options*" COMMAND_LINE_ARGS_VAR_NAME = "*command-line-args*" @@ -155,6 +162,10 @@ ) +def import_namespace(namespace: str) -> types.ModuleType: + return importlib.import_module(munge(namespace) + NS_MODULE_SUFFIX) + + # Reader Conditional default features def _supported_python_versions_features() -> Iterable[kw.Keyword]: """Yield successive reader features corresponding to the various Python @@ -204,7 +215,7 @@ class BasilispModule(types.ModuleType): def _new_module(name: str, doc=None) -> BasilispModule: """Create a new empty Basilisp Python module. Modules are created for each Namespace when it is created.""" - mod = BasilispModule(name, doc=doc) + mod = BasilispModule(name + NS_MODULE_SUFFIX, doc=doc) mod.__loader__ = None mod.__package__ = None mod.__spec__ = None @@ -661,7 +672,7 @@ def require(self, ns_name: str, *aliases: sym.Symbol) -> BasilispModule: This method is called in code generated for the `require*` special form.""" try: - ns_module = importlib.import_module(munge(ns_name)) + ns_module = import_namespace(ns_name) except ModuleNotFoundError as e: raise ImportError( f"Basilisp namespace '{ns_name}' not found", diff --git a/src/basilisp/main.py b/src/basilisp/main.py index 7cfce8d4c..a34208bf1 100644 --- a/src/basilisp/main.py +++ b/src/basilisp/main.py @@ -1,4 +1,3 @@ -import importlib import logging import site from pathlib import Path @@ -17,32 +16,36 @@ def init(opts: Optional[CompilerOpts] = None) -> None: - """ - Initialize the runtime environment for Basilisp code evaluation. + """Initialize the runtime environment for Basilisp code evaluation. - Basilisp only needs to be initialized once per Python VM invocation. Subsequent - imports of Basilisp namespaces will work using Python's standard ``import`` - statement and :external:py:func:`importlib.import_module` function. + Basilisp only needs to be initialized once per Python VM + invocation. Subsequent imports of Basilisp namespaces will work using + Python's standard ``import`` statement and + :external:py:func:`importlib.import_module` function. (with appropriate + namespace module suffix) If you want to execute a Basilisp file which is stored in a well-formed package or module structure, you probably want to use :py:func:`bootstrap`. + """ runtime.init_ns_var() runtime.bootstrap_core(opts if opts is not None else compiler_opts()) importer.hook_imports() - importlib.import_module("basilisp.core") + runtime.import_namespace("basilisp.core") def bootstrap( target: str, opts: Optional[CompilerOpts] = None ) -> None: # pragma: no cover - """ - Import a Basilisp namespace or function identified by ``target``. If a function + """Import a Basilisp namespace or function identified by ``target``. If a function reference is given, the function will be called with no arguments. - Basilisp only needs to be initialized once per Python VM invocation. Subsequent - imports of Basilisp namespaces will work using Python's standard ``import`` - statement and :external:py:func:`importlib.import_module` function. + Basilisp only needs to be initialized once per Python VM + invocation. Subsequent imports of Basilisp namespaces will work using + Python's standard ``import`` statement and + :external:py:func:`importlib.import_module` function. (with appropriate + namespace module suffix) + ``target`` must be a string naming a Basilisp namespace. Namespace references may be given exactly as they are found in Basilisp code. ``target`` may optionally @@ -51,10 +54,11 @@ def bootstrap( ``opts`` is a mapping of compiler options that may be supplied for bootstrapping. This setting should be left alone unless you know what you are doing. + """ init(opts=opts) pkg_name, *rest = target.split(":", maxsplit=1) - mod = importlib.import_module(munge(pkg_name)) + mod = runtime.import_namespace(pkg_name) if rest: fn_name = munge(rest[0]) getattr(mod, fn_name)() diff --git a/tests/basilisp/compiler_test.py b/tests/basilisp/compiler_test.py index 45ee84049..dc97d85f1 100644 --- a/tests/basilisp/compiler_test.py +++ b/tests/basilisp/compiler_test.py @@ -1363,7 +1363,7 @@ def test_deftype_can_have_classmethod( (cls x y z)) (__eq__ [this other] (operator/eq - [x y z] + [x y z] [(.-x other) (.-y other) (.-z other)])))""" ) assert Point(1, 2, 3) == Point.create(1, 2, 3) @@ -5212,7 +5212,7 @@ def test_require_namespace_must_exist(self, lcompile: CompileFn): @pytest.fixture def _import_ns(self, ns: runtime.Namespace): def _import_ns_module(name: str): - ns_module = importlib.import_module(name) + ns_module = runtime.import_namespace(name) runtime.set_current_ns(ns.name) return ns_module diff --git a/tests/basilisp/core_test.py b/tests/basilisp/core_test.py index 3543d49e0..77795c7e1 100644 --- a/tests/basilisp/core_test.py +++ b/tests/basilisp/core_test.py @@ -29,7 +29,7 @@ def setup_module(): runtime.print_generated_python = orig -import basilisp.core as core # isort:skip +import basilisp.core.basilisp_namespace as core # isort:skip TRUTHY_VALUES = [ True, diff --git a/tests/basilisp/importer_test.py b/tests/basilisp/importer_test.py index 7ccc5eaf1..d98220085 100644 --- a/tests/basilisp/importer_test.py +++ b/tests/basilisp/importer_test.py @@ -56,6 +56,7 @@ def test_demunged_import(pytester: pytest.Pytester): ): importlib.import_module( "long__AMP__namespace_name__PLUS__with___LT__punctuation__GT__" + + runtime.NS_MODULE_SUFFIX ) assert ( @@ -84,7 +85,7 @@ def _ns_and_module(filename: str) -> Tuple[str, str]: # the `basilisp.core` namespace will not be loaded in the child process. _import_module = bootstrap_basilisp else: - _import_module = importlib.import_module + _import_module = runtime.import_namespace class TestImporter: @@ -162,7 +163,7 @@ def load_namespace(self): def _load_namespace(ns_name: str): """Load the named Namespace and return it.""" namespaces.append(ns_name) - importlib.import_module(munge(ns_name)) + runtime.import_namespace(ns_name) return runtime.Namespace.get(sym.symbol(ns_name)) try: @@ -215,7 +216,7 @@ def cached_module_file( # Import the module out of the current process to avoid having to # monkeypatch sys.modules - p = Process(target=_import_module, args=(munge(cached_module_ns),)) + p = Process(target=_import_module, args=(cached_module_ns,)) p.start() p.join() return os.path.join(*file_path) @@ -356,13 +357,15 @@ def test_import_basilisp_child_with_basilisp_init( ): """Load a Basilisp package namespace setup as a Python package and a child Basilisp namespace of that package.""" - make_new_module("core", "__init__.lpy", ns_name="core") + make_new_module( + "core", "__init__.lpy", module_text="""(def pkgval "core.__init__")""" + ) make_new_module("core", "sub.lpy", ns_name="core.sub") - core = load_namespace("core") + core = importlib.import_module("core") core_sub = load_namespace("core.sub") - assert "core" == core.find(sym.symbol("val")).value + assert "core.__init__" == core.pkgval assert "core.sub" == core_sub.find(sym.symbol("val")).value def test_import_module_without_init(self, make_new_module, load_namespace): @@ -377,6 +380,45 @@ def test_import_module_without_init(self, make_new_module, load_namespace): assert "core" == core.find(sym.symbol("val")).value assert "core.child" == core_child.find(sym.symbol("val")).value + def test_import_module_requires_child_namespace( + self, make_new_module, load_namespace + ): + """Load a Basilisp namespace that requires a child namespace""" + make_new_module( + "core.lpy", + module_text=""" + (ns core (:require [core.child])) + (def val core.child/val) + """, + ) + make_new_module("core", "child.lpy", ns_name="core.child") + + core = load_namespace("core") + core_child = load_namespace("core.child") + + assert "core.child" == core.find(sym.symbol("val")).value + assert "core.child" == core_child.find(sym.symbol("val")).value + + def test_import_module_requires_parent_namespace( + self, make_new_module, load_namespace + ): + """Load a Basilisp namespace that requires it's parent namespace""" + make_new_module("core.lpy", ns_name="core") + make_new_module( + "core", + "child.lpy", + module_text=""" + (ns core.child (:require [core])) + (def val core/val) + """, + ) + + core = load_namespace("core") + core_child = load_namespace("core.child") + + assert "core" == core.find(sym.symbol("val")).value + assert "core" == core_child.find(sym.symbol("val")).value + def test_import_module_with_namespace_only_pkg( self, make_new_module, load_namespace ): @@ -400,7 +442,9 @@ def test_no_filename_if_no_module(self): def test_can_get_filename_when_module_exists(self, make_new_module): make_new_module("package", "module.lpy", ns_name="package.module") - filename = importer.BasilispImporter().get_filename("package.module") + filename = importer.BasilispImporter().get_filename( + "package.module" + runtime.NS_MODULE_SUFFIX + ) assert filename is not None p = pathlib.Path(filename) @@ -437,7 +481,9 @@ def test_execute_module_correctly( monkeypatch.setattr("sys.argv", ["whatever", "1", "2", "3"]) - code = importer.BasilispImporter().get_code("package.module") + code = importer.BasilispImporter().get_code( + "package.module" + runtime.NS_MODULE_SUFFIX + ) exec(code) captured = capsys.readouterr() assert captured.out == 'package.module\n["1" "2" "3"]\n' @@ -498,7 +544,12 @@ def test_run_namespace_as_python_module( ) ) res = subprocess.run( - [sys.executable, "-m", "package.test_run_ns_as_pymodule", *args], + [ + sys.executable, + "-m", + "package.test_run_ns_as_pymodule" + runtime.NS_MODULE_SUFFIX, + *args, + ], check=True, capture_output=True, env={**os.environ, "PYTHONPATH": pythonpath},