From 7c4cbfb2cc952f5a210389e7dedefc0b0b076153 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 10:51:32 -0400 Subject: [PATCH 1/7] feat: Updated pythonmonkey_require to use pythonic module loading use the importlib machinery. --- .gitignore | 6 ++-- examples/use-python-module.py | 8 ++++-- examples/use-python-module/index.js | 1 - .../use-python-module/my-python-module.py | 2 +- examples/use-require.py | 8 ++++-- pythonmonkey_require.py | 28 ++++++++++--------- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index d75db893..244f0f60 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ firefox-102.2.0* firefox-102.2.0/* firefox-*/* firefox-*/ -tests/__pycache__/* -tests/python/__pycache__/* +__pycache__ Testing/Temporary -_spidermonkey_install/* \ No newline at end of file +_spidermonkey_install/* +node_modules diff --git a/examples/use-python-module.py b/examples/use-python-module.py index d1f0478f..9181c7d1 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -5,9 +5,11 @@ def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) if (path.exists(__file__)): - fileHnd = open(__file__, "r") - exec(fileHnd.read()) - return locals() + exports = {} + exports['__file__'] = __file__ + with open(__file__, "r") as fileHnd: + exec(fileHnd.read(), exports) + return exports else: raise Exception('file not found: ' + __file__) diff --git a/examples/use-python-module/index.js b/examples/use-python-module/index.js index 363c637a..288dc10b 100644 --- a/examples/use-python-module/index.js +++ b/examples/use-python-module/index.js @@ -1,4 +1,3 @@ const { helloWorld } = require('./my-python-module'); - helloWorld() diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py index 236b873d..adcd839e 100644 --- a/examples/use-python-module/my-python-module.py +++ b/examples/use-python-module/my-python-module.py @@ -1,5 +1,5 @@ def helloWorld(): print('hello, world!') -exports['helloWorld'] = helloWorld +#exports['helloWorld'] = helloWorld diff --git a/examples/use-require.py b/examples/use-require.py index b6129ff4..bfdf0cdf 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -5,9 +5,11 @@ def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) if (path.exists(__file__)): - fileHnd = open(__file__, "r") - exec(fileHnd.read()) - return locals() + exports = {} + exports['__file__'] = __file__ + with open(__file__, "r") as fileHnd: + exec(fileHnd.read(), exports) + return exports else: raise Exception('file not found: ' + __file__) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 06160be4..a1941322 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -24,11 +24,13 @@ # import sys, warnings +import importlib +from os import stat, path, getcwd, getenv + sys.path.append(path.dirname(__file__) + '/build/src') warnings.filterwarnings("ignore", category=DeprecationWarning) import pythonmonkey as pm -from os import stat, path, getcwd, getenv pm.eval(""" globalThis.python = {}; @@ -106,8 +108,8 @@ def statSync_inner(filename): return False def readFileSync(filename, charset): - fileHnd = open(filename, "r") - return fileHnd.read() + with open(filename, "r") as fileHnd: + return fileHnd.read() propSet('fsModule', 'statSync_inner', statSync_inner); propSet('fsModule', 'readFileSync', readFileSync) @@ -130,8 +132,9 @@ def readFileSync(filename, charset): # because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE # parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports # dict to decorate. -ctxModuleSource = open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") -moduleWrapper = pm.eval("""'use strict'; + +with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: + moduleWrapper = pm.eval("""'use strict'; (function moduleWrapper(require, exports) { exports=exports || globalThis; @@ -149,14 +152,13 @@ def readFileSync(filename, charset): pm.eval('const __builtinModules = {}; true'); def load(filename): - __file__ = filename - if (path.exists(__file__)): - exports = {} - fileHnd = open(__file__, "r") - exec(fileHnd.read()) - return exports - else: - raise Exception('file not found: ' + __file__) + name = path.basename(filename) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) + module = sourceFileLoader.load_module(name) + return module + propSet('python', 'load', load) # API - createRequire From ca2d68fd0c15f0ea79d99aa7031a28a8ca05c330 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 11:49:52 -0400 Subject: [PATCH 2/7] chore: Add some pythonic comments and type hinting for devs --- examples/use-python-module.py | 18 ++++++------ examples/use-require.py | 22 ++++++++------- pythonmonkey_require.py | 52 ++++++++++++++++++++++++++--------- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 9181c7d1..0767b15d 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -1,17 +1,17 @@ ### Loader because Python is awkward +import sys from os import path +import importlib from pathlib import Path def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) - if (path.exists(__file__)): - exports = {} - exports['__file__'] = __file__ - with open(__file__, "r") as fileHnd: - exec(fileHnd.read(), exports) - return exports - else: - raise Exception('file not found: ' + __file__) + name = path.basename(__file__) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + module = sourceFileLoader.load_module(name) + return module import sys sys.path.append(path.dirname(__file__) + '/..'); @@ -19,5 +19,5 @@ def include(relative, filename): pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); ### Actual test below -require = pmr['createRequire'](__file__) +require = pmr.createRequire(__file__) require('./use-python-module'); diff --git a/examples/use-require.py b/examples/use-require.py index bfdf0cdf..9a3caa36 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -1,24 +1,26 @@ ### Loader because Python is awkward +import sys from os import path +import importlib from pathlib import Path def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) - if (path.exists(__file__)): - exports = {} - exports['__file__'] = __file__ - with open(__file__, "r") as fileHnd: - exec(fileHnd.read(), exports) - return exports - else: - raise Exception('file not found: ' + __file__) + name = path.basename(__file__) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + module = sourceFileLoader.load_module(name) + return module import sys sys.path.append(path.dirname(__file__) + '/..'); -pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); +#pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); +import pythonmonkey_require as pmr ### Actual test below -require = pmr['createRequire'](__file__) +require = pmr.createRequire(__file__) require('./use-require/test1'); +print("Done") diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index a1941322..c38f94c6 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -24,6 +24,8 @@ # import sys, warnings +import types +from typing import Union, Dict, Callable import importlib from os import stat, path, getcwd, getenv @@ -99,7 +101,14 @@ pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays # Implement enough of require('fs') so that ctx-module can find/load files -def statSync_inner(filename): + +def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: + """ + Inner function for statSync. + + Returns: + Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. + """ from os import stat if (path.exists(filename)): sb = stat(filename) @@ -107,7 +116,12 @@ def statSync_inner(filename): else: return False -def readFileSync(filename, charset): +def readFileSync(filename, charset) -> str: + """ + Utility function for reading files. + Returns: + str: The contents of the file + """ with open(filename, "r") as fileHnd: return fileHnd.read() @@ -151,7 +165,18 @@ def readFileSync(filename, charset): # dict->jsObject in createRequire for every require we create. pm.eval('const __builtinModules = {}; true'); -def load(filename): +def load(filename: str) -> types.ModuleType: + """ + Loads a python module using the importlib machinery sourcefileloader and returns it. + If the module is already loaded, returns it. + + Args: + filename (str): The filename of the python module to load. + + Returns: + types.ModuleType: The loaded python module + """ + name = path.basename(filename) if name in sys.modules: return sys.modules[name] @@ -161,16 +186,17 @@ def load(filename): propSet('python', 'load', load) -# API - createRequire -# returns a require function that resolves modules relative to the filename argument. -# Conceptually the same as node:module.createRequire(). -# -# example: -# from pythonmonkey import createRequire -# require = createRequire(__file__) -# require('./my-javascript-module') -# -createRequire = pm.eval("""( +""" +API - createRequire +returns a require function that resolves modules relative to the filename argument. +Conceptually the same as node:module.createRequire(). + +example: + from pythonmonkey import createRequire + require = createRequire(__file__) + require('./my-javascript-module') +""" +createRequire: Callable = pm.eval("""( function createRequire(filename) { function loadPythonModule(module, filename) From 55bff26a4202aa091d5a651e951ea0064df57c04 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 15:04:23 -0400 Subject: [PATCH 3/7] fix: Fixed import for importlib machinery to keep it in scope and better managed python.load to give exports as dict --- examples/use-python-module.py | 3 ++- examples/use-python-module/my-python-module.py | 3 ++- examples/use-require.py | 7 +++---- pythonmonkey_require.py | 12 ++++++++---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 0767b15d..54599d9e 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -2,6 +2,7 @@ import sys from os import path import importlib +from importlib import machinery from pathlib import Path def include(relative, filename): @@ -9,7 +10,7 @@ def include(relative, filename): name = path.basename(__file__) if name in sys.modules: return sys.modules[name] - sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, __file__) module = sourceFileLoader.load_module(name) return module diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py index adcd839e..b16f8146 100644 --- a/examples/use-python-module/my-python-module.py +++ b/examples/use-python-module/my-python-module.py @@ -1,5 +1,6 @@ def helloWorld(): print('hello, world!') -#exports['helloWorld'] = helloWorld +exports = {} +exports['helloWorld'] = helloWorld diff --git a/examples/use-require.py b/examples/use-require.py index 9a3caa36..09ab6191 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -1,7 +1,7 @@ ### Loader because Python is awkward import sys from os import path -import importlib +from importlib import machinery from pathlib import Path def include(relative, filename): @@ -9,15 +9,14 @@ def include(relative, filename): name = path.basename(__file__) if name in sys.modules: return sys.modules[name] - sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, __file__) module = sourceFileLoader.load_module(name) return module import sys sys.path.append(path.dirname(__file__) + '/..'); -#pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); -import pythonmonkey_require as pmr +pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); ### Actual test below require = pmr.createRequire(__file__) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index c38f94c6..a2c42d0d 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -27,6 +27,7 @@ import types from typing import Union, Dict, Callable import importlib +from importlib import machinery from os import stat, path, getcwd, getenv sys.path.append(path.dirname(__file__) + '/build/src') @@ -165,7 +166,7 @@ def readFileSync(filename, charset) -> str: # dict->jsObject in createRequire for every require we create. pm.eval('const __builtinModules = {}; true'); -def load(filename: str) -> types.ModuleType: +def load(filename: str) -> Dict: """ Loads a python module using the importlib machinery sourcefileloader and returns it. If the module is already loaded, returns it. @@ -174,15 +175,18 @@ def load(filename: str) -> types.ModuleType: filename (str): The filename of the python module to load. Returns: - types.ModuleType: The loaded python module + : The loaded python module """ name = path.basename(filename) if name in sys.modules: return sys.modules[name] - sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) + sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) - return module + exports = {} + for key in dir(module): + exports[key] = getattr(module, key) + return exports propSet('python', 'load', load) From 9bc5d09e23a58df5b5f36bd7ece50b56b9282922 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 15:06:55 -0400 Subject: [PATCH 4/7] rename exports to module_exports to make sure semantics aren't confused --- pythonmonkey_require.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index a2c42d0d..0205ae8f 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -183,10 +183,10 @@ def load(filename: str) -> Dict: return sys.modules[name] sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) - exports = {} + module_exports = {} for key in dir(module): - exports[key] = getattr(module, key) - return exports + module_exports[key] = getattr(module, key) + return module_exports propSet('python', 'load', load) From cca1218b94cbabf74104a95db5d0ee07674ff550 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 15:09:36 -0400 Subject: [PATCH 5/7] fix: pythonmonkey_require - Make sure memoized modules are also exported properly --- pythonmonkey_require.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 0205ae8f..e5f78273 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -179,10 +179,11 @@ def load(filename: str) -> Dict: """ name = path.basename(filename) - if name in sys.modules: - return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) + if name not in sys.modules: + sourceFileLoader = machinery.SourceFileLoader(name, filename) + module = sourceFileLoader.load_module(name) + else: + module = sys.modules[name] module_exports = {} for key in dir(module): module_exports[key] = getattr(module, key) From 8f4348294727f02ccbe09403bf0cdbf16ef43211 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Tue, 30 May 2023 09:01:11 -0400 Subject: [PATCH 6/7] feat: Updated variable names in examples, more accurate python module loading, and better fs.readfilesync impl --- ' | 222 ++++++++++++++++++ examples/use-python-module.py | 6 +- .../use-python-module/my-python-module.py | 1 - examples/use-require.py | 6 +- pythonmonkey_require.py | 10 +- 5 files changed, 235 insertions(+), 10 deletions(-) create mode 100644 ' diff --git a/' b/' new file mode 100644 index 00000000..46be6c33 --- /dev/null +++ b/' @@ -0,0 +1,222 @@ +# @file require.py +# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the +# ctx-module npm package to do the heavy lifting. That package makes a complete module +# system, obstensibly in a separate context, but our implementation here reuses the +# PythonMonkey global context for both. +# +# The context that ctx-module runs in needs a require function supporting +# - require('debug') => returns a debug function which prints to the console when +# debugging; see the node debug built-in +# - require('fs') => returns an object which has an implementation of readFileSync +# and statSync. The implementation of statSync only needs to +# return the mode member. The fs module also needs +# constants.S_IFDIR available. +# - require('vm') => returns an object which has an implementation of evalInContext +# +# In order to implement this basic require function for bootstrapping ctxModule, we +# have simply made global variables of the form xxxModule where xxx is the module +# identifier, and injected a require function which understands this. A better +# implementation in Python that doesn't leak global symbols should be possible once +# some PythonMonkey bugs are fixed. +# +# @author Wes Garland, wes@distributive.network +# @date May 2023 +# + +import sys, warnings +import types +from typing import Union, Dict, Callable +import importlib +from importlib import machinery +from os import stat, path, getcwd, getenv + +sys.path.append(path.dirname(__file__) + '/build/src') +warnings.filterwarnings("ignore", category=DeprecationWarning) + +import pythonmonkey as pm + +pm.eval(""" +globalThis.python = {}; +globalThis.global = globalThis; +globalThis.vmModule = { runInContext: eval }; +globalThis.require = function outerRequire(mid) { + const module = globalThis[mid + 'Module']; + if (module) + return module; + throw new Error('module not found: ' + mid); +}; +globalThis.debugModule = function debug(selector) { + var idx, colour; + const esc = String.fromCharCode(27); + const noColour = `${esc}[0m`; + + debug.selectors = debug.selectors || []; + idx = debug.selectors.indexOf(selector); + if (idx === -1) + { + idx = debug.selectors.length; + debug.selectors.push(selector); + } + + colour = `${esc}[${91 + ((idx + 1) % 6)}m`; + const debugEnv = python.getenv('DEBUG'); + + if (debugEnv) + { + for (let sym of debugEnv.split(' ')) + { + const re = new RegExp('^' + sym.replace('*', '.*') + '$'); + if (re.test(selector)) + { + return (function debugInner() { + python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) + }); + } + } + } + + /* no match => silent */ + return (function debugDummy() {}); +}; + +function globalSet(name, prop) +{ + globalThis[name] = prop; +} + +function propSet(objName, propName, propValue) +{ + globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; +} + +""") + +# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. +globalSet = pm.eval("globalSet"); +propSet = pm.eval("propSet") + +# Add some python functions to the global python object for code in this file to use. +propSet('python', 'print', print); +propSet('python', 'getenv', getenv); +propSet('python', 'paths', ':'.join(sys.path)); +pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays + +# Implement enough of require('fs') so that ctx-module can find/load files + +def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: + """ + Inner function for statSync. + + Returns: + Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. + """ + from os import stat + if (path.exists(filename)): + sb = stat(filename) + return { 'mode': sb.st_mode } + else: + return False + +def readFileSync(filename, charset) -> str: + """ + Utility function for reading files. + Returns: + str: The contents of the file + """ + with open(filename, "r") as fileHnd: + return fileHnd.read() + +propSet('fsModule', 'statSync_inner', statSync_inner); +propSet('fsModule', 'readFileSync', readFileSync) +propSet('fsModule', 'existsSync', path.exists) +pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") +pm.eval("""fsModule.statSync = +function statSync(filename) +{ + const ret = require('fs').statSync_inner(filename); + if (ret) + return ret; + + const err = new Error('file not found: ' + filename); + err.code='ENOENT'; + throw err; +}"""); + +# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is +# now the corresponding CommonJS require() function. We use the globalThis as the module's exports +# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE +# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports +# dict to decorate. + +with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: + moduleWrapper = pm.eval("""'use strict'; +(function moduleWrapper(require, exports) +{ + exports=exports || globalThis; + require=require || globalThis.require; +""" + ctxModuleSource.read() + """ +}) +"""); + +# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have +# to use globalThis and pollute the global scope. +moduleWrapper() + +# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same +# dict->jsObject in createRequire for every require we create. +pm.eval('const __builtinModules = {}; true'); + +def load(filename: str) -> Dict: + """ + Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. + If the module is already loaded, returns it. + + Args: + filename (str): The filename of the python module to load. + + Returns: + : The loaded python module + """ + + name = path.basename(filename) + if name not in sys.modules: + sourceFileLoader = machinery.SourceFileLoader(name, filename) + spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + module.exports = {} + spec.loader.exec_module(module) + else: + module = sys.modules[name] + module_exports = {} + for key in dir(module): + module_exports[key] = getattr(module, key) + return module_exports + +propSet('python', 'load', load) + +""" +API - createRequire +returns a require function that resolves modules relative to the filename argument. +Conceptually the same as node:module.createRequire(). + +example: + from pythonmonkey import createRequire + require = createRequire(__file__) + require('./my-javascript-module') +""" +createRequire: Callable = pm.eval("""( +function createRequire(filename) +{ + function loadPythonModule(module, filename) + { + module.exports = python.load(filename); + } + + const module = new CtxModule(globalThis, filename, __builtinModules); + for (let path of python.paths) + module.paths.push(path + '/node_modules'); + module.require.extensions['.py'] = loadPythonModule; + return module.require; +} +)""") diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 54599d9e..b64d62f0 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -6,11 +6,11 @@ from pathlib import Path def include(relative, filename): - __file__ = str(Path(relative, filename).resolve()) - name = path.basename(__file__) + filename = str(Path(relative, filename).resolve()) + name = path.basename(filename) if name in sys.modules: return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) return module diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py index b16f8146..236b873d 100644 --- a/examples/use-python-module/my-python-module.py +++ b/examples/use-python-module/my-python-module.py @@ -1,6 +1,5 @@ def helloWorld(): print('hello, world!') -exports = {} exports['helloWorld'] = helloWorld diff --git a/examples/use-require.py b/examples/use-require.py index 09ab6191..4d53919c 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -5,11 +5,11 @@ from pathlib import Path def include(relative, filename): - __file__ = str(Path(relative, filename).resolve()) - name = path.basename(__file__) + filename = str(Path(relative, filename).resolve()) + name = path.basename(filename) if name in sys.modules: return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) return module diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index e5f78273..3c81a22f 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -123,7 +123,7 @@ def readFileSync(filename, charset) -> str: Returns: str: The contents of the file """ - with open(filename, "r") as fileHnd: + with open(filename, "r", encoding=charset) as fileHnd: return fileHnd.read() propSet('fsModule', 'statSync_inner', statSync_inner); @@ -168,7 +168,7 @@ def readFileSync(filename, charset) -> str: def load(filename: str) -> Dict: """ - Loads a python module using the importlib machinery sourcefileloader and returns it. + Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. If the module is already loaded, returns it. Args: @@ -181,7 +181,11 @@ def load(filename: str) -> Dict: name = path.basename(filename) if name not in sys.modules: sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) + spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + module.exports = {} + spec.loader.exec_module(module) else: module = sys.modules[name] module_exports = {} From 77d5473d885d91443faf2dbbf92cf09acc74e987 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Fri, 16 Jun 2023 17:10:36 -0400 Subject: [PATCH 7/7] fix: Delete mistakenly copied additional file --- ' | 222 -------------------------------------------------------------- 1 file changed, 222 deletions(-) delete mode 100644 ' diff --git a/' b/' deleted file mode 100644 index 46be6c33..00000000 --- a/' +++ /dev/null @@ -1,222 +0,0 @@ -# @file require.py -# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the -# ctx-module npm package to do the heavy lifting. That package makes a complete module -# system, obstensibly in a separate context, but our implementation here reuses the -# PythonMonkey global context for both. -# -# The context that ctx-module runs in needs a require function supporting -# - require('debug') => returns a debug function which prints to the console when -# debugging; see the node debug built-in -# - require('fs') => returns an object which has an implementation of readFileSync -# and statSync. The implementation of statSync only needs to -# return the mode member. The fs module also needs -# constants.S_IFDIR available. -# - require('vm') => returns an object which has an implementation of evalInContext -# -# In order to implement this basic require function for bootstrapping ctxModule, we -# have simply made global variables of the form xxxModule where xxx is the module -# identifier, and injected a require function which understands this. A better -# implementation in Python that doesn't leak global symbols should be possible once -# some PythonMonkey bugs are fixed. -# -# @author Wes Garland, wes@distributive.network -# @date May 2023 -# - -import sys, warnings -import types -from typing import Union, Dict, Callable -import importlib -from importlib import machinery -from os import stat, path, getcwd, getenv - -sys.path.append(path.dirname(__file__) + '/build/src') -warnings.filterwarnings("ignore", category=DeprecationWarning) - -import pythonmonkey as pm - -pm.eval(""" -globalThis.python = {}; -globalThis.global = globalThis; -globalThis.vmModule = { runInContext: eval }; -globalThis.require = function outerRequire(mid) { - const module = globalThis[mid + 'Module']; - if (module) - return module; - throw new Error('module not found: ' + mid); -}; -globalThis.debugModule = function debug(selector) { - var idx, colour; - const esc = String.fromCharCode(27); - const noColour = `${esc}[0m`; - - debug.selectors = debug.selectors || []; - idx = debug.selectors.indexOf(selector); - if (idx === -1) - { - idx = debug.selectors.length; - debug.selectors.push(selector); - } - - colour = `${esc}[${91 + ((idx + 1) % 6)}m`; - const debugEnv = python.getenv('DEBUG'); - - if (debugEnv) - { - for (let sym of debugEnv.split(' ')) - { - const re = new RegExp('^' + sym.replace('*', '.*') + '$'); - if (re.test(selector)) - { - return (function debugInner() { - python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) - }); - } - } - } - - /* no match => silent */ - return (function debugDummy() {}); -}; - -function globalSet(name, prop) -{ - globalThis[name] = prop; -} - -function propSet(objName, propName, propValue) -{ - globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; -} - -""") - -# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. -globalSet = pm.eval("globalSet"); -propSet = pm.eval("propSet") - -# Add some python functions to the global python object for code in this file to use. -propSet('python', 'print', print); -propSet('python', 'getenv', getenv); -propSet('python', 'paths', ':'.join(sys.path)); -pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays - -# Implement enough of require('fs') so that ctx-module can find/load files - -def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: - """ - Inner function for statSync. - - Returns: - Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. - """ - from os import stat - if (path.exists(filename)): - sb = stat(filename) - return { 'mode': sb.st_mode } - else: - return False - -def readFileSync(filename, charset) -> str: - """ - Utility function for reading files. - Returns: - str: The contents of the file - """ - with open(filename, "r") as fileHnd: - return fileHnd.read() - -propSet('fsModule', 'statSync_inner', statSync_inner); -propSet('fsModule', 'readFileSync', readFileSync) -propSet('fsModule', 'existsSync', path.exists) -pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") -pm.eval("""fsModule.statSync = -function statSync(filename) -{ - const ret = require('fs').statSync_inner(filename); - if (ret) - return ret; - - const err = new Error('file not found: ' + filename); - err.code='ENOENT'; - throw err; -}"""); - -# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is -# now the corresponding CommonJS require() function. We use the globalThis as the module's exports -# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE -# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports -# dict to decorate. - -with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: - moduleWrapper = pm.eval("""'use strict'; -(function moduleWrapper(require, exports) -{ - exports=exports || globalThis; - require=require || globalThis.require; -""" + ctxModuleSource.read() + """ -}) -"""); - -# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have -# to use globalThis and pollute the global scope. -moduleWrapper() - -# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same -# dict->jsObject in createRequire for every require we create. -pm.eval('const __builtinModules = {}; true'); - -def load(filename: str) -> Dict: - """ - Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. - If the module is already loaded, returns it. - - Args: - filename (str): The filename of the python module to load. - - Returns: - : The loaded python module - """ - - name = path.basename(filename) - if name not in sys.modules: - sourceFileLoader = machinery.SourceFileLoader(name, filename) - spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - module.exports = {} - spec.loader.exec_module(module) - else: - module = sys.modules[name] - module_exports = {} - for key in dir(module): - module_exports[key] = getattr(module, key) - return module_exports - -propSet('python', 'load', load) - -""" -API - createRequire -returns a require function that resolves modules relative to the filename argument. -Conceptually the same as node:module.createRequire(). - -example: - from pythonmonkey import createRequire - require = createRequire(__file__) - require('./my-javascript-module') -""" -createRequire: Callable = pm.eval("""( -function createRequire(filename) -{ - function loadPythonModule(module, filename) - { - module.exports = python.load(filename); - } - - const module = new CtxModule(globalThis, filename, __builtinModules); - for (let path of python.paths) - module.paths.push(path + '/node_modules'); - module.require.extensions['.py'] = loadPythonModule; - return module.require; -} -)""")