From f6c35f15db8b2c2efbc1becf0e71bd9695cb0a14 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 26 May 2023 15:04:26 -0400 Subject: [PATCH 01/60] Add package.json for npm/JS dependencies --- build_script.sh | 5 +- package-lock.json | 434 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 26 +++ 3 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 package-lock.json create mode 100644 package.json diff --git a/build_script.sh b/build_script.sh index e985294e..22f2bfe4 100755 --- a/build_script.sh +++ b/build_script.sh @@ -9,4 +9,7 @@ fi cd build cmake .. -cmake --build . -j$CPUS \ No newline at end of file +cmake --build . -j$CPUS + +# npm is used to load JS components, see package.json +npm i diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..c0903f08 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,434 @@ +{ + "name": "pythonmonkey", + "version": "0.0.1", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "requires": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bn.js": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", + "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" + }, + "brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" + }, + "browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "requires": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "requires": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "requires": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "browserify-rsa": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "requires": { + "bn.js": "^5.0.0", + "randombytes": "^2.0.1" + } + }, + "browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "requires": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" + }, + "cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "requires": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "requires": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "requires": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "requires": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + } + }, + "ctx-module": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/ctx-module/-/ctx-module-1.0.8.tgz", + "integrity": "sha512-7PJPjd7j4HqQAplynqCk/PGom7j+RzNYmdGKfS58cGXb0o5bz0gQmt/TBH/Uu5IwtpssFfDGibEsq9+W0NCZKw==", + "requires": { + "buffer": "^6.0.3", + "crypto-browserify": "^3.12.0" + } + }, + "des.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", + "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "requires": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, + "diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "requires": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "elliptic": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "requires": { + "bn.js": "^4.11.9", + "brorand": "^1.1.0", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "requires": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, + "hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "requires": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", + "requires": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "requires": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" + }, + "parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "requires": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, + "pbkdf2": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", + "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "requires": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "requires": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "requires": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..d908709a --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "pythonmonkey", + "version": "0.0.1", + "description": "PythonMonkey - JS Components", + "main": "index.js", + "directories": { + "doc": "docs", + "test": "tests" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/Distributive-Network/PythonMonkey.git" + }, + "author": "Distributive Corp.", + "license": "MIT", + "bugs": { + "url": "https://github.com/Distributive-Network/PythonMonkey/issues" + }, + "homepage": "https://github.com/Distributive-Network/PythonMonkey#readme", + "dependencies": { + "ctx-module": "^1.0.8" + } +} From 9b313fcd5046fb49555586a696c8d4e44f323213 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 26 May 2023 22:56:40 -0400 Subject: [PATCH 02/60] Add node-compatible require() for CommonJS modules --- examples/use-require.py | 20 +++++++ examples/use-require/test1.js | 5 ++ examples/use-require/test2.js | 9 +++ pythonmonkey_require.py | 103 ++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 examples/use-require.py create mode 100644 examples/use-require/test1.js create mode 100644 examples/use-require/test2.js create mode 100644 pythonmonkey_require.py diff --git a/examples/use-require.py b/examples/use-require.py new file mode 100644 index 00000000..e73219f3 --- /dev/null +++ b/examples/use-require.py @@ -0,0 +1,20 @@ +### Loader because Python is awkward +from os import path +from pathlib import Path + +def include(relative, filename): + if (path.exists(filename)): + fileHnd = open(filename, "r") + __file__ = str(Path(relative, filename).resolve()) + exec(fileHnd.read()) + return locals() + +import sys +sys.path.append(path.dirname(__file__) + '/..'); + +pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); + +### Actual test below +require = pmr['createRequire'](__file__) +require('./use-require/test1'); + diff --git a/examples/use-require/test1.js b/examples/use-require/test1.js new file mode 100644 index 00000000..c58753e8 --- /dev/null +++ b/examples/use-require/test1.js @@ -0,0 +1,5 @@ +'use strict' + +const makeOutput = require('./test2').makeOutput; + +makeOutput('hello world'); diff --git a/examples/use-require/test2.js b/examples/use-require/test2.js new file mode 100644 index 00000000..9688c578 --- /dev/null +++ b/examples/use-require/test2.js @@ -0,0 +1,9 @@ +'use strict' + +exports.makeOutput = function makeOutput() +{ + const argv = Array.from(arguments); + argv.unshift('TEST OUTPUT: '); + debugPrint.apply(null, argv); +} + diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py new file mode 100644 index 00000000..4e92efc2 --- /dev/null +++ b/pythonmonkey_require.py @@ -0,0 +1,103 @@ +# @file require.py +# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the +# ctx-module npm package to do the heavy lifting. That package makes a complete module +# system, obstensibly in a separate context, but our implementation here reuses the +# PythonMonkey global context for both. +# +# The context that ctx-module runs in needs a require function supporting +# - require('debug') => returns a debug function which prints to the console when +# debugging; see the node debug built-in +# - require('fs') => returns an object which has an implementation of readFileSync +# and statSync. The implementation of statSync only needs to +# return the mode member. The fs module also needs +# constants.S_IFDIR available. +# - require('vm') => returns an object which has an implementation of evalInContext +# +# In order to implement this basic require function for bootstrapping ctxModule, we +# have simply made global variables of the form xxxModule where xxx is the module +# identifier, and injected a require function which understands this. A better +# implementation in Python that doesn't leak global symbols should be possible once +# some PythonMonkey bugs are fixed. +# +# @author Wes Garland, wes@distributive.network +# @date May 2023 +# + +import sys +sys.path.append(path.dirname(__file__) + '/build/src'); + +import pythonmonkey as pm +from os import stat, path, getcwd + +pm.eval("globalThis.vmModule = { runInContext: eval }; true") +pm.eval("globalThis.require = function outerRequire(mid) { const module = globalThis[mid + 'Module']; if (module) return module; throw new Error('module not found: ' + mid) }"); +pm.eval("globalThis.debugModule = function debug() { return function debugInner() { debugPrint('DEBUG: ' + Array.from(arguments).join(' ')) }}; true"); +globalSet = pm.eval('(function globalSet(name, prop) { globalThis[name] = prop; })') +propSet = pm.eval('(function propSet(objName, propName, propValue) { globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; })') + +# Make global.debugPrint equivalent to Python's print() function +globalSet('debugPrint', print); + +# Implement enough of require('fs') so that ctx-module can find/load files +def statSync_inner(filename): + if (path.exists(filename)): + sb = stat(filename) + return { 'mode': 0 } + else: + return False + +def readFileSync(filename, charset): + fileHnd = open(filename, "r") + return fileHnd.read() + +propSet('fsModule', 'statSync_inner', statSync_inner); +propSet('fsModule', 'readFileSync', readFileSync) +propSet('fsModule', 'existsSync', path.exists) +pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") +pm.eval("""fsModule.statSync = +function statSync(filename) +{ + const ret = require('fs').statSync_inner(filename); + if (ret) + return ret; + + const err = new Error('file not found: ' + filename); + err.code='ENOENT'; + throw err; +}"""); + +# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is +# now the corresponding CommonJS require() function. We use the globalThis as the module's exports +# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE +# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports +# dict to decorate. +ctxModuleSource = open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") +moduleWrapper = pm.eval("""'use strict'; +(function moduleWrapper(require, exports) +{ + exports=exports || globalThis; + require=require || globalThis.require; +""" + ctxModuleSource.read() + """ +}) +"""); + +# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have +# to use globalThis and pollute the global scope. +moduleWrapper() + +# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same +# dict->jsObject in createRequire for every require we create. +pm.eval('const __builtinModules = {}; true'); + +# API - createRequire +# returns a require function which returns a require function that resolves modules relative to the +# filename argument. Conceptually the same as node:module.createRequire(). +# +# example: +# from pythonmonkey import createRequire +# require = createRequire(__file__) +# require('./my-module') +# +def createRequire(filename): + import pythonmonkey as pm + return (pm.eval("((new CtxModule(globalThis, '" + filename + "', __builtinModules)).require)")) From 156feb8b621a00109da9b914a6521a04c99feb0e Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 27 May 2023 10:31:30 -0400 Subject: [PATCH 03/60] use-require example: update include() so that cwd doesn't matter --- examples/use-require.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/use-require.py b/examples/use-require.py index e73219f3..b6129ff4 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -3,11 +3,13 @@ from pathlib import Path def include(relative, filename): - if (path.exists(filename)): - fileHnd = open(filename, "r") - __file__ = str(Path(relative, filename).resolve()) + __file__ = str(Path(relative, filename).resolve()) + if (path.exists(__file__)): + fileHnd = open(__file__, "r") exec(fileHnd.read()) return locals() + else: + raise Exception('file not found: ' + __file__) import sys sys.path.append(path.dirname(__file__) + '/..'); From 8f480edf1d0727249ebad0dab3729fbaa53160ff Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 27 May 2023 12:04:55 -0400 Subject: [PATCH 04/60] pythonmonkey_require - improve code clarity; now compatible with latest npm ctx-module out of the box --- package-lock.json | 6 +++--- package.json | 2 +- pythonmonkey_require.py | 43 ++++++++++++++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index c0903f08..46cf7ab3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -179,9 +179,9 @@ } }, "ctx-module": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/ctx-module/-/ctx-module-1.0.8.tgz", - "integrity": "sha512-7PJPjd7j4HqQAplynqCk/PGom7j+RzNYmdGKfS58cGXb0o5bz0gQmt/TBH/Uu5IwtpssFfDGibEsq9+W0NCZKw==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/ctx-module/-/ctx-module-1.0.9.tgz", + "integrity": "sha512-MN1mKVJI8xn4NbfqxCOqoTuoGJBs5SX7uLfqooWQxQFqMkoCFoaXGHSbZxOKccAO8SgWNn6gSxR8TsUxLsHp1g==", "requires": { "buffer": "^6.0.3", "crypto-browserify": "^3.12.0" diff --git a/package.json b/package.json index d908709a..b5e51b46 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,6 @@ }, "homepage": "https://github.com/Distributive-Network/PythonMonkey#readme", "dependencies": { - "ctx-module": "^1.0.8" + "ctx-module": "^1.0.9" } } diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 4e92efc2..b556163d 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -28,12 +28,37 @@ import pythonmonkey as pm from os import stat, path, getcwd - -pm.eval("globalThis.vmModule = { runInContext: eval }; true") -pm.eval("globalThis.require = function outerRequire(mid) { const module = globalThis[mid + 'Module']; if (module) return module; throw new Error('module not found: ' + mid) }"); -pm.eval("globalThis.debugModule = function debug() { return function debugInner() { debugPrint('DEBUG: ' + Array.from(arguments).join(' ')) }}; true"); -globalSet = pm.eval('(function globalSet(name, prop) { globalThis[name] = prop; })') -propSet = pm.eval('(function propSet(objName, propName, propValue) { globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; })') + +pm.eval(""" +globalThis.global = globalThis; +globalThis.vmModule = { runInContext: eval }; +globalThis.require = function outerRequire(mid) { + const module = globalThis[mid + 'Module']; + if (module) + return module; + throw new Error('module not found: ' + mid); +}; +globalThis.debugModule = function debug() { + return function debugInner() { + debugPrint('DEBUG: ' + Array.from(arguments).join(' ')) + } +}; + +function globalSet(name, prop) +{ + globalThis[name] = prop; +} + +function propSet(objName, propName, propValue) +{ + globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; +} + +""") + +# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. +globalSet = pm.eval("globalSet"); +propSet = pm.eval("propSet") # Make global.debugPrint equivalent to Python's print() function globalSet('debugPrint', print); @@ -90,13 +115,13 @@ def readFileSync(filename, charset): pm.eval('const __builtinModules = {}; true'); # API - createRequire -# returns a require function which returns a require function that resolves modules relative to the -# filename argument. Conceptually the same as node:module.createRequire(). +# returns a require function that resolves modules relative to the filename argument. +# Conceptually the same as node:module.createRequire(). # # example: # from pythonmonkey import createRequire # require = createRequire(__file__) -# require('./my-module') +# require('./my-javascript-module') # def createRequire(filename): import pythonmonkey as pm From 72f4f467d76597bba83047b4c1212f6335468d37 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 27 May 2023 13:10:04 -0400 Subject: [PATCH 05/60] pythonmonkey_require - add python JS global, improve debug function, deprecate debugPrint in favour of python.print --- examples/use-require/test2.js | 2 +- pythonmonkey_require.py | 47 +++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/examples/use-require/test2.js b/examples/use-require/test2.js index 9688c578..92076222 100644 --- a/examples/use-require/test2.js +++ b/examples/use-require/test2.js @@ -4,6 +4,6 @@ exports.makeOutput = function makeOutput() { const argv = Array.from(arguments); argv.unshift('TEST OUTPUT: '); - debugPrint.apply(null, argv); + python.print.apply(null, argv); } diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index b556163d..3e5a0c03 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -23,13 +23,15 @@ # @date May 2023 # -import sys -sys.path.append(path.dirname(__file__) + '/build/src'); +import sys, warnings +sys.path.append(path.dirname(__file__) + '/build/src') +warnings.filterwarnings("ignore", category=DeprecationWarning) import pythonmonkey as pm -from os import stat, path, getcwd +from os import stat, path, getcwd, getenv pm.eval(""" +globalThis.python = {}; globalThis.global = globalThis; globalThis.vmModule = { runInContext: eval }; globalThis.require = function outerRequire(mid) { @@ -38,10 +40,38 @@ return module; throw new Error('module not found: ' + mid); }; -globalThis.debugModule = function debug() { - return function debugInner() { - debugPrint('DEBUG: ' + Array.from(arguments).join(' ')) +globalThis.debugModule = function debug(selector) { + var idx, colour; + const esc = String.fromCharCode(27); + const noColour = `${esc}[0m`; + + debug.selectors = debug.selectors || []; + idx = debug.selectors.indexOf(selector); + if (idx === -1) + { + idx = debug.selectors.length; + debug.selectors.push(selector); } + + colour = `${esc}[${91 + ((idx + 1) % 6)}m`; + const debugEnv = python.getenv('DEBUG'); + + if (debugEnv) + { + for (let sym of debugEnv.split(' ')) + { + const re = new RegExp('^' + sym.replace('*', '.*') + '$'); + if (re.test(selector)) + { + return (function debugInner() { + python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) + }); + } + } + } + + /* no match => silent */ + return (function debugDummy() {}); }; function globalSet(name, prop) @@ -60,8 +90,9 @@ globalSet = pm.eval("globalSet"); propSet = pm.eval("propSet") -# Make global.debugPrint equivalent to Python's print() function -globalSet('debugPrint', print); +# Add some python functions to the global python object for code in this file to use. +propSet('python', 'print', print); +propSet('python', 'getenv', getenv); # Implement enough of require('fs') so that ctx-module can find/load files def statSync_inner(filename): From c89036b29aa2447895a0c3d3e514365e01dea743 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 27 May 2023 13:58:03 -0400 Subject: [PATCH 06/60] pythonmonkey_require - add sys.paths+/node_modules to module.paths --- pythonmonkey_require.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 3e5a0c03..52eef533 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -93,6 +93,8 @@ # Add some python functions to the global python object for code in this file to use. propSet('python', 'print', print); propSet('python', 'getenv', getenv); +propSet('python', 'paths', ':'.join(sys.path)); +pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays # Implement enough of require('fs') so that ctx-module can find/load files def statSync_inner(filename): @@ -154,6 +156,12 @@ def readFileSync(filename, charset): # require = createRequire(__file__) # require('./my-javascript-module') # -def createRequire(filename): - import pythonmonkey as pm - return (pm.eval("((new CtxModule(globalThis, '" + filename + "', __builtinModules)).require)")) +createRequire = pm.eval("""( +function createRequire(filename) +{ + const module = new CtxModule(globalThis, filename, __builtinModules); + for (let path of python.paths) + module.paths.push(path + '/node_modules'); + return module.require; +} +)""") From 264a9d4c9e9c50d1440b8a2b4263d386e0e3e2a6 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 27 May 2023 14:32:27 -0400 Subject: [PATCH 07/60] pythonmonkey_require - add support for .py modules --- examples/use-python-module.py | 22 +++++++++++++++++++ examples/use-python-module/index.js | 4 ++++ .../use-python-module/my-python-module.py | 5 +++++ pythonmonkey_require.py | 17 ++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 examples/use-python-module.py create mode 100644 examples/use-python-module/index.js create mode 100644 examples/use-python-module/my-python-module.py diff --git a/examples/use-python-module.py b/examples/use-python-module.py new file mode 100644 index 00000000..b28985c5 --- /dev/null +++ b/examples/use-python-module.py @@ -0,0 +1,22 @@ +### Loader because Python is awkward +from os import path +from pathlib import Path + +def include(relative, filename): + __file__ = str(Path(relative, filename).resolve()) + if (path.exists(__file__)): + fileHnd = open(__file__, "r") + exec(fileHnd.read()) + return locals() + else: + raise Exception('file not found: ' + __file__) + +import sys +sys.path.append(path.dirname(__file__) + '/..'); + +pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); + +### Actual test below +require = pmr['createRequire'](__file__) +require('./use-python-module/index'); + diff --git a/examples/use-python-module/index.js b/examples/use-python-module/index.js new file mode 100644 index 00000000..363c637a --- /dev/null +++ b/examples/use-python-module/index.js @@ -0,0 +1,4 @@ +const { helloWorld } = require('./my-python-module'); + +helloWorld() + diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py new file mode 100644 index 00000000..236b873d --- /dev/null +++ b/examples/use-python-module/my-python-module.py @@ -0,0 +1,5 @@ +def helloWorld(): + print('hello, world!') + +exports['helloWorld'] = helloWorld + diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 52eef533..3278be0e 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -147,6 +147,17 @@ def readFileSync(filename, charset): # dict->jsObject in createRequire for every require we create. pm.eval('const __builtinModules = {}; true'); +def load(filename): + __file__ = filename + if (path.exists(__file__)): + exports = {} + fileHnd = open(__file__, "r") + exec(fileHnd.read()) + return exports + else: + raise Exception('file not found: ' + __file__) +propSet('python', 'load', load) + # API - createRequire # returns a require function that resolves modules relative to the filename argument. # Conceptually the same as node:module.createRequire(). @@ -159,9 +170,15 @@ def readFileSync(filename, charset): createRequire = pm.eval("""( function createRequire(filename) { + function loadPythonModule(module, filename) + { + module.exports = python.load(filename); + } + const module = new CtxModule(globalThis, filename, __builtinModules); for (let path of python.paths) module.paths.push(path + '/node_modules'); + module.require.extensions['.py'] = loadPythonModule; return module.require; } )""") From 704bacad722f303dbc9ae8f61bda1f2bfce764b5 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 27 May 2023 15:12:31 -0400 Subject: [PATCH 08/60] pythonmonkey_require - fix support for directory/index modules --- examples/use-python-module.py | 3 +-- pythonmonkey_require.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index b28985c5..d1f0478f 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -18,5 +18,4 @@ def include(relative, filename): ### Actual test below require = pmr['createRequire'](__file__) -require('./use-python-module/index'); - +require('./use-python-module'); diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 3278be0e..06160be4 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -98,9 +98,10 @@ # Implement enough of require('fs') so that ctx-module can find/load files def statSync_inner(filename): + from os import stat if (path.exists(filename)): sb = stat(filename) - return { 'mode': 0 } + return { 'mode': sb.st_mode } else: return False From aed267299a95b215f755d72b9aa9edfeb8e95245 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Mon, 29 May 2023 07:45:19 -0400 Subject: [PATCH 09/60] Update README to reflect module system progress --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1f25badc..7e0df614 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ PythonMonkey is a Mozilla [SpiderMonkey](https://firefox-source-docs.mozilla.org/js/index.html) JavaScript engine embedded into the Python VM, using the Python engine to provide the JS host environment. -This product is in an early stage, approximately 65% to MVP as of March 2023. It is under active development by Distributive Corp., +This product is in an early stage, approximately 75% to MVP as of May 2023. It is under active development by Distributive Corp., https://distributive.network/. External contributions and feedback are welcome and encouraged. The goal is to make writing code in either JS or Python a developer preference, with libraries commonly used in either language @@ -30,13 +30,13 @@ this package to execute our complex `dcp-client` library, which is written in JS - [done] JS functions coerce to Python function wrappers - [done] JS exceptions propagate to Python - [done] Implement `eval()` function in Python which accepts JS code and returns JS->Python coerced values -- [underway] NodeJS+NPM-compatible CommonJS module system +- [done] NodeJS+NPM-compatible CommonJS module system - [done] Python strings coerce to JS strings - [done] Python intrinsics coerce to JS intrinsics - Python dicts coerce to JS objects -- Python `require` function, returns a coerced dict of module exports +- [done] Python `require` function, returns a coerced dict of module exports - [done] Python functions coerce to JS function wrappers -- CommonJS module system .py loader, loads Python modules for use by JS +- [done] CommonJS module system .py loader, loads Python modules for use by JS - JS object->Python dict coercion supports inherited-property lookup (via __getattribute__?) - Python host environment supplies event loop, including EventEmitter, setTimeout, etc. - Python host environment supplies XMLHttpRequest (other project?) @@ -56,6 +56,7 @@ this package to execute our complex `dcp-client` library, which is written in JS - rust - python3.9 or later - spidermonkey 102.2.0 or later + - npm (nodejs) 2. Compile pythonmonkey in ``/build`` (which can be done automatically by running ``./build_script.sh``) @@ -71,6 +72,8 @@ this package to execute our complex `dcp-client` library, which is written in JS ## Using the library +See also: examples/ + ### Method 1 After compiling the project in the `build/src` folder you will find a `.so` file named `pythonmonkey.so`. This is the shared object file that contains the pythonmonkey module. @@ -86,4 +89,4 @@ Type "help", "copyright", "credits" or "license" for more information. >>> hello = pm.eval("() => {return 'Hello from Spidermonkey!'}") >>> hello() 'Hello from Spidermonkey!' -``` \ No newline at end of file +``` From 7c4cbfb2cc952f5a210389e7dedefc0b0b076153 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 10:51:32 -0400 Subject: [PATCH 10/60] feat: Updated pythonmonkey_require to use pythonic module loading use the importlib machinery. --- .gitignore | 6 ++-- examples/use-python-module.py | 8 ++++-- examples/use-python-module/index.js | 1 - .../use-python-module/my-python-module.py | 2 +- examples/use-require.py | 8 ++++-- pythonmonkey_require.py | 28 ++++++++++--------- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index d75db893..244f0f60 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ firefox-102.2.0* firefox-102.2.0/* firefox-*/* firefox-*/ -tests/__pycache__/* -tests/python/__pycache__/* +__pycache__ Testing/Temporary -_spidermonkey_install/* \ No newline at end of file +_spidermonkey_install/* +node_modules diff --git a/examples/use-python-module.py b/examples/use-python-module.py index d1f0478f..9181c7d1 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -5,9 +5,11 @@ def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) if (path.exists(__file__)): - fileHnd = open(__file__, "r") - exec(fileHnd.read()) - return locals() + exports = {} + exports['__file__'] = __file__ + with open(__file__, "r") as fileHnd: + exec(fileHnd.read(), exports) + return exports else: raise Exception('file not found: ' + __file__) diff --git a/examples/use-python-module/index.js b/examples/use-python-module/index.js index 363c637a..288dc10b 100644 --- a/examples/use-python-module/index.js +++ b/examples/use-python-module/index.js @@ -1,4 +1,3 @@ const { helloWorld } = require('./my-python-module'); - helloWorld() diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py index 236b873d..adcd839e 100644 --- a/examples/use-python-module/my-python-module.py +++ b/examples/use-python-module/my-python-module.py @@ -1,5 +1,5 @@ def helloWorld(): print('hello, world!') -exports['helloWorld'] = helloWorld +#exports['helloWorld'] = helloWorld diff --git a/examples/use-require.py b/examples/use-require.py index b6129ff4..bfdf0cdf 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -5,9 +5,11 @@ def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) if (path.exists(__file__)): - fileHnd = open(__file__, "r") - exec(fileHnd.read()) - return locals() + exports = {} + exports['__file__'] = __file__ + with open(__file__, "r") as fileHnd: + exec(fileHnd.read(), exports) + return exports else: raise Exception('file not found: ' + __file__) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 06160be4..a1941322 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -24,11 +24,13 @@ # import sys, warnings +import importlib +from os import stat, path, getcwd, getenv + sys.path.append(path.dirname(__file__) + '/build/src') warnings.filterwarnings("ignore", category=DeprecationWarning) import pythonmonkey as pm -from os import stat, path, getcwd, getenv pm.eval(""" globalThis.python = {}; @@ -106,8 +108,8 @@ def statSync_inner(filename): return False def readFileSync(filename, charset): - fileHnd = open(filename, "r") - return fileHnd.read() + with open(filename, "r") as fileHnd: + return fileHnd.read() propSet('fsModule', 'statSync_inner', statSync_inner); propSet('fsModule', 'readFileSync', readFileSync) @@ -130,8 +132,9 @@ def readFileSync(filename, charset): # because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE # parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports # dict to decorate. -ctxModuleSource = open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") -moduleWrapper = pm.eval("""'use strict'; + +with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: + moduleWrapper = pm.eval("""'use strict'; (function moduleWrapper(require, exports) { exports=exports || globalThis; @@ -149,14 +152,13 @@ def readFileSync(filename, charset): pm.eval('const __builtinModules = {}; true'); def load(filename): - __file__ = filename - if (path.exists(__file__)): - exports = {} - fileHnd = open(__file__, "r") - exec(fileHnd.read()) - return exports - else: - raise Exception('file not found: ' + __file__) + name = path.basename(filename) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) + module = sourceFileLoader.load_module(name) + return module + propSet('python', 'load', load) # API - createRequire From ca2d68fd0c15f0ea79d99aa7031a28a8ca05c330 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 11:49:52 -0400 Subject: [PATCH 11/60] chore: Add some pythonic comments and type hinting for devs --- examples/use-python-module.py | 18 ++++++------ examples/use-require.py | 22 ++++++++------- pythonmonkey_require.py | 52 ++++++++++++++++++++++++++--------- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 9181c7d1..0767b15d 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -1,17 +1,17 @@ ### Loader because Python is awkward +import sys from os import path +import importlib from pathlib import Path def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) - if (path.exists(__file__)): - exports = {} - exports['__file__'] = __file__ - with open(__file__, "r") as fileHnd: - exec(fileHnd.read(), exports) - return exports - else: - raise Exception('file not found: ' + __file__) + name = path.basename(__file__) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + module = sourceFileLoader.load_module(name) + return module import sys sys.path.append(path.dirname(__file__) + '/..'); @@ -19,5 +19,5 @@ def include(relative, filename): pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); ### Actual test below -require = pmr['createRequire'](__file__) +require = pmr.createRequire(__file__) require('./use-python-module'); diff --git a/examples/use-require.py b/examples/use-require.py index bfdf0cdf..9a3caa36 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -1,24 +1,26 @@ ### Loader because Python is awkward +import sys from os import path +import importlib from pathlib import Path def include(relative, filename): __file__ = str(Path(relative, filename).resolve()) - if (path.exists(__file__)): - exports = {} - exports['__file__'] = __file__ - with open(__file__, "r") as fileHnd: - exec(fileHnd.read(), exports) - return exports - else: - raise Exception('file not found: ' + __file__) + name = path.basename(__file__) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + module = sourceFileLoader.load_module(name) + return module import sys sys.path.append(path.dirname(__file__) + '/..'); -pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); +#pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); +import pythonmonkey_require as pmr ### Actual test below -require = pmr['createRequire'](__file__) +require = pmr.createRequire(__file__) require('./use-require/test1'); +print("Done") diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index a1941322..c38f94c6 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -24,6 +24,8 @@ # import sys, warnings +import types +from typing import Union, Dict, Callable import importlib from os import stat, path, getcwd, getenv @@ -99,7 +101,14 @@ pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays # Implement enough of require('fs') so that ctx-module can find/load files -def statSync_inner(filename): + +def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: + """ + Inner function for statSync. + + Returns: + Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. + """ from os import stat if (path.exists(filename)): sb = stat(filename) @@ -107,7 +116,12 @@ def statSync_inner(filename): else: return False -def readFileSync(filename, charset): +def readFileSync(filename, charset) -> str: + """ + Utility function for reading files. + Returns: + str: The contents of the file + """ with open(filename, "r") as fileHnd: return fileHnd.read() @@ -151,7 +165,18 @@ def readFileSync(filename, charset): # dict->jsObject in createRequire for every require we create. pm.eval('const __builtinModules = {}; true'); -def load(filename): +def load(filename: str) -> types.ModuleType: + """ + Loads a python module using the importlib machinery sourcefileloader and returns it. + If the module is already loaded, returns it. + + Args: + filename (str): The filename of the python module to load. + + Returns: + types.ModuleType: The loaded python module + """ + name = path.basename(filename) if name in sys.modules: return sys.modules[name] @@ -161,16 +186,17 @@ def load(filename): propSet('python', 'load', load) -# API - createRequire -# returns a require function that resolves modules relative to the filename argument. -# Conceptually the same as node:module.createRequire(). -# -# example: -# from pythonmonkey import createRequire -# require = createRequire(__file__) -# require('./my-javascript-module') -# -createRequire = pm.eval("""( +""" +API - createRequire +returns a require function that resolves modules relative to the filename argument. +Conceptually the same as node:module.createRequire(). + +example: + from pythonmonkey import createRequire + require = createRequire(__file__) + require('./my-javascript-module') +""" +createRequire: Callable = pm.eval("""( function createRequire(filename) { function loadPythonModule(module, filename) From 55bff26a4202aa091d5a651e951ea0064df57c04 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 15:04:23 -0400 Subject: [PATCH 12/60] fix: Fixed import for importlib machinery to keep it in scope and better managed python.load to give exports as dict --- examples/use-python-module.py | 3 ++- examples/use-python-module/my-python-module.py | 3 ++- examples/use-require.py | 7 +++---- pythonmonkey_require.py | 12 ++++++++---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 0767b15d..54599d9e 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -2,6 +2,7 @@ import sys from os import path import importlib +from importlib import machinery from pathlib import Path def include(relative, filename): @@ -9,7 +10,7 @@ def include(relative, filename): name = path.basename(__file__) if name in sys.modules: return sys.modules[name] - sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, __file__) module = sourceFileLoader.load_module(name) return module diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py index adcd839e..b16f8146 100644 --- a/examples/use-python-module/my-python-module.py +++ b/examples/use-python-module/my-python-module.py @@ -1,5 +1,6 @@ def helloWorld(): print('hello, world!') -#exports['helloWorld'] = helloWorld +exports = {} +exports['helloWorld'] = helloWorld diff --git a/examples/use-require.py b/examples/use-require.py index 9a3caa36..09ab6191 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -1,7 +1,7 @@ ### Loader because Python is awkward import sys from os import path -import importlib +from importlib import machinery from pathlib import Path def include(relative, filename): @@ -9,15 +9,14 @@ def include(relative, filename): name = path.basename(__file__) if name in sys.modules: return sys.modules[name] - sourceFileLoader = importlib.machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, __file__) module = sourceFileLoader.load_module(name) return module import sys sys.path.append(path.dirname(__file__) + '/..'); -#pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); -import pythonmonkey_require as pmr +pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); ### Actual test below require = pmr.createRequire(__file__) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index c38f94c6..a2c42d0d 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -27,6 +27,7 @@ import types from typing import Union, Dict, Callable import importlib +from importlib import machinery from os import stat, path, getcwd, getenv sys.path.append(path.dirname(__file__) + '/build/src') @@ -165,7 +166,7 @@ def readFileSync(filename, charset) -> str: # dict->jsObject in createRequire for every require we create. pm.eval('const __builtinModules = {}; true'); -def load(filename: str) -> types.ModuleType: +def load(filename: str) -> Dict: """ Loads a python module using the importlib machinery sourcefileloader and returns it. If the module is already loaded, returns it. @@ -174,15 +175,18 @@ def load(filename: str) -> types.ModuleType: filename (str): The filename of the python module to load. Returns: - types.ModuleType: The loaded python module + : The loaded python module """ name = path.basename(filename) if name in sys.modules: return sys.modules[name] - sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) + sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) - return module + exports = {} + for key in dir(module): + exports[key] = getattr(module, key) + return exports propSet('python', 'load', load) From 9bc5d09e23a58df5b5f36bd7ece50b56b9282922 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 15:06:55 -0400 Subject: [PATCH 13/60] rename exports to module_exports to make sure semantics aren't confused --- pythonmonkey_require.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index a2c42d0d..0205ae8f 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -183,10 +183,10 @@ def load(filename: str) -> Dict: return sys.modules[name] sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) - exports = {} + module_exports = {} for key in dir(module): - exports[key] = getattr(module, key) - return exports + module_exports[key] = getattr(module, key) + return module_exports propSet('python', 'load', load) From cca1218b94cbabf74104a95db5d0ee07674ff550 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 29 May 2023 15:09:36 -0400 Subject: [PATCH 14/60] fix: pythonmonkey_require - Make sure memoized modules are also exported properly --- pythonmonkey_require.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 0205ae8f..e5f78273 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -179,10 +179,11 @@ def load(filename: str) -> Dict: """ name = path.basename(filename) - if name in sys.modules: - return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) + if name not in sys.modules: + sourceFileLoader = machinery.SourceFileLoader(name, filename) + module = sourceFileLoader.load_module(name) + else: + module = sys.modules[name] module_exports = {} for key in dir(module): module_exports[key] = getattr(module, key) From 8f4348294727f02ccbe09403bf0cdbf16ef43211 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Tue, 30 May 2023 09:01:11 -0400 Subject: [PATCH 15/60] feat: Updated variable names in examples, more accurate python module loading, and better fs.readfilesync impl --- ' | 222 ++++++++++++++++++ examples/use-python-module.py | 6 +- .../use-python-module/my-python-module.py | 1 - examples/use-require.py | 6 +- pythonmonkey_require.py | 10 +- 5 files changed, 235 insertions(+), 10 deletions(-) create mode 100644 ' diff --git a/' b/' new file mode 100644 index 00000000..46be6c33 --- /dev/null +++ b/' @@ -0,0 +1,222 @@ +# @file require.py +# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the +# ctx-module npm package to do the heavy lifting. That package makes a complete module +# system, obstensibly in a separate context, but our implementation here reuses the +# PythonMonkey global context for both. +# +# The context that ctx-module runs in needs a require function supporting +# - require('debug') => returns a debug function which prints to the console when +# debugging; see the node debug built-in +# - require('fs') => returns an object which has an implementation of readFileSync +# and statSync. The implementation of statSync only needs to +# return the mode member. The fs module also needs +# constants.S_IFDIR available. +# - require('vm') => returns an object which has an implementation of evalInContext +# +# In order to implement this basic require function for bootstrapping ctxModule, we +# have simply made global variables of the form xxxModule where xxx is the module +# identifier, and injected a require function which understands this. A better +# implementation in Python that doesn't leak global symbols should be possible once +# some PythonMonkey bugs are fixed. +# +# @author Wes Garland, wes@distributive.network +# @date May 2023 +# + +import sys, warnings +import types +from typing import Union, Dict, Callable +import importlib +from importlib import machinery +from os import stat, path, getcwd, getenv + +sys.path.append(path.dirname(__file__) + '/build/src') +warnings.filterwarnings("ignore", category=DeprecationWarning) + +import pythonmonkey as pm + +pm.eval(""" +globalThis.python = {}; +globalThis.global = globalThis; +globalThis.vmModule = { runInContext: eval }; +globalThis.require = function outerRequire(mid) { + const module = globalThis[mid + 'Module']; + if (module) + return module; + throw new Error('module not found: ' + mid); +}; +globalThis.debugModule = function debug(selector) { + var idx, colour; + const esc = String.fromCharCode(27); + const noColour = `${esc}[0m`; + + debug.selectors = debug.selectors || []; + idx = debug.selectors.indexOf(selector); + if (idx === -1) + { + idx = debug.selectors.length; + debug.selectors.push(selector); + } + + colour = `${esc}[${91 + ((idx + 1) % 6)}m`; + const debugEnv = python.getenv('DEBUG'); + + if (debugEnv) + { + for (let sym of debugEnv.split(' ')) + { + const re = new RegExp('^' + sym.replace('*', '.*') + '$'); + if (re.test(selector)) + { + return (function debugInner() { + python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) + }); + } + } + } + + /* no match => silent */ + return (function debugDummy() {}); +}; + +function globalSet(name, prop) +{ + globalThis[name] = prop; +} + +function propSet(objName, propName, propValue) +{ + globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; +} + +""") + +# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. +globalSet = pm.eval("globalSet"); +propSet = pm.eval("propSet") + +# Add some python functions to the global python object for code in this file to use. +propSet('python', 'print', print); +propSet('python', 'getenv', getenv); +propSet('python', 'paths', ':'.join(sys.path)); +pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays + +# Implement enough of require('fs') so that ctx-module can find/load files + +def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: + """ + Inner function for statSync. + + Returns: + Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. + """ + from os import stat + if (path.exists(filename)): + sb = stat(filename) + return { 'mode': sb.st_mode } + else: + return False + +def readFileSync(filename, charset) -> str: + """ + Utility function for reading files. + Returns: + str: The contents of the file + """ + with open(filename, "r") as fileHnd: + return fileHnd.read() + +propSet('fsModule', 'statSync_inner', statSync_inner); +propSet('fsModule', 'readFileSync', readFileSync) +propSet('fsModule', 'existsSync', path.exists) +pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") +pm.eval("""fsModule.statSync = +function statSync(filename) +{ + const ret = require('fs').statSync_inner(filename); + if (ret) + return ret; + + const err = new Error('file not found: ' + filename); + err.code='ENOENT'; + throw err; +}"""); + +# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is +# now the corresponding CommonJS require() function. We use the globalThis as the module's exports +# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE +# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports +# dict to decorate. + +with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: + moduleWrapper = pm.eval("""'use strict'; +(function moduleWrapper(require, exports) +{ + exports=exports || globalThis; + require=require || globalThis.require; +""" + ctxModuleSource.read() + """ +}) +"""); + +# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have +# to use globalThis and pollute the global scope. +moduleWrapper() + +# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same +# dict->jsObject in createRequire for every require we create. +pm.eval('const __builtinModules = {}; true'); + +def load(filename: str) -> Dict: + """ + Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. + If the module is already loaded, returns it. + + Args: + filename (str): The filename of the python module to load. + + Returns: + : The loaded python module + """ + + name = path.basename(filename) + if name not in sys.modules: + sourceFileLoader = machinery.SourceFileLoader(name, filename) + spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + module.exports = {} + spec.loader.exec_module(module) + else: + module = sys.modules[name] + module_exports = {} + for key in dir(module): + module_exports[key] = getattr(module, key) + return module_exports + +propSet('python', 'load', load) + +""" +API - createRequire +returns a require function that resolves modules relative to the filename argument. +Conceptually the same as node:module.createRequire(). + +example: + from pythonmonkey import createRequire + require = createRequire(__file__) + require('./my-javascript-module') +""" +createRequire: Callable = pm.eval("""( +function createRequire(filename) +{ + function loadPythonModule(module, filename) + { + module.exports = python.load(filename); + } + + const module = new CtxModule(globalThis, filename, __builtinModules); + for (let path of python.paths) + module.paths.push(path + '/node_modules'); + module.require.extensions['.py'] = loadPythonModule; + return module.require; +} +)""") diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 54599d9e..b64d62f0 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -6,11 +6,11 @@ from pathlib import Path def include(relative, filename): - __file__ = str(Path(relative, filename).resolve()) - name = path.basename(__file__) + filename = str(Path(relative, filename).resolve()) + name = path.basename(filename) if name in sys.modules: return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) return module diff --git a/examples/use-python-module/my-python-module.py b/examples/use-python-module/my-python-module.py index b16f8146..236b873d 100644 --- a/examples/use-python-module/my-python-module.py +++ b/examples/use-python-module/my-python-module.py @@ -1,6 +1,5 @@ def helloWorld(): print('hello, world!') -exports = {} exports['helloWorld'] = helloWorld diff --git a/examples/use-require.py b/examples/use-require.py index 09ab6191..4d53919c 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -5,11 +5,11 @@ from pathlib import Path def include(relative, filename): - __file__ = str(Path(relative, filename).resolve()) - name = path.basename(__file__) + filename = str(Path(relative, filename).resolve()) + name = path.basename(filename) if name in sys.modules: return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, __file__) + sourceFileLoader = machinery.SourceFileLoader(name, filename) module = sourceFileLoader.load_module(name) return module diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index e5f78273..3c81a22f 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -123,7 +123,7 @@ def readFileSync(filename, charset) -> str: Returns: str: The contents of the file """ - with open(filename, "r") as fileHnd: + with open(filename, "r", encoding=charset) as fileHnd: return fileHnd.read() propSet('fsModule', 'statSync_inner', statSync_inner); @@ -168,7 +168,7 @@ def readFileSync(filename, charset) -> str: def load(filename: str) -> Dict: """ - Loads a python module using the importlib machinery sourcefileloader and returns it. + Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. If the module is already loaded, returns it. Args: @@ -181,7 +181,11 @@ def load(filename: str) -> Dict: name = path.basename(filename) if name not in sys.modules: sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) + spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + module.exports = {} + spec.loader.exec_module(module) else: module = sys.modules[name] module_exports = {} From 77d5473d885d91443faf2dbbf92cf09acc74e987 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Fri, 16 Jun 2023 17:10:36 -0400 Subject: [PATCH 16/60] fix: Delete mistakenly copied additional file --- ' | 222 -------------------------------------------------------------- 1 file changed, 222 deletions(-) delete mode 100644 ' diff --git a/' b/' deleted file mode 100644 index 46be6c33..00000000 --- a/' +++ /dev/null @@ -1,222 +0,0 @@ -# @file require.py -# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the -# ctx-module npm package to do the heavy lifting. That package makes a complete module -# system, obstensibly in a separate context, but our implementation here reuses the -# PythonMonkey global context for both. -# -# The context that ctx-module runs in needs a require function supporting -# - require('debug') => returns a debug function which prints to the console when -# debugging; see the node debug built-in -# - require('fs') => returns an object which has an implementation of readFileSync -# and statSync. The implementation of statSync only needs to -# return the mode member. The fs module also needs -# constants.S_IFDIR available. -# - require('vm') => returns an object which has an implementation of evalInContext -# -# In order to implement this basic require function for bootstrapping ctxModule, we -# have simply made global variables of the form xxxModule where xxx is the module -# identifier, and injected a require function which understands this. A better -# implementation in Python that doesn't leak global symbols should be possible once -# some PythonMonkey bugs are fixed. -# -# @author Wes Garland, wes@distributive.network -# @date May 2023 -# - -import sys, warnings -import types -from typing import Union, Dict, Callable -import importlib -from importlib import machinery -from os import stat, path, getcwd, getenv - -sys.path.append(path.dirname(__file__) + '/build/src') -warnings.filterwarnings("ignore", category=DeprecationWarning) - -import pythonmonkey as pm - -pm.eval(""" -globalThis.python = {}; -globalThis.global = globalThis; -globalThis.vmModule = { runInContext: eval }; -globalThis.require = function outerRequire(mid) { - const module = globalThis[mid + 'Module']; - if (module) - return module; - throw new Error('module not found: ' + mid); -}; -globalThis.debugModule = function debug(selector) { - var idx, colour; - const esc = String.fromCharCode(27); - const noColour = `${esc}[0m`; - - debug.selectors = debug.selectors || []; - idx = debug.selectors.indexOf(selector); - if (idx === -1) - { - idx = debug.selectors.length; - debug.selectors.push(selector); - } - - colour = `${esc}[${91 + ((idx + 1) % 6)}m`; - const debugEnv = python.getenv('DEBUG'); - - if (debugEnv) - { - for (let sym of debugEnv.split(' ')) - { - const re = new RegExp('^' + sym.replace('*', '.*') + '$'); - if (re.test(selector)) - { - return (function debugInner() { - python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) - }); - } - } - } - - /* no match => silent */ - return (function debugDummy() {}); -}; - -function globalSet(name, prop) -{ - globalThis[name] = prop; -} - -function propSet(objName, propName, propValue) -{ - globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; -} - -""") - -# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. -globalSet = pm.eval("globalSet"); -propSet = pm.eval("propSet") - -# Add some python functions to the global python object for code in this file to use. -propSet('python', 'print', print); -propSet('python', 'getenv', getenv); -propSet('python', 'paths', ':'.join(sys.path)); -pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays - -# Implement enough of require('fs') so that ctx-module can find/load files - -def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: - """ - Inner function for statSync. - - Returns: - Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. - """ - from os import stat - if (path.exists(filename)): - sb = stat(filename) - return { 'mode': sb.st_mode } - else: - return False - -def readFileSync(filename, charset) -> str: - """ - Utility function for reading files. - Returns: - str: The contents of the file - """ - with open(filename, "r") as fileHnd: - return fileHnd.read() - -propSet('fsModule', 'statSync_inner', statSync_inner); -propSet('fsModule', 'readFileSync', readFileSync) -propSet('fsModule', 'existsSync', path.exists) -pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") -pm.eval("""fsModule.statSync = -function statSync(filename) -{ - const ret = require('fs').statSync_inner(filename); - if (ret) - return ret; - - const err = new Error('file not found: ' + filename); - err.code='ENOENT'; - throw err; -}"""); - -# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is -# now the corresponding CommonJS require() function. We use the globalThis as the module's exports -# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE -# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports -# dict to decorate. - -with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: - moduleWrapper = pm.eval("""'use strict'; -(function moduleWrapper(require, exports) -{ - exports=exports || globalThis; - require=require || globalThis.require; -""" + ctxModuleSource.read() + """ -}) -"""); - -# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have -# to use globalThis and pollute the global scope. -moduleWrapper() - -# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same -# dict->jsObject in createRequire for every require we create. -pm.eval('const __builtinModules = {}; true'); - -def load(filename: str) -> Dict: - """ - Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. - If the module is already loaded, returns it. - - Args: - filename (str): The filename of the python module to load. - - Returns: - : The loaded python module - """ - - name = path.basename(filename) - if name not in sys.modules: - sourceFileLoader = machinery.SourceFileLoader(name, filename) - spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - module.exports = {} - spec.loader.exec_module(module) - else: - module = sys.modules[name] - module_exports = {} - for key in dir(module): - module_exports[key] = getattr(module, key) - return module_exports - -propSet('python', 'load', load) - -""" -API - createRequire -returns a require function that resolves modules relative to the filename argument. -Conceptually the same as node:module.createRequire(). - -example: - from pythonmonkey import createRequire - require = createRequire(__file__) - require('./my-javascript-module') -""" -createRequire: Callable = pm.eval("""( -function createRequire(filename) -{ - function loadPythonModule(module, filename) - { - module.exports = python.load(filename); - } - - const module = new CtxModule(globalThis, filename, __builtinModules); - for (let path of python.paths) - module.paths.push(path + '/node_modules'); - module.require.extensions['.py'] = loadPythonModule; - return module.require; -} -)""") From 15445206d40520d8d91208f45252634fdfee4fcd Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Thu, 22 Jun 2023 15:16:43 -0400 Subject: [PATCH 17/60] WIP: Updated pythonmonkey_require to be in pythonmonkey directory and moved associated requirements --- CMakeLists.txt | 2 +- python/pythonmonkey/__init__.py | 1 + .../pythonmonkey/package-lock.json | 308 +++++++++++------- .../pythonmonkey/package.json | 0 .../pythonmonkey/require.py | 3 +- 5 files changed, 192 insertions(+), 122 deletions(-) rename package-lock.json => python/pythonmonkey/package-lock.json (71%) rename package.json => python/pythonmonkey/package.json (100%) rename pythonmonkey_require.py => python/pythonmonkey/require.py (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f0eec96..efe79b87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,7 +44,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) set(CMAKE_CXX_EXTENSIONS OFF) if(UNIX) - SET(COMPILE_FLAGS "-g -ggdb -O0") + SET(COMPILE_FLAGS "-g -ggdb -O0 -coverage -fprofile-arcs -ftest-coverage -fno-rtti") elseif(WIN32) SET(COMPILE_FLAGS "") endif() diff --git a/python/pythonmonkey/__init__.py b/python/pythonmonkey/__init__.py index f643b61f..ef416b37 100644 --- a/python/pythonmonkey/__init__.py +++ b/python/pythonmonkey/__init__.py @@ -1 +1,2 @@ from .pythonmonkey import * +from .require import createRequire diff --git a/package-lock.json b/python/pythonmonkey/package-lock.json similarity index 71% rename from package-lock.json rename to python/pythonmonkey/package-lock.json index 46cf7ab3..bf4ae335 100644 --- a/package-lock.json +++ b/python/pythonmonkey/package-lock.json @@ -1,47 +1,67 @@ { "name": "pythonmonkey", "version": "0.0.1", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "asn1.js": { + "packages": { + "": { + "name": "pythonmonkey", + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "ctx-module": "^1.0.9" + } + }, + "node_modules/asn1.js": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", - "requires": { + "dependencies": { "bn.js": "^4.0.0", "inherits": "^2.0.1", "minimalistic-assert": "^1.0.0", "safer-buffer": "^2.1.0" - }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } } }, - "base64-js": { + "node_modules/asn1.js/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, - "bn.js": { + "node_modules/bn.js": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" }, - "brorand": { + "node_modules/brorand": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" }, - "browserify-aes": { + "node_modules/browserify-aes": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", - "requires": { + "dependencies": { "buffer-xor": "^1.0.3", "cipher-base": "^1.0.0", "create-hash": "^1.1.0", @@ -50,41 +70,41 @@ "safe-buffer": "^5.0.1" } }, - "browserify-cipher": { + "node_modules/browserify-cipher": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", - "requires": { + "dependencies": { "browserify-aes": "^1.0.4", "browserify-des": "^1.0.0", "evp_bytestokey": "^1.0.0" } }, - "browserify-des": { + "node_modules/browserify-des": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", - "requires": { + "dependencies": { "cipher-base": "^1.0.1", "des.js": "^1.0.0", "inherits": "^2.0.1", "safe-buffer": "^5.1.2" } }, - "browserify-rsa": { + "node_modules/browserify-rsa": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", - "requires": { + "dependencies": { "bn.js": "^5.0.0", "randombytes": "^2.0.1" } }, - "browserify-sign": { + "node_modules/browserify-sign": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", - "requires": { + "dependencies": { "bn.js": "^5.1.1", "browserify-rsa": "^4.0.1", "create-hash": "^1.2.0", @@ -96,50 +116,62 @@ "safe-buffer": "^5.2.0" } }, - "buffer": { + "node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "requires": { + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, - "buffer-xor": { + "node_modules/buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" }, - "cipher-base": { + "node_modules/cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", - "requires": { + "dependencies": { "inherits": "^2.0.1", "safe-buffer": "^5.0.1" } }, - "create-ecdh": { + "node_modules/create-ecdh": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", - "requires": { + "dependencies": { "bn.js": "^4.1.0", "elliptic": "^6.5.3" - }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } } }, - "create-hash": { + "node_modules/create-ecdh/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/create-hash": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", - "requires": { + "dependencies": { "cipher-base": "^1.0.1", "inherits": "^2.0.1", "md5.js": "^1.3.4", @@ -147,11 +179,11 @@ "sha.js": "^2.4.0" } }, - "create-hmac": { + "node_modules/create-hmac": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", - "requires": { + "dependencies": { "cipher-base": "^1.0.3", "create-hash": "^1.1.0", "inherits": "^2.0.1", @@ -160,11 +192,11 @@ "sha.js": "^2.4.8" } }, - "crypto-browserify": { + "node_modules/crypto-browserify": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", - "requires": { + "dependencies": { "browserify-cipher": "^1.0.0", "browserify-sign": "^4.0.0", "create-ecdh": "^4.0.0", @@ -176,48 +208,49 @@ "public-encrypt": "^4.0.0", "randombytes": "^2.0.0", "randomfill": "^1.0.3" + }, + "engines": { + "node": "*" } }, - "ctx-module": { + "node_modules/ctx-module": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/ctx-module/-/ctx-module-1.0.9.tgz", "integrity": "sha512-MN1mKVJI8xn4NbfqxCOqoTuoGJBs5SX7uLfqooWQxQFqMkoCFoaXGHSbZxOKccAO8SgWNn6gSxR8TsUxLsHp1g==", - "requires": { + "dependencies": { "buffer": "^6.0.3", "crypto-browserify": "^3.12.0" } }, - "des.js": { + "node_modules/des.js": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", - "requires": { + "dependencies": { "inherits": "^2.0.1", "minimalistic-assert": "^1.0.0" } }, - "diffie-hellman": { + "node_modules/diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", - "requires": { + "dependencies": { "bn.js": "^4.1.0", "miller-rabin": "^4.0.0", "randombytes": "^2.0.0" - }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } } }, - "elliptic": { + "node_modules/diffie-hellman/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/elliptic": { "version": "6.5.4", "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", - "requires": { + "dependencies": { "bn.js": "^4.11.9", "brorand": "^1.1.0", "hash.js": "^1.0.0", @@ -225,104 +258,120 @@ "inherits": "^2.0.4", "minimalistic-assert": "^1.0.1", "minimalistic-crypto-utils": "^1.0.1" - }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } } }, - "evp_bytestokey": { + "node_modules/elliptic/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/evp_bytestokey": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", - "requires": { + "dependencies": { "md5.js": "^1.3.4", "safe-buffer": "^5.1.1" } }, - "hash-base": { + "node_modules/hash-base": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", - "requires": { + "dependencies": { "inherits": "^2.0.4", "readable-stream": "^3.6.0", "safe-buffer": "^5.2.0" + }, + "engines": { + "node": ">=4" } }, - "hash.js": { + "node_modules/hash.js": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", - "requires": { + "dependencies": { "inherits": "^2.0.3", "minimalistic-assert": "^1.0.1" } }, - "hmac-drbg": { + "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", - "requires": { + "dependencies": { "hash.js": "^1.0.3", "minimalistic-assert": "^1.0.0", "minimalistic-crypto-utils": "^1.0.1" } }, - "ieee754": { + "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, - "inherits": { + "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "md5.js": { + "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", - "requires": { + "dependencies": { "hash-base": "^3.0.0", "inherits": "^2.0.1", "safe-buffer": "^5.1.2" } }, - "miller-rabin": { + "node_modules/miller-rabin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", - "requires": { + "dependencies": { "bn.js": "^4.0.0", "brorand": "^1.0.1" }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } + "bin": { + "miller-rabin": "bin/miller-rabin" } }, - "minimalistic-assert": { + "node_modules/miller-rabin/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" }, - "minimalistic-crypto-utils": { + "node_modules/minimalistic-crypto-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" }, - "parse-asn1": { + "node_modules/parse-asn1": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", - "requires": { + "dependencies": { "asn1.js": "^5.2.0", "browserify-aes": "^1.0.0", "evp_bytestokey": "^1.0.0", @@ -330,102 +379,123 @@ "safe-buffer": "^5.1.1" } }, - "pbkdf2": { + "node_modules/pbkdf2": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", - "requires": { + "dependencies": { "create-hash": "^1.1.2", "create-hmac": "^1.1.4", "ripemd160": "^2.0.1", "safe-buffer": "^5.0.1", "sha.js": "^2.4.8" + }, + "engines": { + "node": ">=0.12" } }, - "public-encrypt": { + "node_modules/public-encrypt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", - "requires": { + "dependencies": { "bn.js": "^4.1.0", "browserify-rsa": "^4.0.0", "create-hash": "^1.1.0", "parse-asn1": "^5.0.0", "randombytes": "^2.0.1", "safe-buffer": "^5.1.2" - }, - "dependencies": { - "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" - } } }, - "randombytes": { + "node_modules/public-encrypt/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "requires": { + "dependencies": { "safe-buffer": "^5.1.0" } }, - "randomfill": { + "node_modules/randomfill": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", - "requires": { + "dependencies": { "randombytes": "^2.0.5", "safe-buffer": "^5.1.0" } }, - "readable-stream": { + "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { + "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" } }, - "ripemd160": { + "node_modules/ripemd160": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", - "requires": { + "dependencies": { "hash-base": "^3.0.0", "inherits": "^2.0.1" } }, - "safe-buffer": { + "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, - "safer-buffer": { + "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "sha.js": { + "node_modules/sha.js": { "version": "2.4.11", "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", - "requires": { + "dependencies": { "inherits": "^2.0.1", "safe-buffer": "^5.0.1" + }, + "bin": { + "sha.js": "bin.js" } }, - "string_decoder": { + "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { + "dependencies": { "safe-buffer": "~5.2.0" } }, - "util-deprecate": { + "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" diff --git a/package.json b/python/pythonmonkey/package.json similarity index 100% rename from package.json rename to python/pythonmonkey/package.json diff --git a/pythonmonkey_require.py b/python/pythonmonkey/require.py similarity index 98% rename from pythonmonkey_require.py rename to python/pythonmonkey/require.py index 3c81a22f..a73f8c91 100644 --- a/pythonmonkey_require.py +++ b/python/pythonmonkey/require.py @@ -30,10 +30,9 @@ from importlib import machinery from os import stat, path, getcwd, getenv -sys.path.append(path.dirname(__file__) + '/build/src') warnings.filterwarnings("ignore", category=DeprecationWarning) -import pythonmonkey as pm +from pythonmonkey import pythonmonkey as pm pm.eval(""" globalThis.python = {}; From ddbdb92470b2b7f64e27cac61ae98614e85b3676 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 22 Jun 2023 22:19:51 +0000 Subject: [PATCH 18/60] chore: `npm i` should be run in `python/pythonmonkey` directory --- build_script.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_script.sh b/build_script.sh index 22f2bfe4..0947ee9a 100755 --- a/build_script.sh +++ b/build_script.sh @@ -11,5 +11,6 @@ cd build cmake .. cmake --build . -j$CPUS +cd ../python/pythonmonkey # npm is used to load JS components, see package.json npm i From bcd6f6785a05d0b87ab5881dc4b662e12ff67b61 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 22 Jun 2023 22:22:02 +0000 Subject: [PATCH 19/60] test(promise): fix regex doesn't match when having `ctx-module` The input string contains the newline character `\n` , but the regex doesn't match across multiple lines --- tests/python/test_pythonmonkey_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index eae889c5..ef068318 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -842,7 +842,7 @@ async def coro_fn(x): # JS Promise to Python awaitable coercion assert 100 == await pm.eval("new Promise((r)=>{ r(100) })") assert 10010 == await pm.eval("Promise.resolve(10010)") - with pytest.raises(pm.SpiderMonkeyError, match="TypeError: .+ is not a constructor"): + with pytest.raises(pm.SpiderMonkeyError, match="^TypeError: (.|\\n)+ is not a constructor$"): await pm.eval("Promise.resolve")(10086) assert 10086 == await pm.eval("Promise.resolve.bind(Promise)")(10086) From 3423f195d8858811623c56073a78b17e840f4620 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Thu, 22 Jun 2023 22:33:44 +0000 Subject: [PATCH 20/60] chore: must explicitly include `node_modules` into the wheel package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a VCS is being used for a package, Poetry exclude field will be seeded with the VCS’ ignore settings (.gitignore for git for example). --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 87dc9e88..55f95d68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ include = [ "python/pythonmonkey/pythonmonkey.so", "python/pythonmonkey/libmozjs*", + "python/pythonmonkey/node_modules/**/*", + # include all files for source distribution { path = "src", format = "sdist" }, { path = "include", format = "sdist" }, From 336b0cc3a7282d8e93c910e005cf4f36e01d9210 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 11:33:37 -0400 Subject: [PATCH 21/60] Add reentrance_smoke test --- tests/python/reentrance_smoke.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/python/reentrance_smoke.py diff --git a/tests/python/reentrance_smoke.py b/tests/python/reentrance_smoke.py new file mode 100644 index 00000000..52dd56a4 --- /dev/null +++ b/tests/python/reentrance_smoke.py @@ -0,0 +1,17 @@ +# @file reentrance-smoke.py +# Basic smoke test which shows that JS->Python->JS->Python calls work. Failures are +# indicated by segfaults. +# @author Wes Garland, wes@distributive.network +# @date June 2023 + +import sys, os +sys.path.append(os.path.dirname(__file__) + '/../../python') # location of pythonmonkey module +import pythonmonkey as pm + +globalThis = pm.eval("globalThis;"); +globalThis.pmEval = pm.eval; +globalThis.pyEval = eval; + +abc=(pm.eval("() => { return {def: pyEval('123')} };"))() +assert(abc['def'] == 123) +print(pm.eval("pmEval(`pyEval(\"'test passed'\")`)")) From 32b09f420b36fb14c41e9b077ed91c2fa00f6e05 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 11:49:33 -0400 Subject: [PATCH 22/60] eval - implement support for compilation options --- src/modules/pythonmonkey/pythonmonkey.cc | 68 ++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index 25cc538b..cf8c1031 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -143,17 +144,72 @@ static PyObject *asUCS4(PyObject *self, PyObject *args) { return str->asUCS4(); } -static PyObject *eval(PyObject *self, PyObject *args) { +static bool getEvalOption(PyObject *evalOptions, const char *optionName, const char **s_p) { + PyObject *value; + + value = PyDict_GetItemString(evalOptions, optionName); + if (value) + *s_p = PyUnicode_AsUTF8(value); + return value != NULL; +} + +static bool getEvalOption(PyObject *evalOptions, const char *optionName, unsigned long *l_p) { + PyObject *value; + + value = PyDict_GetItemString(evalOptions, optionName); + if (value) + *l_p = PyLong_AsUnsignedLong(value); + return value != NULL; +} + +static bool getEvalOption(PyObject *evalOptions, const char *optionName, bool *b_p) { + PyObject *value; + + value = PyDict_GetItemString(evalOptions, optionName); + if (value) { + if (PyLong_Check(value)) + *b_p = PyBool_FromLong(PyLong_AsLong(value)); + else + *b_p = value != Py_False; + } + return value != NULL; +} +static PyObject *eval(PyObject *self, PyObject *args) { StrType *code = new StrType(PyTuple_GetItem(args, 0)); + PyObject *evalOptions = PyTuple_GET_SIZE(args) == 2 ? PyTuple_GetItem(args, 1) : NULL; + if (!PyUnicode_Check(code->getPyObject())) { PyErr_SetString(PyExc_TypeError, "pythonmonkey.eval expects a string as its first argument"); return NULL; } + if (evalOptions && !PyDict_Check(evalOptions)) { + PyErr_SetString(PyExc_TypeError, "pythonmonkey.eval expects a dict as its (option) second argument"); + return NULL; + } + JSAutoRealm ar(GLOBAL_CX, *global); JS::CompileOptions options (GLOBAL_CX); - options.setFileAndLine("noname", 1); + options.setFileAndLine("@evaluate", 1) + .setIsRunOnce(true) + .setNoScriptRval(false) + .setIntroductionType("pythonmonkey eval"); + + if (evalOptions) { + const char *s; + unsigned long l; + bool b; + + if (getEvalOption(evalOptions, "filename", &s)) options.setFile(s); + if (getEvalOption(evalOptions, "lineno", &l)) options.setLine(l); + if (getEvalOption(evalOptions, "column", &l)) options.setColumn(l); + if (getEvalOption(evalOptions, "mutedErrors", &b)) options.setMutedErrors(b); + if (getEvalOption(evalOptions, "noScriptRval", &b)) options.setNoScriptRval(b); + if (getEvalOption(evalOptions, "selfHosting", &b)) options.setSelfHostingMode(b); + if (getEvalOption(evalOptions, "strict", &b)) if (b) options.setForceStrictMode(); + if (getEvalOption(evalOptions, "module", &b)) if (b) options.setModule(); + } // initialize JS context JS::SourceText source; @@ -309,6 +365,12 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) return NULL; } + JS::ContextOptionsRef(GLOBAL_CX) + .setWasm(true) + .setAsmJS(true) + .setAsyncStack(true) + .setSourcePragmas(true); + JOB_QUEUE = new JobQueue(); if (!JOB_QUEUE->init(GLOBAL_CX)) { PyErr_SetString(SpiderMonkeyError, "Spidermonkey could not create the event-loop."); @@ -374,4 +436,4 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) return NULL; } return pyModule; -} \ No newline at end of file +} From 0ea3b5431e06b3e9901cda776106434654e71235 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 11:51:28 -0400 Subject: [PATCH 23/60] Add simple console.log placeholder --- console.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 console.js diff --git a/console.js b/console.js new file mode 100644 index 00000000..8d4cef9d --- /dev/null +++ b/console.js @@ -0,0 +1,21 @@ +/** + * @file console.js + * Temporary implementation of console.log etc. + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ +return {} + +function Console(print) +{ + this.log = print; + this.debug = print; + this.error = print; + this.warn = print; + this.info = print; +} + +if (!global.console) + global.console = new Console(python.print); + +exports.Console = Console; From 7456148b00b0633f403504e014ec5b1634d20ca5 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 11:52:21 -0400 Subject: [PATCH 24/60] Add simple js testing --- js-test-runner | 29 +++++++++++++++++++++++++++++ tests/js/not-strict-mode.simple | 16 ++++++++++++++++ tests/js/use-strict.simple | 18 ++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100755 js-test-runner create mode 100644 tests/js/not-strict-mode.simple create mode 100644 tests/js/use-strict.simple diff --git a/js-test-runner b/js-test-runner new file mode 100755 index 00000000..39313983 --- /dev/null +++ b/js-test-runner @@ -0,0 +1,29 @@ +#! /usr/bin/env python3 + +### Loader because Python is awkward +import sys +from os import path +from importlib import machinery +from pathlib import Path + +def include(relative, filename): + filename = str(Path(relative, filename).resolve()) + name = path.basename(filename) + if name in sys.modules: + return sys.modules[name] + sourceFileLoader = machinery.SourceFileLoader(name, filename) + module = sourceFileLoader.load_module(name) + return module + +import sys +sys.path.append(path.dirname(__file__) + '/.'); + +pmr = include(path.dirname(__file__), './pythonmonkey_require.py'); + +# Main +require = pmr.createRequire(__file__) +#require('./console'); + +testName = path.realpath(sys.argv[1]); +testDir = path.dirname(testName); +pmr.createRequire(testName)(testName); diff --git a/tests/js/not-strict-mode.simple b/tests/js/not-strict-mode.simple new file mode 100644 index 00000000..9b202115 --- /dev/null +++ b/tests/js/not-strict-mode.simple @@ -0,0 +1,16 @@ +/** + * @file not-strict-mode.simple + * Simple test which ensures that tests are evaluated correctly, such that the interpreter + * does not run tests in strict mode unless explicitly directed to do so. + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ +function fun(abc) +{ + arguments[0] = 123; + + if (abc !== 123) + throw new Error('interpreter is in strict mode'); +} + +fun(456); diff --git a/tests/js/use-strict.simple b/tests/js/use-strict.simple new file mode 100644 index 00000000..8e927de4 --- /dev/null +++ b/tests/js/use-strict.simple @@ -0,0 +1,18 @@ +/** + * @file use-strict.simple + * Simple test which ensures that tests are evaluated correctly, such that the "use strict" + * directive has meaning. + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ +"use strict"; + +function fun(abc) +{ + arguments[0] = 123; + + if (abc === 123) + throw new Error('"use strict" did not put interpreter in strict mode'); +} + +fun(456); From fc3314bcb4dd3110bcde43d6f6f75c4bef367acf Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 14:01:31 -0400 Subject: [PATCH 25/60] peter-jr - improve output --- peter-jr | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 peter-jr diff --git a/peter-jr b/peter-jr new file mode 100755 index 00000000..6fe4bab6 --- /dev/null +++ b/peter-jr @@ -0,0 +1,101 @@ +#! /bin/bash +# +# @file peter-js +# A simple test framework in the spirit of Peter (npmjs.com/packages/peter) for testing +# basic PythonMonkey functionality. +# @author Wes Garland, wes@distributive.network +# @date Jun 2023 +# +runDir=`pwd` +cd `dirname "$0"` +topDir=`pwd` +cd "$runDir" + +set -u +set -o pipefail + +if [ "${1:-}" = "-v" ]; then + VERBOSE=1 + shift +else + VERBOSE="${VERBOSE:-}" +fi + +[ "${1:-}" ] || set "${topDir}/tests/js" +testLocations="$@" + +function panic() +{ + echo "PANIC: $*" >&2 + exit 2 +} + +TMP=`mktemp -d` +([ "${TMP}" ] && [ -d "${TMP}" ]) || exit 3 + +trap "rm -r \"${TMP}\"" EXIT + +if [ "$VERBOSE" ]; then + stdout="/dev/stdout" + stderr="/dev/stderr" +else + stdout="$TMP/both" + stderr="$TMP/both" +fi + +red() +{ + printf "\e[0;31m%s\e[0m" "$*" +} + +green() +{ + printf "\e[0;32m%s\e[0m" "$*" +} + +( + for loc in "${testLocations[@]}" + do + find $(realpath "$loc") -type f -name \*.simple + find $(realpath "$loc") -type f -name \*.bash + done +) \ +| while read file + do + sfile=$(realpath --relative-to="${runDir}" "${file}") + printf 'Testing %-40s ... ' "${sfile})" + ext="${file##*.}" + ( + case "$ext" in + "simple") + ./js-test-runner "$file" + exitCode="$?" + ;; + "bash") + bash "$file" + exitCode="$?" + ;; + *) + echo + panic "${file}: invalid extension '$ext'" + ;; + esac + + exit "$exitCode" + )> $stdout 2>$stderr + exitCode="$?" + + [ "$VERBOSE" ] || cat "$stderr" + if [ "$exitCode" = "0" ]; then + echo "$(green PASS)" + else + echo "$(red FAIL)" + if [ ! "$VERBOSE" ]; then + echo "-- ${file} vvvvvvvvvvvvvv" + cat "$stderr" | sed 's/^/ /' + [ "$stderr" = "$stdout" ] || cat "$stderr" | sed 's/^/ /' + echo "-- ${file} ^^^^^^^^^^^^^^" + echo + fi + fi + done From 7eaa6c76a49a3427d03cbd84f200bf26298923fc Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 15:33:10 -0400 Subject: [PATCH 26/60] Add trivial REPL; pmjs --- pmjs | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 pmjs diff --git a/pmjs b/pmjs new file mode 100755 index 00000000..f868c028 --- /dev/null +++ b/pmjs @@ -0,0 +1,71 @@ +#! /usr/bin/env python3 +# @file pmjs - PythonMonkey REPL +# @author Wes Garland, wes@distributive.network +# @date June 2023 + +import sys, os, cmd +sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module +import pythonmonkey as pm + +globalThis = pm.eval("globalThis;"); +globalThis.pmEval = pm.eval; + +globalThis = pm.eval("globalThis;"); +globalThis.python = {'exit': exit }; + +pm.eval(""" +const cmds = { + help: function help() { + return '' + +`.exit Exit the REPL +.help Print this help message +.load Load JS from a file into the REPL session +.save Save all evaluated commands in this REPL session to a file + +Press Ctrl+C to abort current expression, Ctrl+D to exit the REPL` + }, +}; + +globalThis.replEval = function replEval(line) +{ + try + { + if (line[0] === '.') + return cmds[line.slice(1)](); + if (line === String.fromCharCode(4)) + python.exit(0); + + const result = eval(line); + switch (typeof result) + { + case 'string': + case 'number': + case 'boolean': + return result; + case 'object': + if (result instanceof Error) + return result.stack || result.message; + return JSON.stringify(result); + default: + return String(result); + } + } + catch(error) + { + return error.stack || result.message; + } +} +"""); + +class JSShell(cmd.Cmd): + intro = 'Welcome to PythonMonkey v' + pm.__version__ +'.\nType ".help" for more information.'; + + prompt = '> ' + + def parseline(self, line): + print(globalThis.replEval(line)) + dummyRet = cmd.Cmd.parseline(self, '') + return dummyRet + +if __name__ == '__main__': + JSShell().cmdloop() From 4fdb52da8dd1598648628ee1a318c6d67a89658d Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 15:33:34 -0400 Subject: [PATCH 27/60] Add hard-coded __version__ to pythonmonkey module --- python/pythonmonkey/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pythonmonkey/__init__.py b/python/pythonmonkey/__init__.py index f643b61f..53b6e1e3 100644 --- a/python/pythonmonkey/__init__.py +++ b/python/pythonmonkey/__init__.py @@ -1 +1,2 @@ from .pythonmonkey import * +__version__ = '0.1.0' From ab1a0a3112f006409c730767960de71c462fbdef Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 15:33:58 -0400 Subject: [PATCH 28/60] Add emacs tmp files to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b5ed8779..e7bc5990 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ dist *.so _spidermonkey_install/* node_modules +*~ From baa2c669e54cdd6b6e0973d683f403de0d365c07 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 15:35:01 -0400 Subject: [PATCH 29/60] Add (dormant) implementation of runInContext for require --- build_script.sh | 1 + examples/use-require.py | 1 + pythonmonkey_require.py | 40 +++++++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/build_script.sh b/build_script.sh index 22f2bfe4..689c1359 100755 --- a/build_script.sh +++ b/build_script.sh @@ -10,6 +10,7 @@ fi cd build cmake .. cmake --build . -j$CPUS +cp -f ../build/src/pythonmonkey.so ../python/pythonmonkey/ # npm is used to load JS components, see package.json npm i diff --git a/examples/use-require.py b/examples/use-require.py index 4d53919c..52843c62 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -1,5 +1,6 @@ ### Loader because Python is awkward import sys +import importlib from os import path from importlib import machinery from pathlib import Path diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py index 3c81a22f..3b11f5b3 100644 --- a/pythonmonkey_require.py +++ b/pythonmonkey_require.py @@ -23,19 +23,33 @@ # @date May 2023 # -import sys, warnings -import types +import sys, os, types from typing import Union, Dict, Callable import importlib -from importlib import machinery -from os import stat, path, getcwd, getenv - -sys.path.append(path.dirname(__file__) + '/build/src') -warnings.filterwarnings("ignore", category=DeprecationWarning) +sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm +globalThis = pm.eval("globalThis;"); +globalThis.pmEval = pm.eval; + pm.eval(""" +'use strict'; + +function runInContext(code, _unused_contextifiedObject, options) +{ + var evalOptions = {}; + + if (arguments.length === 2) + options = arguments[2]; + + if (options.filename) evalOptions.filename = options.filename; + if (options.lineOffset) evalOptions.lineno = options.lineOffset; + if (options.columnOffset) evalOptions.column = options.columnOffset; + + return pmEval(code, evalOptions); +} + globalThis.python = {}; globalThis.global = globalThis; globalThis.vmModule = { runInContext: eval }; @@ -97,7 +111,7 @@ # Add some python functions to the global python object for code in this file to use. propSet('python', 'print', print); -propSet('python', 'getenv', getenv); +propSet('python', 'getenv', os.getenv); propSet('python', 'paths', ':'.join(sys.path)); pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays @@ -111,7 +125,7 @@ def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. """ from os import stat - if (path.exists(filename)): + if (os.path.exists(filename)): sb = stat(filename) return { 'mode': sb.st_mode } else: @@ -128,7 +142,7 @@ def readFileSync(filename, charset) -> str: propSet('fsModule', 'statSync_inner', statSync_inner); propSet('fsModule', 'readFileSync', readFileSync) -propSet('fsModule', 'existsSync', path.exists) +propSet('fsModule', 'existsSync', os.path.exists) pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") pm.eval("""fsModule.statSync = function statSync(filename) @@ -148,7 +162,7 @@ def readFileSync(filename, charset) -> str: # parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports # dict to decorate. -with open(path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: +with open(os.path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: moduleWrapper = pm.eval("""'use strict'; (function moduleWrapper(require, exports) { @@ -178,9 +192,9 @@ def load(filename: str) -> Dict: : The loaded python module """ - name = path.basename(filename) + name = os.path.basename(filename) if name not in sys.modules: - sourceFileLoader = machinery.SourceFileLoader(name, filename) + sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) module = importlib.util.module_from_spec(spec) sys.modules[name] = module From 01d34e234bb9a56322b1a988fa30b298941ae0b0 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 20:35:33 -0400 Subject: [PATCH 30/60] Generate __version__ directly from pyproject.toml at build time --- .gitignore | 7 ++++++- build_script.sh | 19 ++++++++++++++++--- pyproject.toml | 4 ++-- python/pythonmonkey/__init__.py | 6 +++++- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e7bc5990..90da3601 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ build/ .vscode/ pyvenv.cfg +python/pythonmonkey/version.py bin/ lib/* .pytest_cache @@ -10,9 +11,13 @@ firefox-*/ __pycache__ Testing/Temporary _spidermonkey_install -__pycache__ +__pycache__/* dist *.so _spidermonkey_install/* node_modules *~ +Makefile +CMakeDoxyfile.in +CMakeDoxygenDefaults.cmake +CMakeCache.txt diff --git a/build_script.sh b/build_script.sh index 689c1359..6a63cb68 100755 --- a/build_script.sh +++ b/build_script.sh @@ -1,3 +1,12 @@ +#! /bin/bash +# +# @file build.sh +# @author Giovanni Tedesco +# @date Aug 2022 + +cd `dirname "$0"` +topDir=`pwd` + # Get number of CPU cores CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || getconf NPROCESSORS_ONLN 2>/dev/null || echo 1) @@ -10,7 +19,11 @@ fi cd build cmake .. cmake --build . -j$CPUS -cp -f ../build/src/pythonmonkey.so ../python/pythonmonkey/ -# npm is used to load JS components, see package.json -npm i +cd "$topDir" +npm i # npm is used to load JS components, see package.json +cp -f build/src/pythonmonkey.so python/pythonmonkey/ +grep '^version' pyproject.toml \ +| head -1 \ +| sed 's/^version /__version__ /' \ +> python/pythonmonkey/version.py diff --git a/pyproject.toml b/pyproject.toml index e5672feb..81b9e2b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [tool.poetry] name = "pythonmonkey" -version = "0.1.0" +version = "0.1.1" description = "" -authors = ["Caleb Aikens ", "Tom Tang "] +authors = ["Caleb Aikens ", "Tom Tang ", "Wes Garland ", "Hamada Gasmallah "] readme = "README.md" packages = [ { include = "pythonmonkey", from = "python" }, diff --git a/python/pythonmonkey/__init__.py b/python/pythonmonkey/__init__.py index 53b6e1e3..a5294900 100644 --- a/python/pythonmonkey/__init__.py +++ b/python/pythonmonkey/__init__.py @@ -1,2 +1,6 @@ from .pythonmonkey import * -__version__ = '0.1.0' +#from .__version__ import * +import os +exec(open(os.path.dirname(__file__) + '/version.py').read()) +#import importlib.metadata +#__version__= importlib.metadata.version(__name__) From a246b51363579b0c3ce6b26a34c6500d19786f21 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 22:47:38 -0400 Subject: [PATCH 31/60] Add console smoke test --- peter-jr | 6 +++--- tests/js/console-smoke.simple | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/js/console-smoke.simple diff --git a/peter-jr b/peter-jr index 6fe4bab6..4dddad26 100755 --- a/peter-jr +++ b/peter-jr @@ -39,8 +39,8 @@ if [ "$VERBOSE" ]; then stdout="/dev/stdout" stderr="/dev/stderr" else - stdout="$TMP/both" - stderr="$TMP/both" + stdout="$TMP/stdout" + stderr="$TMP/stderr" fi red() @@ -85,7 +85,6 @@ green() )> $stdout 2>$stderr exitCode="$?" - [ "$VERBOSE" ] || cat "$stderr" if [ "$exitCode" = "0" ]; then echo "$(green PASS)" else @@ -98,4 +97,5 @@ green() echo fi fi + [ "$VERBOSE" ] || cat "$stderr" done diff --git a/tests/js/console-smoke.simple b/tests/js/console-smoke.simple new file mode 100644 index 00000000..07c203a5 --- /dev/null +++ b/tests/js/console-smoke.simple @@ -0,0 +1,12 @@ +/** + * @file console-smoke.simple + * Simple smoke test which ensures that the global console is initialized + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ +console.log('one'); +console.info('two'); +console.debug('three'); +console.error('four'); +console.warn('five'); + From 7619a00f99182f5932c65f4980f38f81b324a130 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 22:51:55 -0400 Subject: [PATCH 32/60] require - add ./builtin_modules/ to list of search paths --- python/pythonmonkey/require.py | 241 +++++++++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 python/pythonmonkey/require.py diff --git a/python/pythonmonkey/require.py b/python/pythonmonkey/require.py new file mode 100644 index 00000000..492f4cd9 --- /dev/null +++ b/python/pythonmonkey/require.py @@ -0,0 +1,241 @@ +# @file require.py +# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the +# ctx-module npm package to do the heavy lifting. That package makes a complete module +# system, obstensibly in a separate context, but our implementation here reuses the +# PythonMonkey global context for both. +# +# The context that ctx-module runs in needs a require function supporting +# - require('debug') => returns a debug function which prints to the console when +# debugging; see the node debug built-in +# - require('fs') => returns an object which has an implementation of readFileSync +# and statSync. The implementation of statSync only needs to +# return the mode member. The fs module also needs +# constants.S_IFDIR available. +# - require('vm') => returns an object which has an implementation of evalInContext +# +# In order to implement this basic require function for bootstrapping ctxModule, we +# have simply made global variables of the form xxxModule where xxx is the module +# identifier, and injected a require function which understands this. A better +# implementation in Python that doesn't leak global symbols should be possible once +# some PythonMonkey bugs are fixed. +# +# @author Wes Garland, wes@distributive.network +# @date May 2023 +# + +import sys, os, types +from typing import Union, Dict, Callable +import importlib + +sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module +import pythonmonkey as pm + +globalThis = pm.eval("globalThis;"); +globalThis.pmEval = pm.eval; + +pm.eval(""" +'use strict'; + +function runInContext(code, _unused_contextifiedObject, options) +{ + var evalOptions = {}; + + if (arguments.length === 2) + options = arguments[2]; + + if (options.filename) evalOptions.filename = options.filename; + if (options.lineOffset) evalOptions.lineno = options.lineOffset; + if (options.columnOffset) evalOptions.column = options.columnOffset; + + return pmEval(code, evalOptions); +} + +globalThis.python = { pythonMonkey: {} }; +globalThis.global = globalThis; +globalThis.vmModule = { runInContext: eval }; +globalThis.require = function outerRequire(mid) { + const module = globalThis[mid + 'Module']; + if (module) + return module; + throw new Error('module not found: ' + mid); +}; +globalThis.debugModule = function debug(selector) { + var idx, colour; + const esc = String.fromCharCode(27); + const noColour = `${esc}[0m`; + + debug.selectors = debug.selectors || []; + idx = debug.selectors.indexOf(selector); + if (idx === -1) + { + idx = debug.selectors.length; + debug.selectors.push(selector); + } + + colour = `${esc}[${91 + ((idx + 1) % 6)}m`; + const debugEnv = python.getenv('DEBUG'); + + if (debugEnv) + { + for (let sym of debugEnv.split(' ')) + { + const re = new RegExp('^' + sym.replace('*', '.*') + '$'); + if (re.test(selector)) + { + return (function debugInner() { + python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) + }); + } + } + } + + /* no match => silent */ + return (function debugDummy() {}); +}; + +function globalSet(name, prop) +{ + globalThis[name] = prop; +} + +function propSet(objName, propName, propValue) +{ + globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; +} + +""") + +# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. +globalSet = pm.eval("globalSet"); +propSet = pm.eval("propSet") + +# Add some python functions to the global python object for code in this file to use. +propSet('python', 'print', print); +propSet('python', 'getenv', os.getenv); +propSet('python', 'paths', ':'.join(sys.path)); +pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays + +globalThis.python.pythonMonkey.dir = os.path.dirname(__file__); + +# Implement enough of require('fs') so that ctx-module can find/load files + +def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: + """ + Inner function for statSync. + + Returns: + Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. + """ + from os import stat + if (os.path.exists(filename)): + sb = stat(filename) + return { 'mode': sb.st_mode } + else: + return False + +def readFileSync(filename, charset) -> str: + """ + Utility function for reading files. + Returns: + str: The contents of the file + """ + with open(filename, "r", encoding=charset) as fileHnd: + return fileHnd.read() + +propSet('fsModule', 'statSync_inner', statSync_inner); +propSet('fsModule', 'readFileSync', readFileSync) +propSet('fsModule', 'existsSync', os.path.exists) +pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") +pm.eval("""fsModule.statSync = +function statSync(filename) +{ + const ret = require('fs').statSync_inner(filename); + if (ret) + return ret; + + const err = new Error('file not found: ' + filename); + err.code='ENOENT'; + throw err; +}"""); + +# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is +# now the corresponding CommonJS require() function. We use the globalThis as the module's exports +# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE +# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports +# dict to decorate. + +with open(os.path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: + moduleWrapper = pm.eval("""'use strict'; +(function moduleWrapper(require, exports) +{ + exports=exports || globalThis; + require=require || globalThis.require; +""" + ctxModuleSource.read() + """ +}) +"""); + +# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have +# to use globalThis and pollute the global scope. +moduleWrapper() + +# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same +# dict->jsObject in createRequire for every require we create. +pm.eval('const __builtinModules = {}; true'); + +def load(filename: str) -> Dict: + """ + Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. + If the module is already loaded, returns it. + + Args: + filename (str): The filename of the python module to load. + + Returns: + : The loaded python module + """ + + name = os.path.basename(filename) + if name not in sys.modules: + sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) + spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + module.exports = {} + spec.loader.exec_module(module) + else: + module = sys.modules[name] + module_exports = {} + for key in dir(module): + module_exports[key] = getattr(module, key) + return module_exports + +propSet('python', 'load', load) + +""" +API - createRequire +returns a require function that resolves modules relative to the filename argument. +Conceptually the same as node:module.createRequire(). + +example: + from pythonmonkey import createRequire + require = createRequire(__file__) + require('./my-javascript-module') +""" +createRequire: Callable = pm.eval("""( +function createRequire(filename) +{ + function loadPythonModule(module, filename) + { + module.exports = python.load(filename); + } + + const module = new CtxModule(globalThis, filename, __builtinModules); + for (let path of python.paths) + module.paths.push(path + '/node_modules'); + if (module.require.path.length === 0) + module.require.path.push(python.pythonMonkey.dir + '/builtin_modules'); + module.require.extensions['.py'] = loadPythonModule; + + return module.require; +} +)""") From 2c594e5896a507d0147c63f4d896ef64ea007311 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 22:54:39 -0400 Subject: [PATCH 33/60] temp console - plumb-in globalThis.console as a side-effect of loading builtin_modules/console.js --- .../pythonmonkey/builtin_modules/console.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) rename console.js => python/pythonmonkey/builtin_modules/console.js (80%) diff --git a/console.js b/python/pythonmonkey/builtin_modules/console.js similarity index 80% rename from console.js rename to python/pythonmonkey/builtin_modules/console.js index 8d4cef9d..655b0f77 100644 --- a/console.js +++ b/python/pythonmonkey/builtin_modules/console.js @@ -4,8 +4,6 @@ * @author Wes Garland, wes@distributive.network * @date June 2023 */ -return {} - function Console(print) { this.log = print; @@ -15,7 +13,7 @@ function Console(print) this.info = print; } -if (!global.console) - global.console = new Console(python.print); +if (!globalThis.console) + globalThis.console = new Console(python.print); exports.Console = Console; From 5118c58c7c74119e2924f75ef025700ea526ab87 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 22:58:17 -0400 Subject: [PATCH 34/60] Update key file locations and paths so that createRequire can be exported from pythonmonkey --- .gitignore | 2 +- README.md | 11 + build_script.sh | 10 +- js-test-runner | 38 +-- python/pythonmonkey/__init__.py | 4 +- .../pythonmonkey/package-lock.json | 0 .../pythonmonkey/package.json | 0 pythonmonkey_require.py | 236 ------------------ 8 files changed, 33 insertions(+), 268 deletions(-) rename package-lock.json => python/pythonmonkey/package-lock.json (100%) rename package.json => python/pythonmonkey/package.json (100%) delete mode 100644 pythonmonkey_require.py diff --git a/.gitignore b/.gitignore index 90da3601..830f8bec 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ .vscode/ pyvenv.cfg python/pythonmonkey/version.py +python/pythonmonkey/node_modules bin/ lib/* .pytest_cache @@ -15,7 +16,6 @@ __pycache__/* dist *.so _spidermonkey_install/* -node_modules *~ Makefile CMakeDoxyfile.in diff --git a/README.md b/README.md index f444265b..9b8c10fe 100644 --- a/README.md +++ b/README.md @@ -96,3 +96,14 @@ Type "help", "copyright", "credits" or "license" for more information. ``` Alternatively, you can build a `wheel` package by running `poetry build --format=wheel`, and install it by `pip install dist/*.whl`. + +# Troubleshooting Tips + +## CommonJS (require) +If you are having trouble with the CommonJS require function, set DEBUG='ctx-module*' and you can see the filenames it tries to laod + +### Extra Symbols +Loading the CommonJS subsystem declares some extra symbols which may be helpful in debugging - +- `python.print` - the Python print function +- `python.getenv` - the Python getenv function + diff --git a/build_script.sh b/build_script.sh index 6a63cb68..7c71b9d1 100755 --- a/build_script.sh +++ b/build_script.sh @@ -20,10 +20,14 @@ cd build cmake .. cmake --build . -j$CPUS -cd "$topDir" -npm i # npm is used to load JS components, see package.json +cd "${topDir}" cp -f build/src/pythonmonkey.so python/pythonmonkey/ +echo "# This file was generated via $0 by `id -un` on `date` - do not edit by hand!" grep '^version' pyproject.toml \ | head -1 \ | sed 's/^version /__version__ /' \ -> python/pythonmonkey/version.py +>> python/pythonmonkey/version.py + +# npm is used to load JS components, see package.json +cd "${topDir}/python/pythonmonkey/" +npm i diff --git a/js-test-runner b/js-test-runner index 39313983..2ef96847 100755 --- a/js-test-runner +++ b/js-test-runner @@ -1,29 +1,17 @@ #! /usr/bin/env python3 - -### Loader because Python is awkward -import sys -from os import path -from importlib import machinery -from pathlib import Path - -def include(relative, filename): - filename = str(Path(relative, filename).resolve()) - name = path.basename(filename) - if name in sys.modules: - return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) - return module - -import sys -sys.path.append(path.dirname(__file__) + '/.'); - -pmr = include(path.dirname(__file__), './pythonmonkey_require.py'); +# @file js-test-runner +# A simple test driver for peter-jr that runs JS code via PythonMonkey.require +# @author Wes Garland, wes@distributive.network +# @date Jun 2023 + +import sys, os +sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module +import pythonmonkey as pm # Main -require = pmr.createRequire(__file__) -#require('./console'); +require = pm.createRequire(__file__) +require('console'); -testName = path.realpath(sys.argv[1]); -testDir = path.dirname(testName); -pmr.createRequire(testName)(testName); +testName = os.path.realpath(sys.argv[1]); +testDir = os.path.dirname(testName); +pm.createRequire(testName)(testName); diff --git a/python/pythonmonkey/__init__.py b/python/pythonmonkey/__init__.py index a5294900..fd788b97 100644 --- a/python/pythonmonkey/__init__.py +++ b/python/pythonmonkey/__init__.py @@ -1,6 +1,4 @@ from .pythonmonkey import * -#from .__version__ import * +from .require import * import os exec(open(os.path.dirname(__file__) + '/version.py').read()) -#import importlib.metadata -#__version__= importlib.metadata.version(__name__) diff --git a/package-lock.json b/python/pythonmonkey/package-lock.json similarity index 100% rename from package-lock.json rename to python/pythonmonkey/package-lock.json diff --git a/package.json b/python/pythonmonkey/package.json similarity index 100% rename from package.json rename to python/pythonmonkey/package.json diff --git a/pythonmonkey_require.py b/pythonmonkey_require.py deleted file mode 100644 index 3b11f5b3..00000000 --- a/pythonmonkey_require.py +++ /dev/null @@ -1,236 +0,0 @@ -# @file require.py -# Implementation of CommonJS "require" for PythonMonkey. This implementation uses the -# ctx-module npm package to do the heavy lifting. That package makes a complete module -# system, obstensibly in a separate context, but our implementation here reuses the -# PythonMonkey global context for both. -# -# The context that ctx-module runs in needs a require function supporting -# - require('debug') => returns a debug function which prints to the console when -# debugging; see the node debug built-in -# - require('fs') => returns an object which has an implementation of readFileSync -# and statSync. The implementation of statSync only needs to -# return the mode member. The fs module also needs -# constants.S_IFDIR available. -# - require('vm') => returns an object which has an implementation of evalInContext -# -# In order to implement this basic require function for bootstrapping ctxModule, we -# have simply made global variables of the form xxxModule where xxx is the module -# identifier, and injected a require function which understands this. A better -# implementation in Python that doesn't leak global symbols should be possible once -# some PythonMonkey bugs are fixed. -# -# @author Wes Garland, wes@distributive.network -# @date May 2023 -# - -import sys, os, types -from typing import Union, Dict, Callable -import importlib - -sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module -import pythonmonkey as pm - -globalThis = pm.eval("globalThis;"); -globalThis.pmEval = pm.eval; - -pm.eval(""" -'use strict'; - -function runInContext(code, _unused_contextifiedObject, options) -{ - var evalOptions = {}; - - if (arguments.length === 2) - options = arguments[2]; - - if (options.filename) evalOptions.filename = options.filename; - if (options.lineOffset) evalOptions.lineno = options.lineOffset; - if (options.columnOffset) evalOptions.column = options.columnOffset; - - return pmEval(code, evalOptions); -} - -globalThis.python = {}; -globalThis.global = globalThis; -globalThis.vmModule = { runInContext: eval }; -globalThis.require = function outerRequire(mid) { - const module = globalThis[mid + 'Module']; - if (module) - return module; - throw new Error('module not found: ' + mid); -}; -globalThis.debugModule = function debug(selector) { - var idx, colour; - const esc = String.fromCharCode(27); - const noColour = `${esc}[0m`; - - debug.selectors = debug.selectors || []; - idx = debug.selectors.indexOf(selector); - if (idx === -1) - { - idx = debug.selectors.length; - debug.selectors.push(selector); - } - - colour = `${esc}[${91 + ((idx + 1) % 6)}m`; - const debugEnv = python.getenv('DEBUG'); - - if (debugEnv) - { - for (let sym of debugEnv.split(' ')) - { - const re = new RegExp('^' + sym.replace('*', '.*') + '$'); - if (re.test(selector)) - { - return (function debugInner() { - python.print(`${colour}${selector}${noColour} ` + Array.from(arguments).join(' ')) - }); - } - } - } - - /* no match => silent */ - return (function debugDummy() {}); -}; - -function globalSet(name, prop) -{ - globalThis[name] = prop; -} - -function propSet(objName, propName, propValue) -{ - globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; -} - -""") - -# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. -globalSet = pm.eval("globalSet"); -propSet = pm.eval("propSet") - -# Add some python functions to the global python object for code in this file to use. -propSet('python', 'print', print); -propSet('python', 'getenv', os.getenv); -propSet('python', 'paths', ':'.join(sys.path)); -pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays - -# Implement enough of require('fs') so that ctx-module can find/load files - -def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: - """ - Inner function for statSync. - - Returns: - Union[Dict[str, int], False]: The mode of the file or False if the file doesn't exist. - """ - from os import stat - if (os.path.exists(filename)): - sb = stat(filename) - return { 'mode': sb.st_mode } - else: - return False - -def readFileSync(filename, charset) -> str: - """ - Utility function for reading files. - Returns: - str: The contents of the file - """ - with open(filename, "r", encoding=charset) as fileHnd: - return fileHnd.read() - -propSet('fsModule', 'statSync_inner', statSync_inner); -propSet('fsModule', 'readFileSync', readFileSync) -propSet('fsModule', 'existsSync', os.path.exists) -pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") -pm.eval("""fsModule.statSync = -function statSync(filename) -{ - const ret = require('fs').statSync_inner(filename); - if (ret) - return ret; - - const err = new Error('file not found: ' + filename); - err.code='ENOENT'; - throw err; -}"""); - -# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is -# now the corresponding CommonJS require() function. We use the globalThis as the module's exports -# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE -# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports -# dict to decorate. - -with open(os.path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: - moduleWrapper = pm.eval("""'use strict'; -(function moduleWrapper(require, exports) -{ - exports=exports || globalThis; - require=require || globalThis.require; -""" + ctxModuleSource.read() + """ -}) -"""); - -# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have -# to use globalThis and pollute the global scope. -moduleWrapper() - -# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same -# dict->jsObject in createRequire for every require we create. -pm.eval('const __builtinModules = {}; true'); - -def load(filename: str) -> Dict: - """ - Loads a python module using the importlib machinery sourcefileloader, prefills it with an exports object and returns the module. - If the module is already loaded, returns it. - - Args: - filename (str): The filename of the python module to load. - - Returns: - : The loaded python module - """ - - name = os.path.basename(filename) - if name not in sys.modules: - sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) - spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - module.exports = {} - spec.loader.exec_module(module) - else: - module = sys.modules[name] - module_exports = {} - for key in dir(module): - module_exports[key] = getattr(module, key) - return module_exports - -propSet('python', 'load', load) - -""" -API - createRequire -returns a require function that resolves modules relative to the filename argument. -Conceptually the same as node:module.createRequire(). - -example: - from pythonmonkey import createRequire - require = createRequire(__file__) - require('./my-javascript-module') -""" -createRequire: Callable = pm.eval("""( -function createRequire(filename) -{ - function loadPythonModule(module, filename) - { - module.exports = python.load(filename); - } - - const module = new CtxModule(globalThis, filename, __builtinModules); - for (let path of python.paths) - module.paths.push(path + '/node_modules'); - module.require.extensions['.py'] = loadPythonModule; - return module.require; -} -)""") From be297251c5fb1ebd0fa76b725e42ec2e5243a6e1 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 23:14:23 -0400 Subject: [PATCH 35/60] Update examples to load createRequire from pythonmonkey module --- examples/use-python-module.py | 29 ++++++++--------------------- examples/use-require.py | 29 ++++++++--------------------- python/pythonmonkey/require.py | 4 +++- 3 files changed, 19 insertions(+), 43 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index b64d62f0..65018bd4 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -1,24 +1,11 @@ -### Loader because Python is awkward -import sys -from os import path -import importlib -from importlib import machinery -from pathlib import Path +# @file use-require.py +# Sample code which demonstrates how to use require +# @author Wes Garland, wes@distributive.network +# @date Jun 2023 -def include(relative, filename): - filename = str(Path(relative, filename).resolve()) - name = path.basename(filename) - if name in sys.modules: - return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) - return module +import sys, os +sys.path.append(os.path.dirname(__file__) + '/../python') # location of pythonmonkey module +import pythonmonkey as pm -import sys -sys.path.append(path.dirname(__file__) + '/..'); - -pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); - -### Actual test below -require = pmr.createRequire(__file__) +require = pm.createRequire(__file__) require('./use-python-module'); diff --git a/examples/use-require.py b/examples/use-require.py index 52843c62..6ef787a4 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -1,26 +1,13 @@ -### Loader because Python is awkward -import sys -import importlib -from os import path -from importlib import machinery -from pathlib import Path +# @file use-require.py +# Sample code which demonstrates how to use require +# @author Wes Garland, wes@distributive.network +# @date Jun 2023 -def include(relative, filename): - filename = str(Path(relative, filename).resolve()) - name = path.basename(filename) - if name in sys.modules: - return sys.modules[name] - sourceFileLoader = machinery.SourceFileLoader(name, filename) - module = sourceFileLoader.load_module(name) - return module +import sys, os +sys.path.append(os.path.dirname(__file__) + '/../python') # location of pythonmonkey module +import pythonmonkey as pm -import sys -sys.path.append(path.dirname(__file__) + '/..'); - -pmr = include(path.dirname(__file__), '../pythonmonkey_require.py'); - -### Actual test below -require = pmr.createRequire(__file__) +require = pm.createRequire(__file__) require('./use-require/test1'); print("Done") diff --git a/python/pythonmonkey/require.py b/python/pythonmonkey/require.py index 492f4cd9..21b64d6b 100644 --- a/python/pythonmonkey/require.py +++ b/python/pythonmonkey/require.py @@ -26,6 +26,8 @@ import sys, os, types from typing import Union, Dict, Callable import importlib +from importlib import machinery +from importlib import util sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm @@ -196,7 +198,7 @@ def load(filename: str) -> Dict: name = os.path.basename(filename) if name not in sys.modules: - sourceFileLoader = importlib.machinery.SourceFileLoader(name, filename) + sourceFileLoader = machinery.SourceFileLoader(name, filename) spec = importlib.util.spec_from_loader(sourceFileLoader.name, sourceFileLoader) module = importlib.util.module_from_spec(spec) sys.modules[name] = module From 8c74c23c1187bc42612bd3e2a4e67bfc36379c53 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 23:52:12 -0400 Subject: [PATCH 36/60] peter-jr - improve output --- peter-jr | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/peter-jr b/peter-jr index 4dddad26..5535bd09 100755 --- a/peter-jr +++ b/peter-jr @@ -53,6 +53,16 @@ green() printf "\e[0;32m%s\e[0m" "$*" } +yellow() +{ + printf "\e[0;33m%s\e[0m" "$*" +} + +grey() +{ + printf "\e[0;90m%s\e[0m" "$*" +} + ( for loc in "${testLocations[@]}" do @@ -63,7 +73,7 @@ green() | while read file do sfile=$(realpath --relative-to="${runDir}" "${file}") - printf 'Testing %-40s ... ' "${sfile})" + printf 'Testing %-40s ... ' "${sfile}" ext="${file##*.}" ( case "$ext" in @@ -87,15 +97,20 @@ green() if [ "$exitCode" = "0" ]; then echo "$(green PASS)" + printf "\e[0;31m" + [ "$VERBOSE" ] || cat "$stderr" + printf "\e[0m" else echo "$(red FAIL)" if [ ! "$VERBOSE" ]; then - echo "-- ${file} vvvvvvvvvvvvvv" - cat "$stderr" | sed 's/^/ /' - [ "$stderr" = "$stdout" ] || cat "$stderr" | sed 's/^/ /' - echo "-- ${file} ^^^^^^^^^^^^^^" + echo + echo "$(grey --) $(yellow ${file}) $(grey vvvvvvvvvvvvvv)" + cat "$stdout" | sed 's/^/ /' + printf "\e[0;31m" + cat "$stderr" | sed -e 's/^/ /' + printf "\e[0m" + echo "$(grey --) $(yellow ${file}) $(grey ^^^^^^^^^^^^^^)" echo fi fi - [ "$VERBOSE" ] || cat "$stderr" done From ed09e2a69acc412cf8e699ec1dbb584931429fe7 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 23 Jun 2023 23:53:02 -0400 Subject: [PATCH 37/60] Add python-language CommonJS module test --- tests/js/load-cjs-python-module.simple | 11 +++++++++++ tests/js/modules/python-cjs-module.py | 9 +++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/js/load-cjs-python-module.simple create mode 100644 tests/js/modules/python-cjs-module.py diff --git a/tests/js/load-cjs-python-module.simple b/tests/js/load-cjs-python-module.simple new file mode 100644 index 00000000..9e02dca0 --- /dev/null +++ b/tests/js/load-cjs-python-module.simple @@ -0,0 +1,11 @@ +/** + * @file load-cjs-python-module.js + * Simple smoke test which ensures that we can load a CommonJS Module which happens to be + * written in Python instead of JavaScript + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ + +const { helloWorld } = require('./modules/python-cjs-module'); + +helloWorld(); diff --git a/tests/js/modules/python-cjs-module.py b/tests/js/modules/python-cjs-module.py new file mode 100644 index 00000000..71c892de --- /dev/null +++ b/tests/js/modules/python-cjs-module.py @@ -0,0 +1,9 @@ +# @file python-cjs-module.py +# A CommonJS Module written Python. +# @author Wes Garland, wes@distributive.network +# @date Jun 2023 + +def helloWorld(): + print('hello, world!') + +exports['helloWorld'] = helloWorld From 89a78ca1d780647ee7f03bed6001b9be93cd4a86 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 24 Jun 2023 10:49:34 -0400 Subject: [PATCH 38/60] Add isCompilableUnit to support writing JS REPLs --- pmjs | 29 ++++++++++++++++++++---- src/modules/pythonmonkey/pythonmonkey.cc | 26 ++++++++++++++++++--- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/pmjs b/pmjs index f868c028..de60f64a 100755 --- a/pmjs +++ b/pmjs @@ -8,10 +8,17 @@ sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonke import pythonmonkey as pm globalThis = pm.eval("globalThis;"); -globalThis.pmEval = pm.eval; +globalThis.require = pm.createRequire(os.getcwd() + '/dummy') +globalThis.python.isCompilableUnit = pm.isCompilableUnit -globalThis = pm.eval("globalThis;"); -globalThis.python = {'exit': exit }; +def write(buf: str): + print('buf', end="") +globalThis.python.write = write + +# Wrap exit; pythonmonkey can't pass objects of type Quitter +def quit(arg): + exit(arg) +globalThis.python.quit = quit pm.eval(""" const cmds = { @@ -28,6 +35,8 @@ Press Ctrl+C to abort current expression, Ctrl+D to exit the REPL` globalThis.replEval = function replEval(line) { + const lineBuf = []; + try { if (line[0] === '.') @@ -35,6 +44,16 @@ globalThis.replEval = function replEval(line) if (line === String.fromCharCode(4)) python.exit(0); + lineBuf.push(line); + line = lineBuf.join(String.fromCharCode(10)); + if (python.isCompilableUnit(line)) + lineBuf.length = 0; + else + { + python.write('... '); + return; /* Back to REPL for more of JS expression */ + } + const result = eval(line); switch (typeof result) { @@ -46,8 +65,10 @@ globalThis.replEval = function replEval(line) if (result instanceof Error) return result.stack || result.message; return JSON.stringify(result); - default: + case 'function': return String(result); + default: + return `${result}`; } } catch(error) diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index cf8c1031..bc8bd49a 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -185,7 +185,7 @@ static PyObject *eval(PyObject *self, PyObject *args) { } if (evalOptions && !PyDict_Check(evalOptions)) { - PyErr_SetString(PyExc_TypeError, "pythonmonkey.eval expects a dict as its (option) second argument"); + PyErr_SetString(PyExc_TypeError, "pythonmonkey.eval expects a dict as its (optional) second argument"); return NULL; } @@ -246,8 +246,28 @@ static PyObject *eval(PyObject *self, PyObject *args) { } } +static PyObject *isCompilableUnit(PyObject *self, PyObject *args) { + StrType *buffer = new StrType(PyTuple_GetItem(args, 0)); + const char *bufferUtf8; + bool compilable; + + if (!PyUnicode_Check(buffer->getPyObject())) { + PyErr_SetString(PyExc_TypeError, "pythonmonkey.eval expects a string as its first argument"); + return NULL; + } + + bufferUtf8 = buffer->getValue(); + compilable = JS_Utf8BufferIsCompilableUnit(GLOBAL_CX, *global, bufferUtf8, strlen(bufferUtf8)); + + if (compilable) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + PyMethodDef PythonMonkeyMethods[] = { {"eval", eval, METH_VARARGS, "Javascript evaluator in Python"}, + {"isCompilableUnit", isCompilableUnit, METH_VARARGS, "Hint if a string might be compilable Javascript"}, {"collect", collect, METH_VARARGS, "Calls the spidermonkey garbage collector"}, {"asUCS4", asUCS4, METH_VARARGS, "Expects a python string in UTF16 encoding, and returns a new equivalent string in UCS4. Undefined behaviour if the string is not in UTF16."}, {NULL, NULL, 0, NULL} @@ -288,9 +308,9 @@ static bool setTimeout(JSContext *cx, unsigned argc, JS::Value *vp) { // Wrap the job function into a bound function with the given additional arguments // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/bind JS::RootedVector bindArgs(cx); - bindArgs.append(JS::ObjectValue(**thisv)); + (void)bindArgs.append(JS::ObjectValue(**thisv)); /** @todo XXXwg handle return value */ for (size_t j = 2; j < args.length(); j++) { - bindArgs.append(args[j]); + (void)bindArgs.append(args[j]); /** @todo XXXwg handle return value */ } JS::RootedObject jobArgObj = JS::RootedObject(cx, &jobArgVal.toObject()); JS_CallFunctionName(cx, jobArgObj, "bind", JS::HandleValueArray(bindArgs), jobArg); // jobArg = jobArg.bind(thisv, ...bindArgs) From e3f8e8f9e3c5e54a893a06b134016b2a3a45e548 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 24 Jun 2023 11:09:00 -0400 Subject: [PATCH 39/60] pmjs - exit REPL when ^D --- pmjs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pmjs b/pmjs index de60f64a..15083576 100755 --- a/pmjs +++ b/pmjs @@ -7,19 +7,16 @@ import sys, os, cmd sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm -globalThis = pm.eval("globalThis;"); -globalThis.require = pm.createRequire(os.getcwd() + '/dummy') +globalThis = pm.eval("globalThis;") +pm.createRequire(__file__)('./pmjs-require') + globalThis.python.isCompilableUnit = pm.isCompilableUnit +globalThis.python.exit = sys.exit def write(buf: str): - print('buf', end="") + print(buf, end="") globalThis.python.write = write -# Wrap exit; pythonmonkey can't pass objects of type Quitter -def quit(arg): - exit(arg) -globalThis.python.quit = quit - pm.eval(""" const cmds = { help: function help() { @@ -41,11 +38,11 @@ globalThis.replEval = function replEval(line) { if (line[0] === '.') return cmds[line.slice(1)](); - if (line === String.fromCharCode(4)) - python.exit(0); - + if (line === 'EOF') + python.exit(''); lineBuf.push(line); line = lineBuf.join(String.fromCharCode(10)); + if (python.isCompilableUnit(line)) lineBuf.length = 0; else From 7a7477c6eac018f7ce42f7baebd67b4538e80e24 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sat, 24 Jun 2023 11:21:29 -0400 Subject: [PATCH 40/60] Add test coverage for isCompilableUnit --- tests/js/is-compilable-unit.simple | 16 ++++++++++++++++ tests/js/modules/vm-tools.py | 12 ++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/js/is-compilable-unit.simple create mode 100644 tests/js/modules/vm-tools.py diff --git a/tests/js/is-compilable-unit.simple b/tests/js/is-compilable-unit.simple new file mode 100644 index 00000000..46de5c37 --- /dev/null +++ b/tests/js/is-compilable-unit.simple @@ -0,0 +1,16 @@ +/** + * @file is-compilabe-unit.simple + * Simple test which ensures that pm.isCompilableUnit works from JS as expected + * written in Python instead of JavaScript + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ + +const { isCompilableUnit } = require('./modules/vm-tools'); + +if (isCompilableUnit('()=>')) + throw new Error('isCompilableUnit lied about ()=>'); + +if (!isCompilableUnit('123')) + throw new Error('isCompilableUnit lied about 123'); + diff --git a/tests/js/modules/vm-tools.py b/tests/js/modules/vm-tools.py new file mode 100644 index 00000000..6082a5cf --- /dev/null +++ b/tests/js/modules/vm-tools.py @@ -0,0 +1,12 @@ +# @file vm-tools.py +# A CommonJS Module written Python which feeds some Python- and JS-VM-related methods +# back up to JS so we can use them during tests. +# @author Wes Garland, wes@distributive.network +# @date Jun 2023 + +import sys, os +sys.path.append(os.path.dirname(__file__) + '/../../../python') # location of pythonmonkey module +import pythonmonkey as pm + +exports['isCompilableUnit'] = pm.isCompilableUnit +print('MODULE LOADED') From e3e7ff4f40ca8c62d1e81e729a1bcbca1bbdd083 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sun, 25 Jun 2023 01:03:25 -0400 Subject: [PATCH 41/60] pmjs - add support for multi-line statements, result colourization, improved evaluation --- pmjs | 89 +++++++++++++++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/pmjs b/pmjs index 15083576..5124eac1 100755 --- a/pmjs +++ b/pmjs @@ -3,21 +3,14 @@ # @author Wes Garland, wes@distributive.network # @date June 2023 -import sys, os, cmd +import sys, os, readline sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm globalThis = pm.eval("globalThis;") pm.createRequire(__file__)('./pmjs-require') -globalThis.python.isCompilableUnit = pm.isCompilableUnit -globalThis.python.exit = sys.exit - -def write(buf: str): - print(buf, end="") -globalThis.python.write = write - -pm.eval(""" +pm.eval("""'use strict'; const cmds = { help: function help() { return '' + @@ -30,42 +23,42 @@ Press Ctrl+C to abort current expression, Ctrl+D to exit the REPL` }, }; -globalThis.replEval = function replEval(line) +globalThis.replCmd = function replCmd(line) +{ + return cmds[line.slice(1)](); +} + +function colour(colourCode, val) { - const lineBuf = []; + const esc=String.fromCharCode(27); + return `${esc}[${colourCode}m${val}${esc}[0m` +} +globalThis.replEval = function replEval(line) +{ + const indirectEval = eval; try { - if (line[0] === '.') - return cmds[line.slice(1)](); - if (line === 'EOF') - python.exit(''); - lineBuf.push(line); - line = lineBuf.join(String.fromCharCode(10)); - - if (python.isCompilableUnit(line)) - lineBuf.length = 0; - else - { - python.write('... '); - return; /* Back to REPL for more of JS expression */ - } - - const result = eval(line); + const result = indirectEval(`${line}`); switch (typeof result) { + case 'undefined': + return colour(90, result); + case 'function': + return colour(36, result); case 'string': - case 'number': + return colour(32, `'${result}'`); case 'boolean': - return result; + case 'number': + return colour(33, result); case 'object': + if (result instanceof Date) + return colour(35, result.toISOString()); if (result instanceof Error) return result.stack || result.message; return JSON.stringify(result); - case 'function': - return String(result); default: - return `${result}`; + return colour(31, `${result}`); } } catch(error) @@ -75,15 +68,25 @@ globalThis.replEval = function replEval(line) } """); -class JSShell(cmd.Cmd): - intro = 'Welcome to PythonMonkey v' + pm.__version__ +'.\nType ".help" for more information.'; - - prompt = '> ' - - def parseline(self, line): - print(globalThis.replEval(line)) - dummyRet = cmd.Cmd.parseline(self, '') - return dummyRet +readline.parse_and_bind('set editing-mode emacs') +print('Welcome to PythonMonkey v' + pm.__version__ +'.') +print('Type ".help" for more information.') -if __name__ == '__main__': - JSShell().cmdloop() +while True: + try: + line = input('> ') + if (line[0] == '.'): + print(globalThis.replCmd(line)) + continue + if (pm.isCompilableUnit(line)): + print(globalThis.replEval(line)) + else: + while True: + more = input('... ') + line = line + '\n' + more + if (pm.isCompilableUnit(line)): + print(globalThis.replEval(line)) + break + except EOFError: + print() + sys.exit(0) From 70c37e895730dd219ef2da2c35747348909fc868 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sun, 25 Jun 2023 02:02:41 -0400 Subject: [PATCH 42/60] pmjs - improve Error output --- pmjs | 83 ++++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/pmjs b/pmjs index 5124eac1..2dbce623 100755 --- a/pmjs +++ b/pmjs @@ -3,11 +3,12 @@ # @author Wes Garland, wes@distributive.network # @date June 2023 -import sys, os, readline +import sys, os, readline, signal sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm globalThis = pm.eval("globalThis;") +globalThis.python.write = sys.stdout.write pm.createRequire(__file__)('./pmjs-require') pm.eval("""'use strict'; @@ -34,59 +35,91 @@ function colour(colourCode, val) return `${esc}[${colourCode}m${val}${esc}[0m` } +globalThis.formatResult = function formatResult(result) +{ + switch (typeof result) + { + case 'undefined': + return colour(90, result); + case 'function': + return colour(36, result); + case 'string': + return colour(32, `'${result}'`); + case 'boolean': + case 'number': + return colour(33, result); + case 'object': + if (result instanceof Date) + return colour(35, result.toISOString()); + if (result instanceof Error) + { + const error = result; + const LF = String.fromCharCode(10); + const stackEls = error.stack + .split(LF) + .filter(a => a.length > 0) + .map(a => ` ${a}`); + return (`${error.name}: ${error.message}` + LF + + stackEls[0] + LF + + colour(90, stackEls.slice(1).join(LF)) + ); + } + return JSON.stringify(result); + default: + return colour(31, `${result}`); + } +} + globalThis.replEval = function replEval(line) { const indirectEval = eval; try { const result = indirectEval(`${line}`); - switch (typeof result) - { - case 'undefined': - return colour(90, result); - case 'function': - return colour(36, result); - case 'string': - return colour(32, `'${result}'`); - case 'boolean': - case 'number': - return colour(33, result); - case 'object': - if (result instanceof Date) - return colour(35, result.toISOString()); - if (result instanceof Error) - return result.stack || result.message; - return JSON.stringify(result); - default: - return colour(31, `${result}`); - } + return formatResult(result); } catch(error) { - return error.stack || result.message; + return formatResult(error); } } """); readline.parse_and_bind('set editing-mode emacs') +histfile = os. path. expanduser('~/.pmjs_history') +if (os.path.exists(histfile)): + readline.read_history_file(histfile) + print('Welcome to PythonMonkey v' + pm.__version__ +'.') print('Type ".help" for more information.') +def quit(): + readline.write_history_file(histfile) + sys.exit(0) + +def showResult(line): + result = globalThis.replEval(line) + if (result != False): + print(result) + while True: try: line = input('> ') + + if (len(line) == 0): + continue if (line[0] == '.'): print(globalThis.replCmd(line)) continue if (pm.isCompilableUnit(line)): - print(globalThis.replEval(line)) + showResult(line) else: while True: more = input('... ') line = line + '\n' + more if (pm.isCompilableUnit(line)): - print(globalThis.replEval(line)) + showResult(line) break except EOFError: print() - sys.exit(0) + quit() From 1c2966c4b15371b9ed30e1669d93c0c88d083c25 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sun, 25 Jun 2023 02:29:56 -0400 Subject: [PATCH 43/60] pmjs - implement half-baked sigint handler; might need to escape readline to make fully-baked --- pmjs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pmjs b/pmjs index 2dbce623..71340d4a 100755 --- a/pmjs +++ b/pmjs @@ -97,13 +97,28 @@ def quit(): readline.write_history_file(histfile) sys.exit(0) -def showResult(line): +def show_result(line): result = globalThis.replEval(line) if (result != False): print(result) -while True: +got_sigint = 0 +building_line = False + +def sigint_handler(signum, frame): + global got_sigint + global building_line + got_sigint = got_sigint + 1 + if (got_sigint == 1 and building_line == False): + print('(To exit, press Ctrl+C again or Ctrl+D or type .exit)') + if (got_sigint > 1 and building_line == False): + quit() + +signal.signal(signal.SIGINT, sigint_handler) + +while got_sigint < 2: try: + building_line = False line = input('> ') if (len(line) == 0): @@ -112,14 +127,19 @@ while True: print(globalThis.replCmd(line)) continue if (pm.isCompilableUnit(line)): - showResult(line) + show_result(line) + got_sigint = 0 else: - while True: + building_line = True + got_sigint = 0 + while (got_sigint == 0): more = input('... ') line = line + '\n' + more if (pm.isCompilableUnit(line)): - showResult(line) + show_result(line) break + got_sigint = 0 + building_line = False except EOFError: print() quit() From a9a9e9b89ef16b83f62ac161036726bf2a6dc684 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sun, 25 Jun 2023 11:38:41 -0400 Subject: [PATCH 44/60] pmjs - improve code readability and command argument parsing --- pmjs | 97 +++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/pmjs b/pmjs index 71340d4a..2398969d 100755 --- a/pmjs +++ b/pmjs @@ -12,29 +12,59 @@ globalThis.python.write = sys.stdout.write pm.createRequire(__file__)('./pmjs-require') pm.eval("""'use strict'; -const cmds = { - help: function help() { - return '' + +const cmds = {}; + +cmds.help = function help() { + return '' + `.exit Exit the REPL .help Print this help message .load Load JS from a file into the REPL session .save Save all evaluated commands in this REPL session to a file Press Ctrl+C to abort current expression, Ctrl+D to exit the REPL` - }, }; -globalThis.replCmd = function replCmd(line) +cmds.exit = python.exit; + +/** + * Handle a .XXX command. Invokes function cmds[XXX], passing arguments that the user typed + * as the function arguments. The function arguments space-delimited arguments; arguments + * surrounded by quotes can include spaces, similar to how bash parses arguments. Argument + * parsing cribbed from stackoverflow user Tsuneo Yoshioka, question 4031900. + * + * @param {string} cmdLine the command the user typed, without the leading . + * @returns {string} to display + */ +globalThis.replCmd = function replCmd(cmdLine) { - return cmds[line.slice(1)](); + const cmdName = (cmdLine.match(/^[^ ]+/) || ['help'])[0]; + const args = cmdLine.slice(cmdName.length).trim(); + const argv = args.match(/\\\\?.|^$/g).reduce((p, c) => { + if (c === '"') + p.quote ^= 1; + else if (!p.quote && c === ' ') + p.a.push(''); + else + p.a[p.a.length-1] += c.replace(/\\\\(.)/,"$1"); + return p; + }, {a: ['']}).a; + + if (!cmds.hasOwnProperty(cmdName)) + return `Invalid REPL keyword`; + return cmds[cmdName].apply(null, argv); } +/** Return String(val) surrounded by appropriate ANSI escape codes to change the console text colour. */ function colour(colourCode, val) { const esc=String.fromCharCode(27); return `${esc}[${colourCode}m${val}${esc}[0m` } +/** + * Format result more intelligently than toString. Inspired by Node.js util.inspect, but far less + * capable. + */ globalThis.formatResult = function formatResult(result) { switch (typeof result) @@ -70,12 +100,15 @@ globalThis.formatResult = function formatResult(result) } } -globalThis.replEval = function replEval(line) +/** + * Evaluate a complete statement, built by the Python readline loop. + */ +globalThis.replEval = function replEval(statement) { const indirectEval = eval; try { - const result = indirectEval(`${line}`); + const result = indirectEval(`${statement}`); return formatResult(result); } catch(error) @@ -96,50 +129,54 @@ print('Type ".help" for more information.') def quit(): readline.write_history_file(histfile) sys.exit(0) - -def show_result(line): - result = globalThis.replEval(line) - if (result != False): - print(result) +globalThis.python.exit = quit got_sigint = 0 -building_line = False +building_statement = False +# Try to handle ^C by aborting the entry of the current line and quitting when double-struck. Note that +# does not currently work properly because there doesn't seem to be an easy way to abort data entry via +# input() via gnu readline in Python. def sigint_handler(signum, frame): global got_sigint - global building_line + global building_statement got_sigint = got_sigint + 1 - if (got_sigint == 1 and building_line == False): + if (got_sigint == 1 and building_statement == False): print('(To exit, press Ctrl+C again or Ctrl+D or type .exit)') - if (got_sigint > 1 and building_line == False): + if (got_sigint > 1 and building_statement == False): quit() - + readline.redisplay() signal.signal(signal.SIGINT, sigint_handler) +# Main Program Loop ##### +# +# Read lines entered by the user and collect them in a statement. Once the statement is a candiate for +# JavaScript evaluation (determined by pm.isCompilableUnit(), send it to replEval(). Statement beginning +# with a . are interpreted as REPL commands and sent to replCmd(). while got_sigint < 2: try: - building_line = False - line = input('> ') + building_statement = False + statement = input('> ') - if (len(line) == 0): + if (len(statement) == 0): continue - if (line[0] == '.'): - print(globalThis.replCmd(line)) + if (statement[0] == '.'): + print(globalThis.replCmd(statement[1:])) continue - if (pm.isCompilableUnit(line)): - show_result(line) + if (pm.isCompilableUnit(statement)): + print(globalThis.replEval(statement)) got_sigint = 0 else: - building_line = True + building_statement = True got_sigint = 0 while (got_sigint == 0): more = input('... ') - line = line + '\n' + more - if (pm.isCompilableUnit(line)): - show_result(line) + statement = statement + '\n' + more + if (pm.isCompilableUnit(statement)): + print(globalThis.replEval(statement)) break got_sigint = 0 - building_line = False + building_statement = False except EOFError: print() quit() From 7206801faca4f940508075561c6edce01bbef70a Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sun, 25 Jun 2023 12:52:21 -0400 Subject: [PATCH 45/60] Add basic smoke-test for CommonJS module loading --- peter-jr | 2 +- tests/js/load-cjs-module.simple | 10 ++++++++++ tests/js/modules/cjs-module.js | 13 +++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tests/js/load-cjs-module.simple create mode 100644 tests/js/modules/cjs-module.js diff --git a/peter-jr b/peter-jr index 5535bd09..39b83e68 100755 --- a/peter-jr +++ b/peter-jr @@ -78,7 +78,7 @@ grey() ( case "$ext" in "simple") - ./js-test-runner "$file" + "${topDir}/js-test-runner" "$file" exitCode="$?" ;; "bash") diff --git a/tests/js/load-cjs-module.simple b/tests/js/load-cjs-module.simple new file mode 100644 index 00000000..ee01061e --- /dev/null +++ b/tests/js/load-cjs-module.simple @@ -0,0 +1,10 @@ +/** + * @file load-cjs-module.js + * Simple smoke test which ensures that we can load a CommonJS Module + * @author Wes Garland, wes@distributive.network + * @date June 2023 + */ + +const { helloWorld } = require('./modules/cjs-module'); + +helloWorld(); diff --git a/tests/js/modules/cjs-module.js b/tests/js/modules/cjs-module.js new file mode 100644 index 00000000..e239dbab --- /dev/null +++ b/tests/js/modules/cjs-module.js @@ -0,0 +1,13 @@ +/** + * @file cjs-module.js + * A CommonJS Module written in JavaScript + * @author Wes Garland, wes@distributive.network + * @date Jun 2023 +*/ + +function helloWorld() +{ + console.log('hello, world!') +} + +exports.helloWorld = helloWorld; From e1fd22022890acf9bc83a142b87e9fdc0b47283f Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Sun, 25 Jun 2023 16:53:28 -0400 Subject: [PATCH 46/60] require.py - remove globalSet and propSet hacks, depends on ctx-module 1.0.12 --- python/pythonmonkey/package-lock.json | 12 +- python/pythonmonkey/package.json | 2 +- python/pythonmonkey/require.py | 156 +++++++++++++------------- 3 files changed, 88 insertions(+), 82 deletions(-) diff --git a/python/pythonmonkey/package-lock.json b/python/pythonmonkey/package-lock.json index 46cf7ab3..b66a51d6 100644 --- a/python/pythonmonkey/package-lock.json +++ b/python/pythonmonkey/package-lock.json @@ -179,18 +179,18 @@ } }, "ctx-module": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/ctx-module/-/ctx-module-1.0.9.tgz", - "integrity": "sha512-MN1mKVJI8xn4NbfqxCOqoTuoGJBs5SX7uLfqooWQxQFqMkoCFoaXGHSbZxOKccAO8SgWNn6gSxR8TsUxLsHp1g==", + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/ctx-module/-/ctx-module-1.0.12.tgz", + "integrity": "sha512-iaple0ZSdEOq5Wdx7/eh6VvVZ5Rm4IICzvEulDiXXER+eGMgUlaPBBPZsV3aPhyy5LJNVwA4pKSqc0hIBvP0sA==", "requires": { "buffer": "^6.0.3", "crypto-browserify": "^3.12.0" } }, "des.js": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", - "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz", + "integrity": "sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg==", "requires": { "inherits": "^2.0.1", "minimalistic-assert": "^1.0.0" diff --git a/python/pythonmonkey/package.json b/python/pythonmonkey/package.json index b5e51b46..71b0f8b6 100644 --- a/python/pythonmonkey/package.json +++ b/python/pythonmonkey/package.json @@ -21,6 +21,6 @@ }, "homepage": "https://github.com/Distributive-Network/PythonMonkey#readme", "dependencies": { - "ctx-module": "^1.0.9" + "ctx-module": "^1.0.12" } } diff --git a/python/pythonmonkey/require.py b/python/pythonmonkey/require.py index 21b64d6b..6dc16f21 100644 --- a/python/pythonmonkey/require.py +++ b/python/pythonmonkey/require.py @@ -32,13 +32,40 @@ sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm +# Add some python functions to the global python object for code in this file to use. globalThis = pm.eval("globalThis;"); +pm.eval("globalThis.python = { pythonMonkey: {} }"); globalThis.pmEval = pm.eval; +globalThis.python.print = print; +globalThis.python.getenv = os.getenv; +globalThis.python.pythonMonkey.dir = os.path.dirname(__file__); +globalThis.python.paths = ':'.join(sys.path); +pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays + +# bootstrap is effectively a scoping object which keeps us from polluting the global JS scope. +# The idea is that we hold a reference to the bootstrap object in Python-load, for use by the +# innermost code in ctx-module, without forcing ourselves to expose this minimalist code to +# userland-require. +bootstrap = pm.eval(""" +'use strict'; (function IIFE(python) { + +const bootstrap = { + modules: { + vm: { runInContext: eval }, + 'ctx-module': {}, + }, +} + +/* ctx-module needs require() when it loads that can find vm and fs */ +bootstrap.require = function bootstrapRequire(mid) +{ + if (bootstrap.modules.hasOwnProperty(mid)) + return bootstrap.modules[mid]; -pm.eval(""" -'use strict'; + throw new Error('module not found: ' + mid); +} -function runInContext(code, _unused_contextifiedObject, options) +bootstrap.modules.vm.runInContext_broken = function runInContext(code, _unused_contextifiedObject, options) { var evalOptions = {}; @@ -52,16 +79,12 @@ return pmEval(code, evalOptions); } -globalThis.python = { pythonMonkey: {} }; -globalThis.global = globalThis; -globalThis.vmModule = { runInContext: eval }; -globalThis.require = function outerRequire(mid) { - const module = globalThis[mid + 'Module']; - if (module) - return module; - throw new Error('module not found: ' + mid); -}; -globalThis.debugModule = function debug(selector) { +/** + * The debug module has as its exports a function which may, depending on the DEBUG env var, emit + * debugging statemnts to stdout. This is quick implementation of the node built-in. + */ +bootstrap.modules.debug = function debug(selector) +{ var idx, colour; const esc = String.fromCharCode(27); const noColour = `${esc}[0m`; @@ -93,33 +116,33 @@ /* no match => silent */ return (function debugDummy() {}); -}; - -function globalSet(name, prop) -{ - globalThis[name] = prop; } -function propSet(objName, propName, propValue) -{ - globalThis[objName] = globalThis[objName] || {}; globalThis[objName][propName] = propValue; -} - -""") - -# globalSet and propSet are work-arounds until PythonMonkey correctly proxies objects. -globalSet = pm.eval("globalSet"); -propSet = pm.eval("propSet") +/** + * The fs module is like the Node.js fs module, except it only implements exactly what the ctx-module + * module requires to load CommonJS modules. It is augmented heavily below by Python methods. + */ +bootstrap.modules.fs = { + constants: { S_IFDIR: 16384 }, + statSync: function statSync(filename) { + const ret = bootstrap.modules.fs.statSync_inner(filename); + if (ret) + return ret; + + const err = new Error('file not found: ' + filename); + err.code='ENOENT'; + throw err; + }, +}; -# Add some python functions to the global python object for code in this file to use. -propSet('python', 'print', print); -propSet('python', 'getenv', os.getenv); -propSet('python', 'paths', ':'.join(sys.path)); -pm.eval("python.paths = python.paths.split(':'); true"); # fix when pm supports arrays +/* Modules which will be available to all requires */ +bootstrap.builtinModules = { debug: bootstrap.modules.debug }; -globalThis.python.pythonMonkey.dir = os.path.dirname(__file__); +/* temp workaround for PythonMonkey bug */ +globalThis.bootstrap = bootstrap; -# Implement enough of require('fs') so that ctx-module can find/load files +return bootstrap; +})(globalThis.python)""") def statSync_inner(filename: str) -> Union[Dict[str, int], bool]: """ @@ -144,45 +167,25 @@ def readFileSync(filename, charset) -> str: with open(filename, "r", encoding=charset) as fileHnd: return fileHnd.read() -propSet('fsModule', 'statSync_inner', statSync_inner); -propSet('fsModule', 'readFileSync', readFileSync) -propSet('fsModule', 'existsSync', os.path.exists) -pm.eval("fsModule.constants = { S_IFDIR: 16384 }; true;") -pm.eval("""fsModule.statSync = -function statSync(filename) -{ - const ret = require('fs').statSync_inner(filename); - if (ret) - return ret; - - const err = new Error('file not found: ' + filename); - err.code='ENOENT'; - throw err; -}"""); - -# Read in ctx-module and invoke so that this file is the "main module" and the Python symbol require is -# now the corresponding CommonJS require() function. We use the globalThis as the module's exports -# because PythonMonkey current segfaults when return objects. Once that is fixed, we will pass moduleIIFE -# parameters which are a python implementation of top-level require(for fs, vm - see top) and an exports -# dict to decorate. +bootstrap.modules.fs.statSync_inner = statSync_inner +bootstrap.modules.fs.readFileSync = readFileSync +bootstrap.modules.fs.existsSync = os.path.exists +# Read ctx-module module from disk and invoke so that this file is the "main module" and ctx-module has +# require and exports symbols injected from the bootstrap object above. Current PythonMonkey bugs +# prevent us from injecting names properly so they are stolen from trail left behind in the global +# scope until that can be fixed. with open(os.path.dirname(__file__) + "/node_modules/ctx-module/ctx-module.js", "r") as ctxModuleSource: - moduleWrapper = pm.eval("""'use strict'; -(function moduleWrapper(require, exports) + initCtxModule = pm.eval("""'use strict'; +(function moduleWrapper_forCtxModule(broken_require, broken_exports) { - exports=exports || globalThis; - require=require || globalThis.require; + const require = bootstrap.require; + const exports = bootstrap.modules['ctx-module']; """ + ctxModuleSource.read() + """ }) """); - -# inject require and exports symbols as moduleWrapper arguments once jsObj->dict fixed so we don't have -# to use globalThis and pollute the global scope. -moduleWrapper() - -# __builtinModules should be a dict that we add built-in modules to in Python, then pass the same -# dict->jsObject in createRequire for every require we create. -pm.eval('const __builtinModules = {}; true'); +#broken initCtxModule(bootstrap.require, bootstrap.modules['ctx-module'].exports); +initCtxModule(); def load(filename: str) -> Dict: """ @@ -210,8 +213,7 @@ def load(filename: str) -> Dict: for key in dir(module): module_exports[key] = getattr(module, key) return module_exports - -propSet('python', 'load', load) +globalThis.python.load = load """ API - createRequire @@ -223,15 +225,19 @@ def load(filename: str) -> Dict: require = createRequire(__file__) require('./my-javascript-module') """ -createRequire: Callable = pm.eval("""( -function createRequire(filename) +def createRequire(filename): + createRequireInner = pm.eval("""'use strict';( +function createRequire(filename, bootstrap_broken) { + const bootstrap = globalThis.bootstrap; /** @bug PM-65 */ + const CtxModule = bootstrap.modules['ctx-module'].CtxModule; + function loadPythonModule(module, filename) { module.exports = python.load(filename); } - const module = new CtxModule(globalThis, filename, __builtinModules); + const module = new CtxModule(globalThis, filename, bootstrap.builtinModules); for (let path of python.paths) module.paths.push(path + '/node_modules'); if (module.require.path.length === 0) @@ -239,5 +245,5 @@ def load(filename: str) -> Dict: module.require.extensions['.py'] = loadPythonModule; return module.require; -} -)""") +})""") + return createRequireInner(filename) From 311b46ed22f638a599ecd9d9ce7b69ac82f87ad2 Mon Sep 17 00:00:00 2001 From: Hamada Gasmallah Date: Mon, 26 Jun 2023 10:51:19 -0400 Subject: [PATCH 47/60] chore: make pythonmonkey python files more pythonic and update tests --- python/pythonmonkey/__init__.py | 4 ++-- python/pythonmonkey/require.py | 3 +-- tests/python/reentrance_smoke.py | 17 ----------------- tests/python/test_pythonmonkey_eval.py | 2 +- tests/python/test_reentrance_smoke.py | 18 ++++++++++++++++++ 5 files changed, 22 insertions(+), 22 deletions(-) delete mode 100644 tests/python/reentrance_smoke.py create mode 100644 tests/python/test_reentrance_smoke.py diff --git a/python/pythonmonkey/__init__.py b/python/pythonmonkey/__init__.py index fd788b97..7e78b1c6 100644 --- a/python/pythonmonkey/__init__.py +++ b/python/pythonmonkey/__init__.py @@ -1,4 +1,4 @@ from .pythonmonkey import * from .require import * -import os -exec(open(os.path.dirname(__file__) + '/version.py').read()) +from .version import __version__ + diff --git a/python/pythonmonkey/require.py b/python/pythonmonkey/require.py index 6dc16f21..b2cd315e 100644 --- a/python/pythonmonkey/require.py +++ b/python/pythonmonkey/require.py @@ -29,8 +29,7 @@ from importlib import machinery from importlib import util -sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module -import pythonmonkey as pm +from . import pythonmonkey as pm # Add some python functions to the global python object for code in this file to use. globalThis = pm.eval("globalThis;"); diff --git a/tests/python/reentrance_smoke.py b/tests/python/reentrance_smoke.py deleted file mode 100644 index 52dd56a4..00000000 --- a/tests/python/reentrance_smoke.py +++ /dev/null @@ -1,17 +0,0 @@ -# @file reentrance-smoke.py -# Basic smoke test which shows that JS->Python->JS->Python calls work. Failures are -# indicated by segfaults. -# @author Wes Garland, wes@distributive.network -# @date June 2023 - -import sys, os -sys.path.append(os.path.dirname(__file__) + '/../../python') # location of pythonmonkey module -import pythonmonkey as pm - -globalThis = pm.eval("globalThis;"); -globalThis.pmEval = pm.eval; -globalThis.pyEval = eval; - -abc=(pm.eval("() => { return {def: pyEval('123')} };"))() -assert(abc['def'] == 123) -print(pm.eval("pmEval(`pyEval(\"'test passed'\")`)")) diff --git a/tests/python/test_pythonmonkey_eval.py b/tests/python/test_pythonmonkey_eval.py index eae889c5..54513663 100644 --- a/tests/python/test_pythonmonkey_eval.py +++ b/tests/python/test_pythonmonkey_eval.py @@ -842,7 +842,7 @@ async def coro_fn(x): # JS Promise to Python awaitable coercion assert 100 == await pm.eval("new Promise((r)=>{ r(100) })") assert 10010 == await pm.eval("Promise.resolve(10010)") - with pytest.raises(pm.SpiderMonkeyError, match="TypeError: .+ is not a constructor"): + with pytest.raises(pm.SpiderMonkeyError, match="^TypeError: (.|\\n)+ is not a constructor$"): await pm.eval("Promise.resolve")(10086) assert 10086 == await pm.eval("Promise.resolve.bind(Promise)")(10086) diff --git a/tests/python/test_reentrance_smoke.py b/tests/python/test_reentrance_smoke.py new file mode 100644 index 00000000..623c6bd4 --- /dev/null +++ b/tests/python/test_reentrance_smoke.py @@ -0,0 +1,18 @@ +# @file reentrance-smoke.py +# Basic smoke test which shows that JS->Python->JS->Python calls work. Failures are +# indicated by segfaults. +# @author Wes Garland, wes@distributive.network +# @date June 2023 + +import sys, os +import pythonmonkey as pm + + +def test_reentrance(): + globalThis = pm.eval("globalThis;"); + globalThis.pmEval = pm.eval; + globalThis.pyEval = eval; + + abc=(pm.eval("() => { return {def: pyEval('123')} };"))() + assert(abc['def'] == 123) + print(pm.eval("pmEval(`pyEval(\"'test passed'\")`)")) From da3a9006532bff8af40aa7960de7f08af2d1e960 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Mon, 26 Jun 2023 14:03:50 -0400 Subject: [PATCH 48/60] build_script.sh - stop growing version.py --- build_script.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/build_script.sh b/build_script.sh index 7c71b9d1..2fd6826a 100755 --- a/build_script.sh +++ b/build_script.sh @@ -22,12 +22,13 @@ cmake --build . -j$CPUS cd "${topDir}" cp -f build/src/pythonmonkey.so python/pythonmonkey/ -echo "# This file was generated via $0 by `id -un` on `date` - do not edit by hand!" -grep '^version' pyproject.toml \ -| head -1 \ -| sed 's/^version /__version__ /' \ ->> python/pythonmonkey/version.py +( + echo "# This file was generated via $0 by `id -un` on `hostname` at `date` - do not edit by hand!" + grep '^version' pyproject.toml \ + | head -1 \ + | sed 's/^version /__version__ /' \ +) > python/pythonmonkey/version.py # npm is used to load JS components, see package.json cd "${topDir}/python/pythonmonkey/" -npm i +npm i From 7270d4c76f10e6e97e6dbef14401ebe7696f6a32 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Mon, 26 Jun 2023 14:21:15 -0400 Subject: [PATCH 49/60] peter-jr - fix argv parsing, add test-failure exit code --- peter-jr | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/peter-jr b/peter-jr index 39b83e68..5adb1635 100755 --- a/peter-jr +++ b/peter-jr @@ -3,6 +3,13 @@ # @file peter-js # A simple test framework in the spirit of Peter (npmjs.com/packages/peter) for testing # basic PythonMonkey functionality. +# +# Exit Codes: +# 0 - all tests passed +# 1 - one or more tests failed +# 2 - no tests ran +# 3 - internal error +# # @author Wes Garland, wes@distributive.network # @date Jun 2023 # @@ -13,6 +20,7 @@ cd "$runDir" set -u set -o pipefail +peter_exit_code=2 if [ "${1:-}" = "-v" ]; then VERBOSE=1 @@ -22,12 +30,12 @@ else fi [ "${1:-}" ] || set "${topDir}/tests/js" -testLocations="$@" +testLocations=("$@") function panic() { echo "PANIC: $*" >&2 - exit 2 + exit 3 } TMP=`mktemp -d` @@ -97,11 +105,13 @@ grey() if [ "$exitCode" = "0" ]; then echo "$(green PASS)" + [ "${peter_exit_code}" = "2" ] && peter_exit_code=0 printf "\e[0;31m" [ "$VERBOSE" ] || cat "$stderr" printf "\e[0m" else echo "$(red FAIL)" + peter_exit_code=1 if [ ! "$VERBOSE" ]; then echo echo "$(grey --) $(yellow ${file}) $(grey vvvvvvvvvvvvvv)" @@ -113,4 +123,8 @@ grey() echo fi fi + (exit ${peter_exit_code}) done +peter_exit_code="$?" + +exit ${peter_exit_code} From 334fb0dfb99e9ebc9e45cc18ed7f497e0ee6d68e Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Mon, 26 Jun 2023 17:21:37 -0400 Subject: [PATCH 50/60] pmjs - police whitespace --- pmjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmjs b/pmjs index 2398969d..81c8a8f5 100755 --- a/pmjs +++ b/pmjs @@ -119,7 +119,7 @@ globalThis.replEval = function replEval(statement) """); readline.parse_and_bind('set editing-mode emacs') -histfile = os. path. expanduser('~/.pmjs_history') +histfile = os.path.expanduser('~/.pmjs_history') if (os.path.exists(histfile)): readline.read_history_file(histfile) From 7bc5a5fbe4d25c748aef7c4588dfdc4025f036b6 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Wed, 28 Jun 2023 10:11:59 -0400 Subject: [PATCH 51/60] add pmjs-require.js --- pmjs-require.js | 1 + 1 file changed, 1 insertion(+) create mode 100644 pmjs-require.js diff --git a/pmjs-require.js b/pmjs-require.js new file mode 100644 index 00000000..005dbd95 --- /dev/null +++ b/pmjs-require.js @@ -0,0 +1 @@ +globalThis.require = require From 4c11666be7f421e90faecca56eae58d0438c3356 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Wed, 28 Jun 2023 11:47:34 -0400 Subject: [PATCH 52/60] pmjs, require examples, etc - let poetry manage sys.path to find pythonmonkey.so --- examples/use-python-module.py | 2 -- examples/use-require.py | 2 -- js-test-runner | 1 - pmjs | 1 - tests/js/modules/vm-tools.py | 2 -- 5 files changed, 8 deletions(-) diff --git a/examples/use-python-module.py b/examples/use-python-module.py index 65018bd4..912a517a 100644 --- a/examples/use-python-module.py +++ b/examples/use-python-module.py @@ -3,8 +3,6 @@ # @author Wes Garland, wes@distributive.network # @date Jun 2023 -import sys, os -sys.path.append(os.path.dirname(__file__) + '/../python') # location of pythonmonkey module import pythonmonkey as pm require = pm.createRequire(__file__) diff --git a/examples/use-require.py b/examples/use-require.py index 6ef787a4..9f0b1b1a 100644 --- a/examples/use-require.py +++ b/examples/use-require.py @@ -3,8 +3,6 @@ # @author Wes Garland, wes@distributive.network # @date Jun 2023 -import sys, os -sys.path.append(os.path.dirname(__file__) + '/../python') # location of pythonmonkey module import pythonmonkey as pm require = pm.createRequire(__file__) diff --git a/js-test-runner b/js-test-runner index 2ef96847..a5490589 100755 --- a/js-test-runner +++ b/js-test-runner @@ -5,7 +5,6 @@ # @date Jun 2023 import sys, os -sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm # Main diff --git a/pmjs b/pmjs index 81c8a8f5..b23b3742 100755 --- a/pmjs +++ b/pmjs @@ -4,7 +4,6 @@ # @date June 2023 import sys, os, readline, signal -sys.path.append(os.path.dirname(__file__) + '/python') # location of pythonmonkey module import pythonmonkey as pm globalThis = pm.eval("globalThis;") diff --git a/tests/js/modules/vm-tools.py b/tests/js/modules/vm-tools.py index 6082a5cf..1ae78843 100644 --- a/tests/js/modules/vm-tools.py +++ b/tests/js/modules/vm-tools.py @@ -4,8 +4,6 @@ # @author Wes Garland, wes@distributive.network # @date Jun 2023 -import sys, os -sys.path.append(os.path.dirname(__file__) + '/../../../python') # location of pythonmonkey module import pythonmonkey as pm exports['isCompilableUnit'] = pm.isCompilableUnit From bfee9f2a26259223597b7a7db2eff1e0719ab855 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Wed, 28 Jun 2023 14:23:19 -0400 Subject: [PATCH 53/60] Add builtin_modules to wheel --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b64999a6..f544bdb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ include = [ # Linux and macOS "python/pythonmonkey/pythonmonkey.so", "python/pythonmonkey/libmozjs*", + "python/pythonmonkey/builtin_modules/**/*", # Windows "python/pythonmonkey/pythonmonkey.pyd", From 4087f3758cc8a2c0d09776ad7f69dfff680c4c3e Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Wed, 28 Jun 2023 16:41:41 -0400 Subject: [PATCH 54/60] Change cmake LINUX variables to UNIX so that the build builds --- CMakeLists.txt | 2 +- src/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 09070173..da771430 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) set(PYTHON_INCLUDE_DIR ${Python_INCLUDE_DIRS}) set(PYTHON_LIBRARIES ${Python_LIBRARIES}) message("Apple - Using Python:${Python_VERSION_MAJOR} - Libraries:${PYTHON_LIBRARIES} - IncludeDirs: ${PYTHON_INCLUDE_DIR}") - elseif(LINUX) + elseif(UNIX) find_package(Python 3.8 COMPONENTS Interpreter Development REQUIRED) set(Python_FIND_VIRTUALENV FIRST) # (require cmake >= v3.15 and this is the default) use the Python version configured by pyenv if available set(PYTHON_LIBRARIES ${Python_LIBRARIES}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c00f9af1..8b1c6ad9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,7 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) -if(LINUX) +if(UNIX) set(CMAKE_INSTALL_RPATH "$ORIGIN") else(APPLE) set(CMAKE_INSTALL_RPATH "@loader_path") @@ -46,4 +46,4 @@ endif() target_link_libraries(pythonmonkey ${SPIDERMONKEY_LIBRARIES}) target_include_directories(pythonmonkey PRIVATE ${PYTHON_INCLUDE_DIR}) -target_include_directories(pythonmonkey PRIVATE ${SPIDERMONKEY_INCLUDE_DIR}) \ No newline at end of file +target_include_directories(pythonmonkey PRIVATE ${SPIDERMONKEY_INCLUDE_DIR}) From 2b1dbf4dfb4eaac569b582471f48fac55eef01bf Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 30 Jun 2023 14:39:36 -0400 Subject: [PATCH 55/60] re-fix rpath for appple in CMakeLists.txt --- src/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8b1c6ad9..50e90d46 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,10 +1,10 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) -if(UNIX) - set(CMAKE_INSTALL_RPATH "$ORIGIN") -else(APPLE) +if(APPLE) set(CMAKE_INSTALL_RPATH "@loader_path") +else(UNIX) + set(CMAKE_INSTALL_RPATH "$ORIGIN") endif() list(APPEND PYTHONMONKEY_SOURCE_FILES ${SOURCE_FILES} "${CMAKE_SOURCE_DIR}/src/modules/pythonmonkey/pythonmonkey.cc") From 3e39ead0cdb1b5b7351eba7feb26c03a149a393b Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Fri, 30 Jun 2023 14:55:13 -0400 Subject: [PATCH 56/60] Remove cmake inputs from .gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index e38ff276..09640479 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,6 @@ dist *.so _spidermonkey_install/* *~ -Makefile -CMakeDoxyfile.in -CMakeDoxygenDefaults.cmake -CMakeCache.txt *.dylib *.dll *.pyd From 1066eebb4802d9f3673ce0f7b09a942785550f15 Mon Sep 17 00:00:00 2001 From: Tom Wenzheng Tang Date: Tue, 4 Jul 2023 16:44:32 -0400 Subject: [PATCH 57/60] docs: update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 0e763f3c..477f21f3 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,6 @@ this package to execute our complex `dcp-client` library, which is written in JS See also: examples/ -### Method 1 -After compiling the project in the `build/src` folder you will find a `.so` file named `pythonmonkey.so`. This is the shared object file that contains the pythonmonkey module. - ### Install from [PyPI](https://pypi.org/project/pythonmonkey/) > PythonMonkey is not release-ready yet. Our first public release is scheduled for mid-June 2023. From 90f981db2b0286ae4d36937e79ded17ae67ab615 Mon Sep 17 00:00:00 2001 From: Tom Wenzheng Tang Date: Tue, 4 Jul 2023 16:51:38 -0400 Subject: [PATCH 58/60] chore: fix src/CMakeLists.txt typo --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 50e90d46..69332a3a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ include_directories(${CMAKE_CURRENT_LIST_DIR}) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) if(APPLE) set(CMAKE_INSTALL_RPATH "@loader_path") -else(UNIX) +elseif(UNIX) set(CMAKE_INSTALL_RPATH "$ORIGIN") endif() From 04d33038f636dd89a5b5c06dddd71c866a0be0b5 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Tue, 4 Jul 2023 20:17:53 -0400 Subject: [PATCH 59/60] Update README.md to reflect roadmap advances --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 477f21f3..ef841ee7 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ PythonMonkey is a Mozilla [SpiderMonkey](https://firefox-source-docs.mozilla.org/js/index.html) JavaScript engine embedded into the Python VM, using the Python engine to provide the JS host environment. -This product is in an early stage, approximately 75% to MVP as of May 2023. It is under active development by Distributive Corp., +This product is in an early stage, approximately 80% to MVP as of May 2023. It is under active development by Distributive Corp., https://distributive.network/. External contributions and feedback are welcome and encouraged. The goal is to make writing code in either JS or Python a developer preference, with libraries commonly used in either language @@ -33,12 +33,12 @@ this package to execute our complex `dcp-client` library, which is written in JS - [done] NodeJS+NPM-compatible CommonJS module system - [done] Python strings coerce to JS strings - [done] Python intrinsics coerce to JS intrinsics -- Python dicts coerce to JS objects +- [done] Python dicts coerce to JS objects - [done] Python `require` function, returns a coerced dict of module exports - [done] Python functions coerce to JS function wrappers - [done] CommonJS module system .py loader, loads Python modules for use by JS - JS object->Python dict coercion supports inherited-property lookup (via __getattribute__?) -- Python host environment supplies event loop, including EventEmitter, setTimeout, etc. +- [done] Python host environment supplies event loop, including EventEmitter, setTimeout, etc. - Python host environment supplies XMLHttpRequest (other project?) - Python host environment supplies basic subsets of NodeJS's fs, path, process, etc, modules; as-needed by dcp-client (other project?) - Python TypedArrays coerce to JS TypeArrays @@ -65,7 +65,7 @@ this package to execute our complex `dcp-client` library, which is written in JS ## Using the library -See also: examples/ +See also: [examples/](examples/) ### Install from [PyPI](https://pypi.org/project/pythonmonkey/) @@ -101,8 +101,11 @@ Alternatively, you can build a `wheel` package by running `poetry build --format # Troubleshooting Tips +## REPL - pmjs +A basic JavaScript shell, `pmjs`, ships with PythonMonkey. + ## CommonJS (require) -If you are having trouble with the CommonJS require function, set DEBUG='ctx-module*' and you can see the filenames it tries to laod +If you are having trouble with the CommonJS require function, set environment variable DEBUG='ctx-module*' and you can see the filenames it tries to laod. ### Extra Symbols Loading the CommonJS subsystem declares some extra symbols which may be helpful in debugging - From 8445eeed0e3ecfdc18f2c1af5ed69720deb43594 Mon Sep 17 00:00:00 2001 From: Wes Garland Date: Tue, 4 Jul 2023 20:41:56 -0400 Subject: [PATCH 60/60] README - add examples section --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ef841ee7..6f89be25 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,6 @@ this package to execute our complex `dcp-client` library, which is written in JS ## Using the library -See also: [examples/](examples/) - ### Install from [PyPI](https://pypi.org/project/pythonmonkey/) > PythonMonkey is not release-ready yet. Our first public release is scheduled for mid-June 2023. @@ -99,6 +97,12 @@ Type "help", "copyright", "credits" or "license" for more information. Alternatively, you can build a `wheel` package by running `poetry build --format=wheel`, and install it by `pip install dist/*.whl`. +## Examples + +* [examples/](examples/) +* https://github.com/Distributive-Network/PythonMonkey-examples +* https://github.com/Distributive-Network/PythonMonkey-Crypto-JS-Fullstack-Example + # Troubleshooting Tips ## REPL - pmjs