Merge pull request #549 from jtpio/murmur-unicode

Switch to a different murmurhash implementation to handle Unicode characters

README.md

@@ -21,7 +21,7 @@ A kernel with support for debugging is required to be able to use the debugger.
 It is generally recommended to create a new `conda` environment to install the dependencies:
 
 ```bash
-conda create -n jupyterlab-debugger -c conda-forge xeus-python=0.8.0 notebook=6 jupyterlab=2 ptvsd nodejs
+conda create -n jupyterlab-debugger -c conda-forge xeus-python=0.8.6 notebook=6 jupyterlab=2 ptvsd nodejs
 conda activate jupyterlab-debugger
 ```
 
@@ -49,7 +49,7 @@ Enable the debugger, set breakpoints and step into the code:
 
 ```bash
 # Create a new conda environment
-conda create -n jupyterlab-debugger -c conda-forge nodejs xeus-python=0.8.0 ptvsd jupyterlab=2
+conda create -n jupyterlab-debugger -c conda-forge nodejs xeus-python=0.8.6 ptvsd jupyterlab=2
 
 # Activate the conda environment
 conda activate jupyterlab-debugger

binder/environment.yml

@@ -7,4 +7,4 @@ dependencies:
 - nodejs
 - notebook=6
 - ptvsd
-- xeus-python=0.8.0
+- xeus-python=0.8.6

jest.config.js

@@ -12,7 +12,8 @@ let local = {
   transform: {
     '\\.(ts|tsx)?$': 'ts-jest',
     '\\.svg$': 'jest-raw-loader'
-  }
+  },
+  testEnvironment: './test/custom-env'
 };
 
 Object.keys(local).forEach(option => {

package.json

@@ -71,7 +71,6 @@
     "@lumino/datagrid": "^0.6.0",
     "@lumino/disposable": "^1.2.0",
     "@lumino/widgets": "^1.8.0",
-    "murmurhash-js": "^1.0.0",
     "vscode-debugprotocol": "^1.37.0"
   },
   "devDependencies": {
@@ -80,7 +79,6 @@
     "@jupyterlab/testutils": "^2.2.0",
     "@types/codemirror": "0.0.76",
     "@types/jest": "^24.0.17",
-    "@types/murmurhash-js": "1.0.3",
     "@types/react-dom": "^16.9.8",
     "@typescript-eslint/eslint-plugin": "^2.33.0",
     "@typescript-eslint/parser": "^2.33.0",

src/config.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
-import { murmur2 } from 'murmurhash-js';
+import { murmur2 } from './hash';
 
 import { IDebugger } from './tokens';
 

src/hash.ts

@@ -0,0 +1,69 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+// Most of the implementation below is adapted from the following repository:
+// https://github.com/garycourt/murmurhash-js/blob/master/murmurhash2_gc.js
+// Which has the following MIT License:
+//
+// Copyright (c) 2011 Gary Court
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+// and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+//  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// The implementation below uses case fallthrough as part of the algorithm.
+/* eslint-disable no-fallthrough */
+
+const m = 0x5bd1e995;
+const encoder = new TextEncoder();
+
+/**
+ * Calculate the murmurhash2 for a given string and seed.
+ *
+ * @param str The string to calculate the Murmur2 hash for.
+ * @param seed The seed.
+ *
+ * @returns The Murmurhash2 hash.
+ */
+export function murmur2(str: string, seed: number): number {
+  const data = encoder.encode(str);
+  let len = data.length;
+  let h = seed ^ len;
+  let i = 0;
+
+  while (len >= 4) {
+    let k =
+      (data[i] & 0xff) |
+      ((data[++i] & 0xff) << 8) |
+      ((data[++i] & 0xff) << 16) |
+      ((data[++i] & 0xff) << 24);
+
+    k = (k & 0xffff) * m + ((((k >>> 16) * m) & 0xffff) << 16);
+    k ^= k >>> 24;
+    k = (k & 0xffff) * m + ((((k >>> 16) * m) & 0xffff) << 16);
+
+    h = ((h & 0xffff) * m + ((((h >>> 16) * m) & 0xffff) << 16)) ^ k;
+
+    len -= 4;
+    ++i;
+  }
+
+  switch (len) {
+    case 3:
+      h ^= (data[i + 2] & 0xff) << 16;
+    case 2:
+      h ^= (data[i + 1] & 0xff) << 8;
+    case 1:
+      h ^= data[i] & 0xff;
+      h = (h & 0xffff) * m + ((((h >>> 16) * m) & 0xffff) << 16);
+  }
+
+  h ^= h >>> 13;
+  h = (h & 0xffff) * m + ((((h >>> 16) * m) & 0xffff) << 16);
+  h ^= h >>> 15;
+
+  return h >>> 0;
+}

test/custom-env.js

@@ -0,0 +1,16 @@
+// See: https://github.com/facebook/jest/issues/9983
+
+const Environment = require('jest-environment-jsdom');
+
+module.exports = class CustomTestEnvironment extends Environment {
+  /**
+   *
+   */
+  async setup() {
+    await super.setup();
+    if (typeof this.global.TextEncoder === 'undefined') {
+      const { TextEncoder } = require('util');
+      this.global.TextEncoder = TextEncoder;
+    }
+  }
+};

yarn.lock

@@ -1791,11 +1791,6 @@
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
   integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
 
-"@types/murmurhash-js@1.0.3":
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/@types/murmurhash-js/-/murmurhash-js-1.0.3.tgz#0e8c0a1db692c062ea4f7e1093fc1ba1d1839298"
-  integrity sha512-PxJwTlcFOBRPqv9pSoC3O1FpKN8GnM5hMJIkG6U3omH8b4GAh28fO1c+TMR4oxj0BG43/ICbrIK3KBfzad2heg==
-
 "@types/node@*":
   version "14.0.23"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.0.23.tgz#676fa0883450ed9da0bb24156213636290892806"
@@ -5260,11 +5255,6 @@ ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
-murmurhash-js@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/murmurhash-js/-/murmurhash-js-1.0.0.tgz#b06278e21fc6c37fa5313732b0412bcb6ae15f51"
-  integrity sha1-sGJ44h/Gw3+lMTcysEEry2rhX1E=
-
 mute-stream@0.0.8:
   version "0.0.8"
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"