Browse Source

Merge pull request #9158 from jtpio/debugger-murmur

Switch to a different murmurhash2 implementation to handle unicode characters
Afshin Taylor Darian 4 years ago
parent
commit
3bc0ab13d8

+ 2 - 2
packages/debugger/package.json

@@ -69,7 +69,6 @@
     "@lumino/signaling": "^1.4.3",
     "@lumino/widgets": "^1.14.0",
     "codemirror": "~5.57.0",
-    "murmurhash-js": "^1.0.0",
     "react": "~16.13.1",
     "vscode-debugprotocol": "^1.37.0"
   },
@@ -79,8 +78,8 @@
     "@jupyterlab/testutils": "^3.0.0-rc.4",
     "@types/codemirror": "^0.0.97",
     "@types/jest": "^26.0.10",
-    "@types/murmurhash-js": "1.0.3",
     "@types/react-dom": "~16.9.8",
+    "@types/text-encoding": "^0.0.35",
     "canvas": "^2.6.1",
     "jest": "^26.4.2",
     "jest-junit": "^11.1.0",
@@ -88,6 +87,7 @@
     "jest-summary-reporter": "^0.0.2",
     "rimraf": "~3.0.0",
     "shell-quote": "^1.7.2",
+    "text-encoding": "^0.7.0",
     "ts-jest": "^26.3.0",
     "typedoc": "0.17.0-3",
     "typescript": "~4.0.2"

+ 1 - 1
packages/debugger/src/config.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
-import { murmur2 } from 'murmurhash-js';
+import { murmur2 } from './hash';
 
 import { IDebugger } from './tokens';
 

+ 69 - 0
packages/debugger/src/hash.ts

@@ -0,0 +1,69 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+// Most of the implementation below is adapted from the following repository:
+// https://github.com/garycourt/murmurhash-js/blob/master/murmurhash2_gc.js
+// Which has the following MIT License:
+//
+// Copyright (c) 2011 Gary Court
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"),
+// to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+// and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+//  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// The implementation below uses case fallthrough as part of the algorithm.
+/* eslint-disable no-fallthrough */
+
+const m = 0x5bd1e995;
+const encoder = new TextEncoder();
+
+/**
+ * Calculate the murmurhash2 for a given string and seed.
+ *
+ * @param str The string to calculate the Murmur2 hash for.
+ * @param seed The seed.
+ *
+ * @returns The Murmurhash2 hash.
+ */
+export function murmur2(str: string, seed: number): number {
+  const data = encoder.encode(str);
+  let len = data.length;
+  let h = seed ^ len;
+  let i = 0;
+
+  while (len >= 4) {
+    let k =
+      (data[i] & 0xff) |
+      ((data[++i] & 0xff) << 8) |
+      ((data[++i] & 0xff) << 16) |
+      ((data[++i] & 0xff) << 24);
+
+    k = (k & 0xffff) * m + ((((k >>> 16) * m) & 0xffff) << 16);
+    k ^= k >>> 24;
+    k = (k & 0xffff) * m + ((((k >>> 16) * m) & 0xffff) << 16);
+
+    h = ((h & 0xffff) * m + ((((h >>> 16) * m) & 0xffff) << 16)) ^ k;
+
+    len -= 4;
+    ++i;
+  }
+
+  switch (len) {
+    case 3:
+      h ^= (data[i + 2] & 0xff) << 16;
+    case 2:
+      h ^= (data[i + 1] & 0xff) << 8;
+    case 1:
+      h ^= data[i] & 0xff;
+      h = (h & 0xffff) * m + ((((h >>> 16) * m) & 0xffff) << 16);
+  }
+
+  h ^= h >>> 13;
+  h = (h & 0xffff) * m + ((((h >>> 16) * m) & 0xffff) << 16);
+  h ^= h >>> 15;
+
+  return h >>> 0;
+}

+ 4 - 0
packages/debugger/test/config.spec.ts

@@ -1,6 +1,10 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
+import { init } from './utils';
+
+init();
+
 import { DebuggerConfig } from '../src/config';
 
 describe('DebuggerConfig', () => {

+ 4 - 0
packages/debugger/test/debugger.spec.ts

@@ -1,6 +1,10 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
+import { init } from './utils';
+
+init();
+
 import { act } from 'react-dom/test-utils';
 
 import { CodeEditorWrapper } from '@jupyterlab/codeeditor';

+ 4 - 0
packages/debugger/test/service.spec.ts

@@ -1,6 +1,10 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
+import { init } from './utils';
+
+init();
+
 import { Session, KernelSpecManager, KernelSpec } from '@jupyterlab/services';
 
 import {

+ 4 - 0
packages/debugger/test/session.spec.ts

@@ -1,6 +1,10 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
+import { init } from './utils';
+
+init();
+
 import { Session } from '@jupyterlab/services';
 
 import {

+ 18 - 0
packages/debugger/test/utils.ts

@@ -1,6 +1,8 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
+import encoding from 'text-encoding';
+
 // Utils from: https://github.com/jupyterlab/jupyterlab/blob/b1e2b83047421bf7196bec5f2a94d0616dcb2329/packages/services/test/utils.ts
 
 import { ServerConnection } from '@jupyterlab/services';
@@ -34,6 +36,22 @@ export const KERNELSPECS: JSONObject = {
   }
 };
 
+// stub for node global
+declare let global: any;
+
+/**
+ * This can be used by test modules that wouldn't otherwise import
+ * this file.
+ */
+export function init(): void {
+  if (typeof global !== 'undefined') {
+    global.TextEncoder = encoding.TextEncoder;
+  }
+}
+
+// Call init.
+init();
+
 /**
  * Create new server connection settings.
  *

+ 0 - 10
yarn.lock

@@ -3188,11 +3188,6 @@
   resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.0.tgz#69a23a3ad29caf0097f06eda59b361ee2f0639f6"
   integrity sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY=
 
-"@types/murmurhash-js@1.0.3":
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/@types/murmurhash-js/-/murmurhash-js-1.0.3.tgz#0e8c0a1db692c062ea4f7e1093fc1ba1d1839298"
-  integrity sha512-PxJwTlcFOBRPqv9pSoC3O1FpKN8GnM5hMJIkG6U3omH8b4GAh28fO1c+TMR4oxj0BG43/ICbrIK3KBfzad2heg==
-
 "@types/node-fetch@^2.5.4":
   version "2.5.7"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.7.tgz#20a2afffa882ab04d44ca786449a276f9f6bbf3c"
@@ -11230,11 +11225,6 @@ multimatch@^3.0.0:
     arrify "^1.0.1"
     minimatch "^3.0.4"
 
-murmurhash-js@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/murmurhash-js/-/murmurhash-js-1.0.0.tgz#b06278e21fc6c37fa5313732b0412bcb6ae15f51"
-  integrity sha1-sGJ44h/Gw3+lMTcysEEry2rhX1E=
-
 mute-stream@0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab"