diff --git a/src/doc-v2.js b/src/doc-v2.js index ebb18e0..c8c2a1f 100644 --- a/src/doc-v2.js +++ b/src/doc-v2.js @@ -470,7 +470,7 @@ var doc = function ( docData, addons ) { cv.l2NormIndex = docData.wordVectors.l2NormIndex; cv.wordIndex = docData.wordVectors.wordIndex; cv.dimensions = docData.wordVectors.dimensions; - cv.unkVector = docData.wordVectors.unkVector; + cv.unkVector = docData.wordVectors.unkVector.slice( 0 ); // Following properties will be determined on the basis of the context. cv.size = 0; cv.words = []; @@ -487,12 +487,12 @@ var doc = function ( docData, addons ) { .out( its.lemma ) .map( ( t ) => t.toLowerCase() ); - for ( let i = 0; i < docTokens.length; i += 1 ) cv.vectors[ docTokens[ i ] ] = awvs[ docTokens[ i ] ] || cv.unkVector; - for ( let i = 0; i < docTokensLemma.length; i += 1 ) cv.vectors[ docTokensLemma[ i ] ] = awvs[ docTokensLemma[ i ] ] || cv.unkVector; + for ( let i = 0; i < docTokens.length; i += 1 ) cv.vectors[ docTokens[ i ] ] = ( awvs[ docTokens[ i ] ] || cv.unkVector ).slice( 0 ); + for ( let i = 0; i < docTokensLemma.length; i += 1 ) cv.vectors[ docTokensLemma[ i ] ] = ( awvs[ docTokensLemma[ i ] ] || cv.unkVector ).slice( 0 ); for ( let i = 0; i < specificWordVectors.length; i += 1 ) { - const spWord = specificWordVectors[ i ].toString().trim(); + const spWord = ( specificWordVectors[ i ] ) ? specificWordVectors[ i ].toString().trim() : false; if ( spWord ) - cv.vectors[ specificWordVectors[ i ] ] = awvs[ specificWordVectors[ i ] ] || cv.unkVector; + cv.vectors[ specificWordVectors[ i ] ] = ( awvs[ specificWordVectors[ i ] ] || cv.unkVector ).slice( 0 ); } if ( similarWordVectors ) { @@ -532,7 +532,8 @@ var doc = function ( docData, addons ) { // Update contextual vectors using the list of similar words; also update their size. for ( let i = 0; i < similarWords.length; i += 1 ) { if ( cv.vectors[ similarWords[ i ] ] === undefined ) { - cv.vectors[ similarWords[ i ] ] = awvs[ similarWords[ i ] ] || cv.unkVector; + // Similar word must exist in `awvs`. + cv.vectors[ similarWords[ i ] ] = awvs[ similarWords[ i ] ].slice( 0 ); cv.size += 1; } } @@ -543,7 +544,7 @@ var doc = function ( docData, addons ) { for ( let i = 0; cv.size < wordVectorsLimit; i += 1 ) { const word = docData.wordVectors.words[ i ]; if ( !cv.vectors[ word ] ) { - cv.vectors[ word ] = awvs[ word ]; + cv.vectors[ word ] = awvs[ word ].slice( 0 ); cv.size += 1; } } @@ -556,6 +557,7 @@ var doc = function ( docData, addons ) { // Update the word index entry inside every vector. for ( let i = 0; i < cv.size; i += 1 ) cv.vectors[ cv.words[ i ] ][ cv.wordIndex ] = i; + return JSON.stringify( cv ); }; // contextualVectors() diff --git a/test/contextual-vectors-specs.js b/test/contextual-vectors-specs.js index f350fc0..cacd732 100644 --- a/test/contextual-vectors-specs.js +++ b/test/contextual-vectors-specs.js @@ -51,6 +51,14 @@ describe( 'contextual vectors', function () { const jsonWithFF0E = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":2,"words":["is","it"],"vectors":{"it":[-0.30664,0.16821,0.98511,-0.33606,-0.2416,0.16186,-0.053496,0.4301,0.57342,-0.071569,0.36101,0.26729,0.27789,-0.072268,0.13838,-0.26714,0.12999,0.22949,-0.18311,0.50163,0.44921,-0.020821,0.42642,-0.068762,0.40337,0.095198,-0.31944,-0.54651,-0.13345,-0.56511,-0.20975,1.1592,-0.194,0.19828,-0.11924,0.41781,0.0068383,-0.20537,-0.53375,-0.52225,-0.38227,-0.0065833,0.14265,-0.42502,-0.3115,0.0027352,0.75093,-0.48218,-0.18595,-0.77104,-0.046406,-0.06914,0.41688,1.3235,-0.81742,-3.3998,-0.11307,-0.34123,2.0775,0.61369,0.14792,0.93753,-0.10138,0.28426,0.97899,-0.32335,0.63697,0.58308,0.2282,-0.31696,0.21061,-0.6506,0.21653,-0.24347,0.55519,-0.34351,-0.095093,-0.14715,-1.2876,0.3931,0.30163,-0.21767,-1.1146,0.51349,-1.341,-0.30381,0.32499,-0.45236,-0.17743,-0.048504,-0.12178,-0.42108,-0.40327,0.038452,-0.36084,0.037738,-0.21885,-0.38775,0.36916,0.54521,6.1432065,1],"is":[-0.54264,0.41476,1.0322,-0.40244,0.46691,0.21816,-0.074864,0.47332,0.080996,-0.22079,-0.12808,-0.1144,0.50891,0.11568,0.028211,-0.3628,0.43823,0.047511,0.20282,0.49857,-0.10068,0.13269,0.16972,0.11653,0.31355,0.25713,0.092783,-0.56826,-0.52975,-0.051456,-0.67326,0.92533,0.2693,0.22734,0.66365,0.26221,0.19719,0.2609,0.18774,-0.3454,-0.42635,0.13975,0.56338,-0.56907,0.12398,-0.12894,0.72484,-0.26105,-0.26314,-0.43605,0.078908,-0.84146,0.51595,1.3997,-0.7646,-3.1453,-0.29202,-0.31247,1.5129,0.52435,0.21456,0.42452,-0.088411,-0.17805,1.1876,0.10579,0.76571,0.21914,0.35824,-0.11636,0.093261,-0.62483,-0.21898,0.21796,0.74056,-0.43735,0.14343,0.14719,-1.1605,-0.050508,0.12677,-0.014395,-0.98676,-0.091297,-1.2054,-0.11974,0.047847,-0.54001,0.52457,-0.70963,-0.32528,-0.1346,-0.41314,0.33435,-0.0072412,0.32253,-0.044219,-1.2969,0.76217,0.46349,5.97709161,0]}}'; const jsonWithTT0E = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":5,"words":["is","it","be","not","this"],"vectors":{"it":[-0.30664,0.16821,0.98511,-0.33606,-0.2416,0.16186,-0.053496,0.4301,0.57342,-0.071569,0.36101,0.26729,0.27789,-0.072268,0.13838,-0.26714,0.12999,0.22949,-0.18311,0.50163,0.44921,-0.020821,0.42642,-0.068762,0.40337,0.095198,-0.31944,-0.54651,-0.13345,-0.56511,-0.20975,1.1592,-0.194,0.19828,-0.11924,0.41781,0.0068383,-0.20537,-0.53375,-0.52225,-0.38227,-0.0065833,0.14265,-0.42502,-0.3115,0.0027352,0.75093,-0.48218,-0.18595,-0.77104,-0.046406,-0.06914,0.41688,1.3235,-0.81742,-3.3998,-0.11307,-0.34123,2.0775,0.61369,0.14792,0.93753,-0.10138,0.28426,0.97899,-0.32335,0.63697,0.58308,0.2282,-0.31696,0.21061,-0.6506,0.21653,-0.24347,0.55519,-0.34351,-0.095093,-0.14715,-1.2876,0.3931,0.30163,-0.21767,-1.1146,0.51349,-1.341,-0.30381,0.32499,-0.45236,-0.17743,-0.048504,-0.12178,-0.42108,-0.40327,0.038452,-0.36084,0.037738,-0.21885,-0.38775,0.36916,0.54521,6.1432065,1],"is":[-0.54264,0.41476,1.0322,-0.40244,0.46691,0.21816,-0.074864,0.47332,0.080996,-0.22079,-0.12808,-0.1144,0.50891,0.11568,0.028211,-0.3628,0.43823,0.047511,0.20282,0.49857,-0.10068,0.13269,0.16972,0.11653,0.31355,0.25713,0.092783,-0.56826,-0.52975,-0.051456,-0.67326,0.92533,0.2693,0.22734,0.66365,0.26221,0.19719,0.2609,0.18774,-0.3454,-0.42635,0.13975,0.56338,-0.56907,0.12398,-0.12894,0.72484,-0.26105,-0.26314,-0.43605,0.078908,-0.84146,0.51595,1.3997,-0.7646,-3.1453,-0.29202,-0.31247,1.5129,0.52435,0.21456,0.42452,-0.088411,-0.17805,1.1876,0.10579,0.76571,0.21914,0.35824,-0.11636,0.093261,-0.62483,-0.21898,0.21796,0.74056,-0.43735,0.14343,0.14719,-1.1605,-0.050508,0.12677,-0.014395,-0.98676,-0.091297,-1.2054,-0.11974,0.047847,-0.54001,0.52457,-0.70963,-0.32528,-0.1346,-0.41314,0.33435,-0.0072412,0.32253,-0.044219,-1.2969,0.76217,0.46349,5.97709161,0],"be":[-0.46953,0.38432,0.54833,-0.63401,0.010133,0.11364,0.10612,0.58529,0.032302,-0.12274,0.030265,0.52662,1.0398,-0.082143,0.19118,-0.83784,0.50763,0.44488,-0.72604,0.036893,0.24211,-0.28878,0.33657,0.13656,0.14579,-0.13221,0.098428,-0.45276,-0.13029,0.015762,-0.010161,0.4967,-0.28461,0.29655,0.92979,0.42447,-0.082773,0.30438,-0.39219,-0.30585,-0.43201,-0.27333,0.24388,-0.58081,0.22679,0.027226,0.53473,-0.37527,-0.16119,-1.1235,0.12768,-0.69898,0.41341,1.2291,-0.41248,-2.5173,-0.15354,-0.043107,1.9111,0.80754,-0.14759,0.9609,-0.84267,0.084422,1.2616,-0.10938,0.54846,0.75255,-0.071289,-0.73987,0.094808,-0.97589,0.0078721,-0.23928,0.2882,-0.41516,0.034366,0.1197,-1.2142,-0.11306,0.52847,-0.42273,-0.93378,-0.046645,-2.122,-0.341,0.64229,-0.10097,-0.22875,-1.0776,-0.68044,-0.26372,-0.18331,-0.051632,-0.30836,0.066537,0.20422,-0.68914,0.4511,0.25125,6.2325238,2],"this":[-0.57058,0.44183,0.70102,-0.41713,-0.34058,0.02339,-0.071537,0.48177,-0.013121,0.16834,-0.13389,0.040626,0.15827,-0.44342,-0.019403,-0.009661,-0.046284,0.093228,-0.27331,0.2285,0.33089,-0.36474,0.078741,0.3585,0.44757,-0.2299,0.18077,-0.6265,0.053852,-0.29154,-0.4256,0.62903,0.14393,-0.046004,-0.21007,0.48879,-0.057698,0.37431,-0.030075,-0.34494,-0.29702,0.15095,0.28248,-0.16578,0.076131,-0.093016,0.79365,-0.60489,-0.18874,-1.0173,0.31962,-0.16344,0.54177,1.1725,-0.47875,-3.3842,-0.081301,-0.3528,1.8372,0.44516,-0.52666,0.99786,-0.32178,0.033462,1.1783,-0.072905,0.39737,0.26166,0.33111,-0.35629,-0.16558,-0.44382,-0.14183,-0.37976,0.28994,-0.029114,-0.35169,-0.27694,-1.344,0.19555,0.16887,0.040237,-0.80212,0.23366,-1.3837,-0.023132,0.085395,-0.74051,-0.073934,-0.58838,-0.085735,-0.10525,-0.51571,0.15038,-0.16694,-0.16372,-0.22702,-0.66102,0.47197,0.37253,5.85110816,4],"not":[-0.19104,0.17601,0.3692,-0.50323,-0.47561,0.15798,-0.11679,0.21052,0.32652,0.12194,0.090944,0.26089,0.76294,0.00069673,-0.050001,-0.44853,0.36239,0.56345,-0.68702,0.33237,0.31285,-0.14207,0.35327,-0.16426,-0.10693,0.077786,-0.17704,-0.92897,0.1468,-0.13585,0.25682,0.66019,-0.35569,0.21838,0.38173,0.54337,0.10197,0.3523,-0.2551,-0.15155,-0.67434,0.16903,0.16413,-0.53843,-0.17457,-0.28539,0.74044,-0.67533,-0.23382,-1.3599,0.30225,-0.14968,0.27043,1.1979,-0.29556,-2.5395,0.0010303,-0.26272,1.8303,0.80008,-0.35691,0.56578,-0.5504,0.070845,1.4275,0.09016,0.7842,0.7849,-0.33538,-0.65751,-0.20112,-1.0297,0.069195,-0.61272,0.11373,-0.19547,-0.21256,0.049763,-1.1619,-0.064512,0.53146,-0.47384,-0.68709,0.13024,-2.0899,-0.41346,0.30364,-0.00057448,-0.18833,-0.54779,-0.32058,-0.36704,-0.1474,-0.19044,-0.47712,0.048228,-0.26215,-0.5968,0.080843,0.27866,5.99066183,3]}}'; const jsonWithTF0E = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":3,"words":["is","it","be"],"vectors":{"it":[-0.30664,0.16821,0.98511,-0.33606,-0.2416,0.16186,-0.053496,0.4301,0.57342,-0.071569,0.36101,0.26729,0.27789,-0.072268,0.13838,-0.26714,0.12999,0.22949,-0.18311,0.50163,0.44921,-0.020821,0.42642,-0.068762,0.40337,0.095198,-0.31944,-0.54651,-0.13345,-0.56511,-0.20975,1.1592,-0.194,0.19828,-0.11924,0.41781,0.0068383,-0.20537,-0.53375,-0.52225,-0.38227,-0.0065833,0.14265,-0.42502,-0.3115,0.0027352,0.75093,-0.48218,-0.18595,-0.77104,-0.046406,-0.06914,0.41688,1.3235,-0.81742,-3.3998,-0.11307,-0.34123,2.0775,0.61369,0.14792,0.93753,-0.10138,0.28426,0.97899,-0.32335,0.63697,0.58308,0.2282,-0.31696,0.21061,-0.6506,0.21653,-0.24347,0.55519,-0.34351,-0.095093,-0.14715,-1.2876,0.3931,0.30163,-0.21767,-1.1146,0.51349,-1.341,-0.30381,0.32499,-0.45236,-0.17743,-0.048504,-0.12178,-0.42108,-0.40327,0.038452,-0.36084,0.037738,-0.21885,-0.38775,0.36916,0.54521,6.1432065,1],"is":[-0.54264,0.41476,1.0322,-0.40244,0.46691,0.21816,-0.074864,0.47332,0.080996,-0.22079,-0.12808,-0.1144,0.50891,0.11568,0.028211,-0.3628,0.43823,0.047511,0.20282,0.49857,-0.10068,0.13269,0.16972,0.11653,0.31355,0.25713,0.092783,-0.56826,-0.52975,-0.051456,-0.67326,0.92533,0.2693,0.22734,0.66365,0.26221,0.19719,0.2609,0.18774,-0.3454,-0.42635,0.13975,0.56338,-0.56907,0.12398,-0.12894,0.72484,-0.26105,-0.26314,-0.43605,0.078908,-0.84146,0.51595,1.3997,-0.7646,-3.1453,-0.29202,-0.31247,1.5129,0.52435,0.21456,0.42452,-0.088411,-0.17805,1.1876,0.10579,0.76571,0.21914,0.35824,-0.11636,0.093261,-0.62483,-0.21898,0.21796,0.74056,-0.43735,0.14343,0.14719,-1.1605,-0.050508,0.12677,-0.014395,-0.98676,-0.091297,-1.2054,-0.11974,0.047847,-0.54001,0.52457,-0.70963,-0.32528,-0.1346,-0.41314,0.33435,-0.0072412,0.32253,-0.044219,-1.2969,0.76217,0.46349,5.97709161,0],"be":[-0.46953,0.38432,0.54833,-0.63401,0.010133,0.11364,0.10612,0.58529,0.032302,-0.12274,0.030265,0.52662,1.0398,-0.082143,0.19118,-0.83784,0.50763,0.44488,-0.72604,0.036893,0.24211,-0.28878,0.33657,0.13656,0.14579,-0.13221,0.098428,-0.45276,-0.13029,0.015762,-0.010161,0.4967,-0.28461,0.29655,0.92979,0.42447,-0.082773,0.30438,-0.39219,-0.30585,-0.43201,-0.27333,0.24388,-0.58081,0.22679,0.027226,0.53473,-0.37527,-0.16119,-1.1235,0.12768,-0.69898,0.41341,1.2291,-0.41248,-2.5173,-0.15354,-0.043107,1.9111,0.80754,-0.14759,0.9609,-0.84267,0.084422,1.2616,-0.10938,0.54846,0.75255,-0.071289,-0.73987,0.094808,-0.97589,0.0078721,-0.23928,0.2882,-0.41516,0.034366,0.1197,-1.2142,-0.11306,0.52847,-0.42273,-0.93378,-0.046645,-2.122,-0.341,0.64229,-0.10097,-0.22875,-1.0776,-0.68044,-0.26372,-0.18331,-0.051632,-0.30836,0.066537,0.20422,-0.68914,0.4511,0.25125,6.2325238,2]}}'; + const jsonWithFF01 = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":3,"words":["is","it","apple"],"vectors":{"it":[-0.30664,0.16821,0.98511,-0.33606,-0.2416,0.16186,-0.053496,0.4301,0.57342,-0.071569,0.36101,0.26729,0.27789,-0.072268,0.13838,-0.26714,0.12999,0.22949,-0.18311,0.50163,0.44921,-0.020821,0.42642,-0.068762,0.40337,0.095198,-0.31944,-0.54651,-0.13345,-0.56511,-0.20975,1.1592,-0.194,0.19828,-0.11924,0.41781,0.0068383,-0.20537,-0.53375,-0.52225,-0.38227,-0.0065833,0.14265,-0.42502,-0.3115,0.0027352,0.75093,-0.48218,-0.18595,-0.77104,-0.046406,-0.06914,0.41688,1.3235,-0.81742,-3.3998,-0.11307,-0.34123,2.0775,0.61369,0.14792,0.93753,-0.10138,0.28426,0.97899,-0.32335,0.63697,0.58308,0.2282,-0.31696,0.21061,-0.6506,0.21653,-0.24347,0.55519,-0.34351,-0.095093,-0.14715,-1.2876,0.3931,0.30163,-0.21767,-1.1146,0.51349,-1.341,-0.30381,0.32499,-0.45236,-0.17743,-0.048504,-0.12178,-0.42108,-0.40327,0.038452,-0.36084,0.037738,-0.21885,-0.38775,0.36916,0.54521,6.1432065,1],"is":[-0.54264,0.41476,1.0322,-0.40244,0.46691,0.21816,-0.074864,0.47332,0.080996,-0.22079,-0.12808,-0.1144,0.50891,0.11568,0.028211,-0.3628,0.43823,0.047511,0.20282,0.49857,-0.10068,0.13269,0.16972,0.11653,0.31355,0.25713,0.092783,-0.56826,-0.52975,-0.051456,-0.67326,0.92533,0.2693,0.22734,0.66365,0.26221,0.19719,0.2609,0.18774,-0.3454,-0.42635,0.13975,0.56338,-0.56907,0.12398,-0.12894,0.72484,-0.26105,-0.26314,-0.43605,0.078908,-0.84146,0.51595,1.3997,-0.7646,-3.1453,-0.29202,-0.31247,1.5129,0.52435,0.21456,0.42452,-0.088411,-0.17805,1.1876,0.10579,0.76571,0.21914,0.35824,-0.11636,0.093261,-0.62483,-0.21898,0.21796,0.74056,-0.43735,0.14343,0.14719,-1.1605,-0.050508,0.12677,-0.014395,-0.98676,-0.091297,-1.2054,-0.11974,0.047847,-0.54001,0.52457,-0.70963,-0.32528,-0.1346,-0.41314,0.33435,-0.0072412,0.32253,-0.044219,-1.2969,0.76217,0.46349,5.97709161,0],"apple":[-0.5985,-0.46321,0.13001,-0.019576,0.4603,-0.3018,0.8977,-0.65634,0.66858,-0.49164,0.037557,-0.050889,0.6451,-0.53882,-0.3765,-0.04312,0.51384,0.17783,0.28596,0.92063,-0.49349,-0.48583,0.61321,0.78211,0.19254,0.91228,-0.055596,-0.12512,-0.65688,0.068557,0.55629,1.611,-0.0073642,-0.48879,0.45493,0.96105,-0.063369,0.17432,0.9814,-1.3125,-0.15801,-0.54301,-0.13888,-0.26146,-0.3691,0.26844,-0.24375,-0.19484,0.62583,-0.7377,0.38351,-0.75004,-0.39053,0.091498,-0.36591,-1.4715,-0.45228,0.2256,1.1412,-0.38526,-0.06716,0.57288,-0.39191,0.31302,-0.29235,-0.96157,0.15154,-0.21659,0.25103,0.096967,0.2843,1.4296,-0.50565,-0.51374,-0.47218,0.32036,0.023149,0.22623,-0.09725,0.82126,0.92599,-1.0086,-0.38639,0.86408,-1.206,-0.28528,0.2265,-0.38773,0.40879,0.59303,0.30769,0.83804,-0.63655,-0.44639,-0.43406,-0.79364,-0.28675,-0.034398,1.3431,0.34904,5.99219058,2]}}'; + const jsonWithTF01 = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":4,"words":["is","it","be","apple"],"vectors":{"it":[-0.30664,0.16821,0.98511,-0.33606,-0.2416,0.16186,-0.053496,0.4301,0.57342,-0.071569,0.36101,0.26729,0.27789,-0.072268,0.13838,-0.26714,0.12999,0.22949,-0.18311,0.50163,0.44921,-0.020821,0.42642,-0.068762,0.40337,0.095198,-0.31944,-0.54651,-0.13345,-0.56511,-0.20975,1.1592,-0.194,0.19828,-0.11924,0.41781,0.0068383,-0.20537,-0.53375,-0.52225,-0.38227,-0.0065833,0.14265,-0.42502,-0.3115,0.0027352,0.75093,-0.48218,-0.18595,-0.77104,-0.046406,-0.06914,0.41688,1.3235,-0.81742,-3.3998,-0.11307,-0.34123,2.0775,0.61369,0.14792,0.93753,-0.10138,0.28426,0.97899,-0.32335,0.63697,0.58308,0.2282,-0.31696,0.21061,-0.6506,0.21653,-0.24347,0.55519,-0.34351,-0.095093,-0.14715,-1.2876,0.3931,0.30163,-0.21767,-1.1146,0.51349,-1.341,-0.30381,0.32499,-0.45236,-0.17743,-0.048504,-0.12178,-0.42108,-0.40327,0.038452,-0.36084,0.037738,-0.21885,-0.38775,0.36916,0.54521,6.1432065,1],"is":[-0.54264,0.41476,1.0322,-0.40244,0.46691,0.21816,-0.074864,0.47332,0.080996,-0.22079,-0.12808,-0.1144,0.50891,0.11568,0.028211,-0.3628,0.43823,0.047511,0.20282,0.49857,-0.10068,0.13269,0.16972,0.11653,0.31355,0.25713,0.092783,-0.56826,-0.52975,-0.051456,-0.67326,0.92533,0.2693,0.22734,0.66365,0.26221,0.19719,0.2609,0.18774,-0.3454,-0.42635,0.13975,0.56338,-0.56907,0.12398,-0.12894,0.72484,-0.26105,-0.26314,-0.43605,0.078908,-0.84146,0.51595,1.3997,-0.7646,-3.1453,-0.29202,-0.31247,1.5129,0.52435,0.21456,0.42452,-0.088411,-0.17805,1.1876,0.10579,0.76571,0.21914,0.35824,-0.11636,0.093261,-0.62483,-0.21898,0.21796,0.74056,-0.43735,0.14343,0.14719,-1.1605,-0.050508,0.12677,-0.014395,-0.98676,-0.091297,-1.2054,-0.11974,0.047847,-0.54001,0.52457,-0.70963,-0.32528,-0.1346,-0.41314,0.33435,-0.0072412,0.32253,-0.044219,-1.2969,0.76217,0.46349,5.97709161,0],"be":[-0.46953,0.38432,0.54833,-0.63401,0.010133,0.11364,0.10612,0.58529,0.032302,-0.12274,0.030265,0.52662,1.0398,-0.082143,0.19118,-0.83784,0.50763,0.44488,-0.72604,0.036893,0.24211,-0.28878,0.33657,0.13656,0.14579,-0.13221,0.098428,-0.45276,-0.13029,0.015762,-0.010161,0.4967,-0.28461,0.29655,0.92979,0.42447,-0.082773,0.30438,-0.39219,-0.30585,-0.43201,-0.27333,0.24388,-0.58081,0.22679,0.027226,0.53473,-0.37527,-0.16119,-1.1235,0.12768,-0.69898,0.41341,1.2291,-0.41248,-2.5173,-0.15354,-0.043107,1.9111,0.80754,-0.14759,0.9609,-0.84267,0.084422,1.2616,-0.10938,0.54846,0.75255,-0.071289,-0.73987,0.094808,-0.97589,0.0078721,-0.23928,0.2882,-0.41516,0.034366,0.1197,-1.2142,-0.11306,0.52847,-0.42273,-0.93378,-0.046645,-2.122,-0.341,0.64229,-0.10097,-0.22875,-1.0776,-0.68044,-0.26372,-0.18331,-0.051632,-0.30836,0.066537,0.20422,-0.68914,0.4511,0.25125,6.2325238,2],"apple":[-0.5985,-0.46321,0.13001,-0.019576,0.4603,-0.3018,0.8977,-0.65634,0.66858,-0.49164,0.037557,-0.050889,0.6451,-0.53882,-0.3765,-0.04312,0.51384,0.17783,0.28596,0.92063,-0.49349,-0.48583,0.61321,0.78211,0.19254,0.91228,-0.055596,-0.12512,-0.65688,0.068557,0.55629,1.611,-0.0073642,-0.48879,0.45493,0.96105,-0.063369,0.17432,0.9814,-1.3125,-0.15801,-0.54301,-0.13888,-0.26146,-0.3691,0.26844,-0.24375,-0.19484,0.62583,-0.7377,0.38351,-0.75004,-0.39053,0.091498,-0.36591,-1.4715,-0.45228,0.2256,1.1412,-0.38526,-0.06716,0.57288,-0.39191,0.31302,-0.29235,-0.96157,0.15154,-0.21659,0.25103,0.096967,0.2843,1.4296,-0.50565,-0.51374,-0.47218,0.32036,0.023149,0.22623,-0.09725,0.82126,0.92599,-1.0086,-0.38639,0.86408,-1.206,-0.28528,0.2265,-0.38773,0.40879,0.59303,0.30769,0.83804,-0.63655,-0.44639,-0.43406,-0.79364,-0.28675,-0.034398,1.3431,0.34904,5.99219058,3]}}'; + const jsonWithTF51 = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":5,"words":["the","is","it","be","apple"],"vectors":{"it":[-0.30664,0.16821,0.98511,-0.33606,-0.2416,0.16186,-0.053496,0.4301,0.57342,-0.071569,0.36101,0.26729,0.27789,-0.072268,0.13838,-0.26714,0.12999,0.22949,-0.18311,0.50163,0.44921,-0.020821,0.42642,-0.068762,0.40337,0.095198,-0.31944,-0.54651,-0.13345,-0.56511,-0.20975,1.1592,-0.194,0.19828,-0.11924,0.41781,0.0068383,-0.20537,-0.53375,-0.52225,-0.38227,-0.0065833,0.14265,-0.42502,-0.3115,0.0027352,0.75093,-0.48218,-0.18595,-0.77104,-0.046406,-0.06914,0.41688,1.3235,-0.81742,-3.3998,-0.11307,-0.34123,2.0775,0.61369,0.14792,0.93753,-0.10138,0.28426,0.97899,-0.32335,0.63697,0.58308,0.2282,-0.31696,0.21061,-0.6506,0.21653,-0.24347,0.55519,-0.34351,-0.095093,-0.14715,-1.2876,0.3931,0.30163,-0.21767,-1.1146,0.51349,-1.341,-0.30381,0.32499,-0.45236,-0.17743,-0.048504,-0.12178,-0.42108,-0.40327,0.038452,-0.36084,0.037738,-0.21885,-0.38775,0.36916,0.54521,6.1432065,2],"is":[-0.54264,0.41476,1.0322,-0.40244,0.46691,0.21816,-0.074864,0.47332,0.080996,-0.22079,-0.12808,-0.1144,0.50891,0.11568,0.028211,-0.3628,0.43823,0.047511,0.20282,0.49857,-0.10068,0.13269,0.16972,0.11653,0.31355,0.25713,0.092783,-0.56826,-0.52975,-0.051456,-0.67326,0.92533,0.2693,0.22734,0.66365,0.26221,0.19719,0.2609,0.18774,-0.3454,-0.42635,0.13975,0.56338,-0.56907,0.12398,-0.12894,0.72484,-0.26105,-0.26314,-0.43605,0.078908,-0.84146,0.51595,1.3997,-0.7646,-3.1453,-0.29202,-0.31247,1.5129,0.52435,0.21456,0.42452,-0.088411,-0.17805,1.1876,0.10579,0.76571,0.21914,0.35824,-0.11636,0.093261,-0.62483,-0.21898,0.21796,0.74056,-0.43735,0.14343,0.14719,-1.1605,-0.050508,0.12677,-0.014395,-0.98676,-0.091297,-1.2054,-0.11974,0.047847,-0.54001,0.52457,-0.70963,-0.32528,-0.1346,-0.41314,0.33435,-0.0072412,0.32253,-0.044219,-1.2969,0.76217,0.46349,5.97709161,1],"be":[-0.46953,0.38432,0.54833,-0.63401,0.010133,0.11364,0.10612,0.58529,0.032302,-0.12274,0.030265,0.52662,1.0398,-0.082143,0.19118,-0.83784,0.50763,0.44488,-0.72604,0.036893,0.24211,-0.28878,0.33657,0.13656,0.14579,-0.13221,0.098428,-0.45276,-0.13029,0.015762,-0.010161,0.4967,-0.28461,0.29655,0.92979,0.42447,-0.082773,0.30438,-0.39219,-0.30585,-0.43201,-0.27333,0.24388,-0.58081,0.22679,0.027226,0.53473,-0.37527,-0.16119,-1.1235,0.12768,-0.69898,0.41341,1.2291,-0.41248,-2.5173,-0.15354,-0.043107,1.9111,0.80754,-0.14759,0.9609,-0.84267,0.084422,1.2616,-0.10938,0.54846,0.75255,-0.071289,-0.73987,0.094808,-0.97589,0.0078721,-0.23928,0.2882,-0.41516,0.034366,0.1197,-1.2142,-0.11306,0.52847,-0.42273,-0.93378,-0.046645,-2.122,-0.341,0.64229,-0.10097,-0.22875,-1.0776,-0.68044,-0.26372,-0.18331,-0.051632,-0.30836,0.066537,0.20422,-0.68914,0.4511,0.25125,6.2325238,3],"apple":[-0.5985,-0.46321,0.13001,-0.019576,0.4603,-0.3018,0.8977,-0.65634,0.66858,-0.49164,0.037557,-0.050889,0.6451,-0.53882,-0.3765,-0.04312,0.51384,0.17783,0.28596,0.92063,-0.49349,-0.48583,0.61321,0.78211,0.19254,0.91228,-0.055596,-0.12512,-0.65688,0.068557,0.55629,1.611,-0.0073642,-0.48879,0.45493,0.96105,-0.063369,0.17432,0.9814,-1.3125,-0.15801,-0.54301,-0.13888,-0.26146,-0.3691,0.26844,-0.24375,-0.19484,0.62583,-0.7377,0.38351,-0.75004,-0.39053,0.091498,-0.36591,-1.4715,-0.45228,0.2256,1.1412,-0.38526,-0.06716,0.57288,-0.39191,0.31302,-0.29235,-0.96157,0.15154,-0.21659,0.25103,0.096967,0.2843,1.4296,-0.50565,-0.51374,-0.47218,0.32036,0.023149,0.22623,-0.09725,0.82126,0.92599,-1.0086,-0.38639,0.86408,-1.206,-0.28528,0.2265,-0.38773,0.40879,0.59303,0.30769,0.83804,-0.63655,-0.44639,-0.43406,-0.79364,-0.28675,-0.034398,1.3431,0.34904,5.99219058,4],"the":[-0.038194,-0.24487,0.72812,-0.39961,0.083172,0.043953,-0.39141,0.3344,-0.57545,0.087459,0.28787,-0.06731,0.30906,-0.26384,-0.13231,-0.20757,0.33395,-0.33848,-0.31743,-0.48336,0.1464,-0.37304,0.34577,0.052041,0.44946,-0.46971,0.02628,-0.54155,-0.15518,-0.14107,-0.039722,0.28277,0.14393,0.23464,-0.31021,0.086173,0.20397,0.52624,0.17164,-0.082378,-0.71787,-0.41531,0.20335,-0.12763,0.41367,0.55187,0.57908,-0.33477,-0.36559,-0.54857,-0.062892,0.26584,0.30205,0.99775,-0.80481,-3.0243,0.01254,-0.36942,2.2167,0.72201,-0.24978,0.92136,0.034514,0.46745,1.1079,-0.19358,-0.074575,0.23353,-0.052062,-0.22044,0.057162,-0.15806,-0.30798,-0.41625,0.37972,0.15006,-0.53212,-0.2055,-1.2526,0.071624,0.70565,0.49744,-0.42063,0.26148,-1.538,-0.30223,-0.073438,-0.28312,0.37104,-0.25217,0.016215,-0.017099,-0.38984,0.87424,-0.72569,-0.51058,-0.52028,-0.1459,0.8278,0.27062,5.821154,0]}}'; + + const text2 = 'ItZZ the'; + const doc2 = nlp.readDoc( text2 ); + const json2a = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":7,"words":["the",",",".","one","lastly","itzz","ItZZ"],"vectors":{"itzz":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"the":[-0.038194,-0.24487,0.72812,-0.39961,0.083172,0.043953,-0.39141,0.3344,-0.57545,0.087459,0.28787,-0.06731,0.30906,-0.26384,-0.13231,-0.20757,0.33395,-0.33848,-0.31743,-0.48336,0.1464,-0.37304,0.34577,0.052041,0.44946,-0.46971,0.02628,-0.54155,-0.15518,-0.14107,-0.039722,0.28277,0.14393,0.23464,-0.31021,0.086173,0.20397,0.52624,0.17164,-0.082378,-0.71787,-0.41531,0.20335,-0.12763,0.41367,0.55187,0.57908,-0.33477,-0.36559,-0.54857,-0.062892,0.26584,0.30205,0.99775,-0.80481,-3.0243,0.01254,-0.36942,2.2167,0.72201,-0.24978,0.92136,0.034514,0.46745,1.1079,-0.19358,-0.074575,0.23353,-0.052062,-0.22044,0.057162,-0.15806,-0.30798,-0.41625,0.37972,0.15006,-0.53212,-0.2055,-1.2526,0.071624,0.70565,0.49744,-0.42063,0.26148,-1.538,-0.30223,-0.073438,-0.28312,0.37104,-0.25217,0.016215,-0.017099,-0.38984,0.87424,-0.72569,-0.51058,-0.52028,-0.1459,0.8278,0.27062,5.821154,0],"ItZZ":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6],"lastly":[-0.31914,0.14873,0.058321,0.069012,-0.03982,0.16459,-0.14299,0.20658,0.59777,0.06121,0.18177,-0.22946,0.080205,-0.081114,0.48642,-0.37647,-0.12512,0.27307,-0.30226,0.090638,-0.37042,-0.26611,0.077783,0.1546,0.082182,0.19086,0.0056976,-0.17299,0.17975,-0.32015,-0.21278,0.31478,-0.19037,-0.11547,-0.082077,0.30275,0.0058198,0.042987,0.025284,-0.4098,0.21951,0.20954,-0.10778,-0.088181,-0.256,-0.092912,0.34733,-0.21431,-0.30153,0.028669,-0.20696,0.093091,-0.12626,0.27863,-0.24892,0.10383,0.24438,-0.42254,-0.43662,0.089009,-0.46733,0.26044,-0.022262,0.1317,0.50867,-0.16419,-0.15378,-0.08092,0.58693,-0.02218,0.053947,-0.014776,0.43776,0.0026166,-0.010212,-0.00869,0.00027581,-0.14649,0.069146,-0.38431,-0.21902,-0.090417,-0.24623,0.3754,-0.22321,-0.089054,0.033319,-0.32897,-0.062051,-0.12475,-0.25357,-0.44094,0.036394,0.27727,0.066178,0.026955,0.012135,0.019036,-0.18568,0.15744,2.33216406,4],"one":[-0.22557,0.49418,0.4861,-0.4332,0.13738,0.50617,0.26058,0.30103,-0.091486,0.10876,0.3058,0.051028,0.22303,0.054236,0.068838,-0.24701,0.32689,-0.082203,-0.28866,0.3734,0.73804,-0.040969,0.040201,0.11384,0.69987,-0.49745,-0.06755,-0.42599,-0.10725,-0.010697,-0.01479,0.55976,0.3064,0.053053,0.058034,0.32756,-0.37233,0.46513,0.14285,-0.085003,-0.45476,0.19773,0.6383,-0.31148,0.10858,0.31557,0.36682,-0.35135,-0.48414,-0.33235,-0.33816,-0.39678,0.1908,1.3513,-0.39044,-2.8795,-0.14276,-0.087754,1.7713,0.99332,-0.14129,0.94389,0.050897,0.47373,0.86387,-0.16162,0.67199,0.52344,0.14438,-0.055194,-0.34669,-0.20742,0.18907,-0.19845,0.34862,0.10121,-0.092119,-0.66258,-1.0582,-0.11803,0.70171,0.077776,-0.50546,0.032243,-1.6176,-0.29302,-0.061748,-0.32473,0.3439,-0.44698,0.085689,0.13295,-0.1807,-0.11854,-0.82985,0.13784,-0.34359,-0.45744,0.49646,0.34906,5.50354886,3],",":[-0.10767,0.11053,0.59812,-0.54361,0.67396,0.10663,0.038867,0.35481,0.06351,-0.094189,0.15786,-0.81665,0.14172,0.21939,0.58505,-0.52158,0.22783,-0.16642,-0.68228,0.3587,0.42568,0.19021,0.91963,0.57555,0.46185,0.42363,-0.095399,-0.42749,-0.16567,-0.056842,-0.29595,0.26037,-0.26606,-0.070404,-0.27662,0.15821,0.69825,0.43081,0.27952,-0.45437,-0.33801,-0.58184,0.22364,-0.5778,-0.26862,-0.20425,0.56394,-0.58524,-0.14365,-0.64218,0.0054697,-0.35248,0.16162,1.1796,-0.47674,-2.7553,-0.1321,-0.047729,1.0655,1.1034,-0.2208,0.18669,0.13177,0.15117,0.7131,-0.35215,0.91348,0.61783,0.70992,0.23955,-0.14571,-0.37859,-0.045959,-0.47368,0.2385,0.20536,-0.18996,0.32507,-1.1112,-0.36341,0.98679,-0.084776,-0.54008,0.11726,-1.0194,-0.24424,0.12771,0.013884,0.080374,-0.35414,0.34951,-0.7226,0.37549,0.4441,-0.99059,0.61214,-0.35111,-0.83155,0.45293,0.082577,5.553375,1],".":[-0.33979,0.20941,0.46348,-0.64792,-0.38377,0.038034,0.17127,0.15978,0.46619,-0.019169,0.41479,-0.34349,0.26872,0.04464,0.42131,-0.41032,0.15459,0.022239,-0.64653,0.25256,0.043136,-0.19445,0.46516,0.45651,0.68588,0.091295,0.21875,-0.70351,0.16785,-0.35079,-0.12634,0.66384,-0.2582,0.036542,-0.13605,0.40253,0.14289,0.38132,-0.12283,-0.45886,-0.25282,-0.30432,-0.11215,-0.26182,-0.22482,-0.44554,0.2991,-0.85612,-0.14503,-0.49086,0.0082973,-0.17491,0.27524,1.4401,-0.21239,-2.8435,-0.27958,-0.45722,1.6386,0.78808,-0.55262,0.65,0.086426,0.39012,1.0632,-0.35379,0.48328,0.346,0.84174,0.098707,-0.24213,-0.27053,0.045287,-0.40147,0.11395,0.0062226,0.036673,0.018518,-1.0213,-0.20806,0.64072,-0.068763,-0.58635,0.33476,-1.1432,-0.1148,-0.25091,-0.45907,-0.096819,-0.17946,-0.063351,-0.67412,-0.068895,0.53604,-0.87773,0.31802,-0.39242,-0.23394,0.47298,-0.028803,5.4601503,2]}}'; + const json2b = '{"precision":8,"l2NormIndex":100,"wordIndex":101,"dimensions":100,"unkVector":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-1],"size":7,"words":["the",",",".","of","to","itzz","ItZZ"],"vectors":{"itzz":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5],"the":[-0.038194,-0.24487,0.72812,-0.39961,0.083172,0.043953,-0.39141,0.3344,-0.57545,0.087459,0.28787,-0.06731,0.30906,-0.26384,-0.13231,-0.20757,0.33395,-0.33848,-0.31743,-0.48336,0.1464,-0.37304,0.34577,0.052041,0.44946,-0.46971,0.02628,-0.54155,-0.15518,-0.14107,-0.039722,0.28277,0.14393,0.23464,-0.31021,0.086173,0.20397,0.52624,0.17164,-0.082378,-0.71787,-0.41531,0.20335,-0.12763,0.41367,0.55187,0.57908,-0.33477,-0.36559,-0.54857,-0.062892,0.26584,0.30205,0.99775,-0.80481,-3.0243,0.01254,-0.36942,2.2167,0.72201,-0.24978,0.92136,0.034514,0.46745,1.1079,-0.19358,-0.074575,0.23353,-0.052062,-0.22044,0.057162,-0.15806,-0.30798,-0.41625,0.37972,0.15006,-0.53212,-0.2055,-1.2526,0.071624,0.70565,0.49744,-0.42063,0.26148,-1.538,-0.30223,-0.073438,-0.28312,0.37104,-0.25217,0.016215,-0.017099,-0.38984,0.87424,-0.72569,-0.51058,-0.52028,-0.1459,0.8278,0.27062,5.821154,0],"ItZZ":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6],",":[-0.10767,0.11053,0.59812,-0.54361,0.67396,0.10663,0.038867,0.35481,0.06351,-0.094189,0.15786,-0.81665,0.14172,0.21939,0.58505,-0.52158,0.22783,-0.16642,-0.68228,0.3587,0.42568,0.19021,0.91963,0.57555,0.46185,0.42363,-0.095399,-0.42749,-0.16567,-0.056842,-0.29595,0.26037,-0.26606,-0.070404,-0.27662,0.15821,0.69825,0.43081,0.27952,-0.45437,-0.33801,-0.58184,0.22364,-0.5778,-0.26862,-0.20425,0.56394,-0.58524,-0.14365,-0.64218,0.0054697,-0.35248,0.16162,1.1796,-0.47674,-2.7553,-0.1321,-0.047729,1.0655,1.1034,-0.2208,0.18669,0.13177,0.15117,0.7131,-0.35215,0.91348,0.61783,0.70992,0.23955,-0.14571,-0.37859,-0.045959,-0.47368,0.2385,0.20536,-0.18996,0.32507,-1.1112,-0.36341,0.98679,-0.084776,-0.54008,0.11726,-1.0194,-0.24424,0.12771,0.013884,0.080374,-0.35414,0.34951,-0.7226,0.37549,0.4441,-0.99059,0.61214,-0.35111,-0.83155,0.45293,0.082577,5.553375,1],".":[-0.33979,0.20941,0.46348,-0.64792,-0.38377,0.038034,0.17127,0.15978,0.46619,-0.019169,0.41479,-0.34349,0.26872,0.04464,0.42131,-0.41032,0.15459,0.022239,-0.64653,0.25256,0.043136,-0.19445,0.46516,0.45651,0.68588,0.091295,0.21875,-0.70351,0.16785,-0.35079,-0.12634,0.66384,-0.2582,0.036542,-0.13605,0.40253,0.14289,0.38132,-0.12283,-0.45886,-0.25282,-0.30432,-0.11215,-0.26182,-0.22482,-0.44554,0.2991,-0.85612,-0.14503,-0.49086,0.0082973,-0.17491,0.27524,1.4401,-0.21239,-2.8435,-0.27958,-0.45722,1.6386,0.78808,-0.55262,0.65,0.086426,0.39012,1.0632,-0.35379,0.48328,0.346,0.84174,0.098707,-0.24213,-0.27053,0.045287,-0.40147,0.11395,0.0062226,0.036673,0.018518,-1.0213,-0.20806,0.64072,-0.068763,-0.58635,0.33476,-1.1432,-0.1148,-0.25091,-0.45907,-0.096819,-0.17946,-0.063351,-0.67412,-0.068895,0.53604,-0.87773,0.31802,-0.39242,-0.23394,0.47298,-0.028803,5.4601503,2],"of":[-0.1529,-0.24279,0.89837,0.16996,0.53516,0.48784,-0.58826,-0.17982,-1.3581,0.42541,0.15377,0.24215,0.13474,0.41193,0.67043,-0.56418,0.42985,-0.012183,-0.11677,0.31781,0.054177,-0.054273,0.35516,-0.30241,0.31434,-0.33846,0.71715,-0.26855,-0.15837,-0.47467,0.051581,-0.33252,0.15003,-0.1299,-0.54617,-0.37843,0.64261,0.82187,-0.080006,0.078479,-0.96976,-0.57741,0.56491,-0.39873,-0.057099,0.19743,0.065706,-0.48092,-0.20125,-0.40834,0.39456,-0.02642,-0.11838,1.012,-0.53171,-2.7474,-0.042981,-0.74849,1.7574,0.59085,0.04885,0.78267,0.38497,0.42097,0.67882,0.10337,0.6328,-0.026595,0.58647,-0.44332,0.33057,-0.12022,-0.55645,0.073611,0.20915,0.43395,-0.012761,0.089874,-1.7991,0.084808,0.77112,0.63105,-0.90685,0.60326,-1.7515,0.18596,-0.50687,-0.70203,0.66578,-0.81304,0.18712,-0.018488,-0.26757,0.727,-0.59363,-0.34839,-0.56094,-0.591,1.0039,0.20664,6.2968688,3],"to":[-0.1897,0.050024,0.19084,-0.049184,-0.089737,0.21006,-0.54952,0.098377,-0.20135,0.34241,-0.092677,0.161,-0.13268,-0.2816,0.18737,-0.42959,0.96039,0.13972,-1.0781,0.40518,0.50539,-0.55064,0.4844,0.38044,-0.0029055,-0.34942,-0.099696,-0.78368,1.0363,-0.2314,-0.47121,0.57126,-0.21454,0.35958,-0.48319,1.0875,0.28524,0.12447,-0.039248,-0.076732,-0.76343,-0.32409,-0.5749,-1.0893,-0.41811,0.4512,0.12112,-0.51367,-0.13349,-1.1378,-0.28768,0.16774,0.55804,1.5387,0.018859,-2.9721,-0.24216,-0.92495,2.1992,0.28234,-0.3478,0.51621,-0.43387,0.36852,0.74573,0.072102,0.27931,0.92569,-0.050336,-0.85856,-0.1358,-0.92551,-0.33991,-1.0394,-0.067203,-0.21379,-0.4769,0.21377,-0.84008,0.052536,0.59298,0.29604,-0.67644,0.13916,-1.5504,-0.20765,0.7222,0.52056,-0.076221,-0.15194,-0.13134,0.058617,-0.31869,-0.61419,-0.62393,-0.41548,-0.038175,-0.39804,0.47647,-0.15983,6.4506511,4]}}'; it( 'with default parameters, size should be 3 & json should be correct', function () { const json = doc1.contextualVectors(); @@ -79,4 +87,68 @@ describe( 'contextual vectors', function () { expect( obj ).to.deep.equal( JSON.parse( jsonWithTF0E ) ); expect( obj.size ).equal( 3 ); } ); + + it( 'with default lemma = false, a empty string in specific word vectors should not make any difference', function () { + const json = doc1.contextualVectors( { lemma: true, similarWordVectors: false, specificWordVectors: [ '', null, undefined ] } ); + const obj = JSON.parse( json ); + expect( obj ).to.deep.equal( JSON.parse( jsonWithTF0E ) ); + expect( obj.size ).equal( 3 ); + } ); + + // ICV — In corpus vocabulary + it( 'with default lemma = false, ICV word specific word vectors should not make any difference', function () { + const json = doc1.contextualVectors( { lemma: false, similarWordVectors: false, specificWordVectors: [ 'is' ] } ); + const obj = JSON.parse( json ); + // FF0E is used because 'is' is already part of vocabulary, so effect is as if there is no specific word. + expect( obj ).to.deep.equal( JSON.parse( jsonWithFF0E ) ); + expect( obj.size ).equal( 2 ); + } ); + + // OOCV — Out of corpus voabulary + it( 'with default lemma = false, OOCV word specific word vectors should add new entry for apple', function () { + const json = doc1.contextualVectors( { lemma: false, similarWordVectors: false, specificWordVectors: [ 'apple' ] } ); + const obj = JSON.parse( json ); + expect( obj ).to.deep.equal( JSON.parse( jsonWithFF01 ) ); + expect( obj.size ).equal( 3 ); + } ); + + it( 'with default lemma = true, OOCV word specific word vectors should add new entries for apple & lemmas', function () { + const json = doc1.contextualVectors( { lemma: true, similarWordVectors: false, specificWordVectors: [ 'apple' ] } ); + const obj = JSON.parse( json ); + expect( obj ).to.deep.equal( JSON.parse( jsonWithTF01 ) ); + expect( obj.size ).equal( 4 ); + } ); + + it( 'with default lemma = true, OOCV word specific word vectors should add new entries for apple & lemmas', function () { + const json = doc1.contextualVectors( { lemma: true, similarWordVectors: false, specificWordVectors: [ 'apple' ], wordVectorsLimit: 5 } ); + const obj = JSON.parse( json ); + expect( obj ).to.deep.equal( JSON.parse( jsonWithTF51 ) ); + expect( obj.size ).equal( 5 ); + } ); + + it( 'with default lemma = true, similarWordVector = true, specific word as ItZZ (UNK) size should be 7 & json should be correct', function () { + const json = doc2.contextualVectors( { lemma: true, similarWordVectors: true, wordVectorsLimit: 7, specificWordVectors: [ 'ItZZ' ] } ); + const obj = JSON.parse( json ); + expect( obj ).to.deep.equal( JSON.parse( json2a ) ); + expect( obj.size ).equal( 7 ); + } ); + + it( 'with default lemma = false, similarWordVector = false, specific word as ItZZ (UNK) size should be 7 & json should be correct', function () { + const json = doc2.contextualVectors( { lemma: false, similarWordVectors: false, wordVectorsLimit: 7, specificWordVectors: [ 'ItZZ' ] } ); + const obj = JSON.parse( json ); + expect( obj ).to.deep.equal( JSON.parse( json2b ) ); + expect( obj.size ).equal( 7 ); + } ); + + + // Testing error handling + it( 'incorrect limit should throw error', function () { + expect( doc1.contextualVectors.bind( null, { wordVectorsLimit: 999999 } ) ).to.throw( /^wink-nlp: invalid value or type encou/ ); + expect( doc1.contextualVectors.bind( null, { wordVectorsLimit: 'a' } ) ).to.throw( /^wink-nlp: invalid value or type encou/ ); + } ); + + it( 'incorrect data type in specific words param should throw error', function () { + expect( doc1.contextualVectors.bind( null, { specificWordVectors: {} } ) ).to.throw( /^wink-nlp: expecting a valid Javascript/ ); + expect( doc1.contextualVectors.bind( null, { specificWordVectors: 'a' } ) ).to.throw( /^wink-nlp: expecting a valid Javascript/ ); + } ); } );