human-web.es

/*!
 * Copyright (c) 2014-present Cliqz GmbH. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

import CliqzEvents from '../core/events';
import BloomFilter from '../core/bloom-filter';
import md5 from '../core/helpers/md5';
import { sha1 } from '../core/crypto/utils';
import random from '../core/crypto/random';
import { fetch, httpGet } from '../core/http';
import { getDetailsFromUrl, isIpAddress } from '../core/url';
import URL from '../core/fast-url-parser';
import Storage from '../platform/human-web/storage';
import config from '../core/config';
import { getAllOpenPages } from '../platform/human-web/opentabs';
import { normalizeAclkUrl } from './ad-detection';
import { getActiveTab, isPrivateMode, getWindow } from '../core/browser';
import DoublefetchHandler from './doublefetch-handler';
import ContentExtractionPatternsLoader from './content-extraction-patterns-loader';
import { ContentExtractor, parseQueryString } from './content-extractor';
import logger from './logger';
import { parseHtml, getContentDocument } from './html-helpers';
import { parseURL, Network } from './network';
import prefs from '../core/prefs';
import { isEdge } from '../core/platform';
import Worker from '../platform/worker';
import { toUTF8 } from '../core/encoding';
import pacemaker from '../core/services/pacemaker';
import SafebrowsingEndpoint from './safebrowsing-endpoint';

/*
Configuration for Bloomfilter
*/

const bloomFilterSize = 500001;// false-positive 0.01, hashes 7
const bloomFilterNHashes = 7;
const allowedCountryCodes = config.settings.ALLOWED_COUNTRY_CODES;

function _log(...msg) {
  if  (CliqzHumanWeb.debug) {
    console.log(CliqzHumanWeb.LOG_KEY, ...msg);
  }
}

function getRandomIntInclusive(min, max) {
  const _min = Math.ceil(min);
  const _max = Math.floor(max);
  return Math.floor(random() * (_max - _min + 1)) + min;
}

function cleanFinalUrl(domain, href) {
    /*
    We need to get the final domain, there are 2 elements that we try to capture.
    1. Which is mentioned in the key 'fu' in the scraping rules. It is not clean in all the cases.
    For eg: That URL at times comes as aclk?. In this case we will try and fall back on data-preconnect-urls.
    2. data-preconnect-urls: We are not using it as primary source, because at times, this contains the ad-server domain.
    When we use this attribute, we need to ensure we do not send back the complete chain.
    */

    let cleanDomain = href;


    // Parse domain from href.
    let parsedLink = parseURL(href);
    if (parsedLink && parsedLink.hostname && parsedLink.hostname.indexOf('google') === -1 &&parsedLink.path.indexOf('aclk?') === -1) {
        cleanDomain = parsedLink.hostname;
    } else if (domain) {
        if (domain.indexOf(',') > -1) {
        cleanDomain = domain.split(',')[0];
        } else {
        cleanDomain = domain;
        }
    }
    return cleanDomain;
}

const CliqzHumanWeb = {
    adDetails: {},
    CHANNEL: config.settings.HW_CHANNEL,
    VERSION: '2.7',
    WAIT_TIME: 2000,
    PAGE_WAIT_TIME: 5000,
    LOG_KEY: 'humanweb',
    debug: false,
    httpCache: {},
    httpCache401: {},
    domParser: null,
    queryCache: {},
    privateCache: {},
    UrlsCache :false,
    docCache: {},
    strictMode: false,
    qs_len:30,
    rel_part_len:18,
    rel_segment_len:15,
    doubleFetchTimeInSec: 3600,
    MAX_NUMBER_DOUBLEFETCH_ATTEMPS: 3,
    can_urls: {},
    deadFiveMts: 5,
    deadTwentyMts: 20,
    msgType:'humanweb',
    probHashLogM: [[-1.839225984234144, -1.8009413231413045, -2.5864601561900273, -2.779077348461893, -2.41154163187108, -2.8701669802592216, -3.010853183897427, -2.9831997811803244, -3.0258844950086354, -3.028549665472028, -4.922879838696041, -4.503195676571833, -4.598154161855354, -5.027344698225922, -5.217723368153992, -5.366375752342653, -5.195909278900248, -3.2940850310881387, -5.181468249477294, -6.110371416838184, -5.766202229535159, -5.393352786265537, -4.920024681004049, -6.049294408676102, -6.399015490326156, -4.690163484884073, -7.459851804835359, -5.825337077308845, -4.675000760327133, -5.3635239851672845, -6.331713706782754, -6.169284763337202, -5.951151191317628, -5.768078242705815, -7.332129653597336, -6.766704624275413, -6.01222435569866, -5.268617743244878, -6.279416117827666, -5.861972559492891, -5.838071468059843, -6.267250173459698, -6.283981033030117, -6.454759516784266, -6.5195694451700765, -6.83526729571271, -6.458157413359671, -6.359696067797872, -6.250241538141395, -6.647172527007435, -6.973987597161365, -6.4598607011160425, -7.685547265665763, -6.5912026847359835, -5.794718807999361, -6.474629988987015, -7.135098972554303, -6.880960523371609, -6.747429130747086, -7.733118660641065, -8.398636456057481, -7.6323973914922005], [-2.11332750230783, -2.3060786022344897, -2.3740965556720726, -2.5398307619465608, -2.463417892707307, -2.130953109899374, -2.8009573227909073, -2.9401508827086436, -2.748847204575824, -2.8414237914951976, -5.010884149435608, -4.7716590964777925, -4.868359687249472, -5.322380370933768, -5.037006822424893, -5.279762905036231, -5.635270908532894, -2.9219817650555973, -5.50179037155463, -6.5406369920023, -6.311742070591071, -5.516975693542294, -5.7251712834206305, -6.083331008626942, -6.409455776722731, -4.856277028595425, -7.356357334609669, -6.255977591877104, -5.042089425488359, -5.723688703460408, -6.9306288690141455, -5.879866127104103, -6.269949621534028, -6.969512838475713, -7.475373123729266, -6.802062729887527, -6.566109586768108, -6.661695002244713, -6.634070998648396, -6.128265317507457, -6.0654489711795785, -6.79728322915323, -6.99559673771516, -6.996654379273295, -7.671972036661277, -7.840255377779173, -7.132455920432357, -7.233113706563278, -6.737551430374816, -7.2706736012479585, -7.430290364348156, -6.991903776514698, -8.559572276072503, -7.231105001502197, -6.117005677632925, -6.669675284003577, -7.87022016648111, -7.635233211015519, -7.18343028641663, -8.53463332772525, -8.567176875457722, -7.573077285525099], [-1.6313545940843748, -2.457209014039589, -2.565380346827546, -2.767788786784649, -2.6386222432933284, -2.595661570703591, -2.6893427324813977, -2.7198733792574403, -2.720533202987637, -2.8467380805237736, -5.029918409868431, -4.178710463028381, -5.07588590483658, -5.057436988529895, -4.899938336310833, -5.271744381251988, -5.715419865537325, -3.1739071041721116, -5.367102243680773, -6.224018730172038, -5.866973141306898, -5.730015933050199, -5.360958031920617, -6.20201967617667, -6.328452890969734, -4.837569916970598, -7.520833339090353, -5.8325529174204185, -4.950768880962046, -5.339171801246006, -6.500538943869661, -6.141626179853517, -5.826929659866057, -6.371549418153106, -7.095602530626063, -6.863928261303842, -6.26532998038965, -6.486382994639529, -6.568804701240378, -5.928030195562504, -6.138501621140467, -6.375855129864318, -6.672567424416514, -6.87690014131182, -7.500698430681298, -7.486404619424152, -7.110483757622713, -6.948864783187929, -6.7689909475117735, -7.44680608044408, -7.305175362028806, -6.490004000406482, -8.278142463867583, -7.100043692626031, -6.262757631704339, -6.785644569035485, -7.917859578552653, -6.966883288690608, -6.7663306639529575, -7.827911341889713, -7.709275282211631, -7.650980633730635], [-2.2483023852308777, -2.2601431921520803, -2.543211818297523, -2.6290975996017654, -2.597012293697696, -2.425038774903284, -2.6464962853598264, -2.7268820909874645, -2.7170988469992086, -2.6384412943652213, -4.8059842634060495, -4.801920790322523, -4.881995682865955, -4.399130557719554, -4.977525552072196, -5.254333597613871, -5.4861286120783905, -3.172333855510997, -5.311009967578493, -6.3787775808955045, -6.060833750357407, -5.179488625881995, -5.284849141539821, -6.185226477248653, -6.342526765978808, -5.0477193482508715, -7.229508117457252, -5.92913163950562, -4.781563610047119, -5.175693021409531, -6.686598785853124, -5.953265540842049, -5.765108625614716, -6.249636653394711, -7.053405713688212, -6.871084156894258, -4.746700678954438, -4.435206614193364, -6.2492260600816145, -5.437566316202071, -5.90294982336378, -6.5708663077474965, -6.4774110336687505, -6.642451113295277, -7.490560332753359, -7.638809897851904, -7.037026821604173, -6.792453878834064, -6.557926605402683, -7.183535297458448, -7.556615154408894, -6.675849393697125, -7.929241490870054, -6.920447880012594, -5.937211886970303, -6.536908132533395, -7.266281062975223, -7.138583909596181, -6.9407325511841, -7.611188320541975, -8.063402408224212, -7.7539194100649205], [-2.0350349956205984, -2.348602646771946, -2.41594411405956, -2.651370395735765, -2.563410442434685, -2.706962611141542, -2.7917112923152776, -2.7483543933766477, -2.6785026736567357, -2.8111298756441037, -4.759837736546805, -4.20618015484438, -4.787453629928326, -4.659538126267765, -5.015532806050122, -5.056335259588051, -5.339468177159593, -3.040667184111712, -5.203144447478613, -6.2264472230490675, -5.7097357671058155, -5.585194192596145, -5.035187712987706, -5.542988699592872, -6.206189420830567, -4.864552817623237, -7.387161020524848, -5.130725109803191, -4.710070938057912, -5.015532806050122, -5.563665939332496, -5.8412571088023135, -5.744609136156329, -6.206575148897066, -6.842735398083401, -6.794361813799185, -5.850672107736916, -6.088449236501511, -6.348519254409916, -5.725588459198469, -6.064735388153122, -6.479956671206651, -6.2209527037314265, -6.70094736915976, -7.3405268040104845, -7.140183173146506, -6.467358872953416, -6.7233339772218645, -6.184068000080206, -6.381516598393398, -7.406185056529545, -6.368731036096427, -7.402351186018824, -6.6091509166536255, -5.9623588984476195, -6.352529061520146, -7.133337180638517, -7.24783614109371, -6.901266731308386, -7.234806640803376, -7.432126848507545, -7.494477663675224], [-2.0215812860615885, -2.455637406643175, -2.4330253990141046, -2.6113902739466517, -2.6033787533338284, -2.6337646574813993, -2.7111759743895725, -2.7471499108067454, -2.721672070749707, -2.7644985821403028, -4.771735346669196, -4.967298931497775, -4.806933772233218, -4.796223062219612, -4.702049659123834, -4.615986584200591, -5.422222957790021, -3.024083897225106, -5.341473161524181, -5.691932137893965, -5.870247680316981, -5.6516120254419615, -4.85258639108815, -5.9684706438872475, -6.205102036008844, -4.762283950640148, -7.106211026775939, -5.754565801069223, -4.28552037163959, -5.083154204506742, -6.514533306380853, -5.91282033178054, -5.801594696670316, -6.211408552601299, -6.884100000075919, -6.5060032575028, -6.051550266183317, -6.1528452495165125, -6.237040028392324, -5.845664774991169, -5.83750622157883, -6.376088248894053, -6.3936457603588925, -6.58430154569624, -7.071724850704769, -7.213982427018076, -6.56730052905341, -6.608473779316545, -6.403307671270629, -6.973970456584425, -7.485700648480843, -6.5060032575028, -7.126013654072119, -6.592294210440276, -5.525669597265563, -6.0726189713973024, -6.168884085714272, -7.057863242618145, -6.702467385782517, -7.84259108644045, -7.932742183434748, -6.889071012797939], [-2.3515175517946965, -2.4895377785815813, -2.455794452354539, -2.6177065611128625, -2.3916765698655924, -2.55893036105207, -2.621684157393488, -2.6988517931658524, -2.630805311213068, -2.6634082117951667, -4.639123440452939, -4.108116635400416, -4.850674635561831, -4.879642999369406, -5.020506071184806, -5.079702479761731, -5.3856932140647995, -2.922465393995544, -5.272571266829349, -6.095451326503016, -5.703507123198923, -5.5861660464475555, -5.144197286007818, -5.722031888813277, -6.28586346876822, -4.872555039237073, -6.886991750607913, -5.521389523166842, -4.712785087276782, -4.712557943607434, -6.258849400356963, -5.781080159458192, -5.683518571573409, -6.497978247391754, -6.801813554699944, -6.640350228026399, -6.23670827447975, -6.480528140170159, -6.503409353749256, -6.035279500709713, -6.187868456644626, -6.634900623258835, -6.227883924459252, -6.665648019797556, -7.5457712876636505, -7.422426740346121, -6.235666064901307, -6.769319721223498, -6.343344078699495, -6.975730165249003, -7.353206363454935, -6.635677322327007, -7.937813375439674, -6.762240045635436, -6.147006740966892, -6.741296871790193, -7.715813334827543, -7.183705495090042, -6.938232467377743, -7.593211012735211, -8.021194984378726, -7.326496213127361], [-2.404503840183417, -2.43761707422273, -2.4108252303609095, -2.2999425986283435, -2.542843475674806, -2.6029429346724786, -2.584141922741018, -2.5855127818195496, -2.5510561981552526, -2.647681869737238, -4.782820501953572, -4.808208666935694, -4.979563332219715, -5.008211525808261, -5.046432738628459, -5.142457459370653, -5.485491564024772, -3.1144173778919617, -5.417825855041213, -6.245547233803318, -5.938268453446564, -5.963941744764968, -5.450843208258146, -6.059373833665894, -6.223182084343739, -4.942745700375246, -7.208409268623897, -6.038143491827719, -4.6277437642171355, -4.892513996642899, -6.694766741586792, -6.117821568685912, -5.949820065666706, -6.502648045547855, -6.835922059321975, -6.694766741586792, -6.361312919572388, -6.403294392961231, -6.118769885494771, -5.676056523505988, -6.406451957946552, -6.5244722133881785, -6.6333977952105005, -6.6574953467895615, -7.573786078663716, -7.446115322000925, -6.883692335590661, -6.903250344490318, -6.3498910941692, -6.905331510694142, -7.588115904218541, -6.879623153983935, -8.035692497676832, -6.939233062369824, -5.959077527634608, -6.324513515108117, -7.549688527084656, -7.120692921566297, -6.988939594552065, -7.891731808755324, -7.727259481120012, -7.430143376434873], [-2.2924692182301323, -2.450539739657926, -2.3769703926915984, -2.532604294663703, -2.55393918066343, -2.609718771364289, -2.6246425013179655, -2.661744915258029, -2.6011009513028984, -2.6236780353341946, -4.669477044233787, -4.1521285006409165, -4.955491059584871, -4.818890308190988, -5.030853379357302, -5.002039294312803, -5.278250342600015, -3.076829272811893, -5.307731406455737, -5.826171280206974, -6.022604681193609, -5.8364861502494865, -5.222160875948972, -5.86734719439116, -6.263453577434854, -5.1217643044016885, -7.32073988901422, -5.758783848671988, -4.666476541899311, -5.259001445170042, -6.374002568031498, -6.083417735921755, -5.695962812152638, -6.490715185357981, -6.417147740477192, -6.55681384079998, -5.431378631989581, -5.843302320137157, -6.369727752361356, -5.638321273173884, -6.50456582229188, -6.275027142828877, -6.210627212225794, -6.774279201200845, -7.54625217416398, -7.377527245335997, -6.968174323388823, -6.961544464850153, -6.5738576027632725, -6.808653231033201, -7.247759185607984, -6.667823395181603, -8.06567857870257, -6.862038055489511, -5.923075085846576, -6.772454383176216, -7.732534132174032, -6.980444415980637, -6.935456028765855, -7.926350068393722, -7.409760739910981, -7.357691884279851], [-2.395140177772492, -2.3972905877762423, -2.4380054501613637, -2.59514809923619, -2.5304957080424493, -2.553494441315067, -2.592658670737186, -2.6234135177213647, -2.5068257242437872, -2.493912396989218, -4.415128005729072, -4.8162395242315545, -4.805681457561184, -4.959588437581088, -5.013259465745109, -5.070148436368585, -5.778302005668334, -3.0289910877331776, -5.527143864017425, -6.11105402601701, -6.123887370951674, -5.975929983744964, -5.442482986164537, -6.151079378646925, -6.120416006196215, -5.065655600056955, -7.071974876907828, -5.957253719340739, -4.781773843113505, -5.354948164056644, -6.520450343119245, -6.293473674919533, -5.817016517849025, -6.85914069525248, -6.81356318675616, -6.777584585720929, -5.97764525003172, -6.226877237430227, -6.02961643394924, -5.9729353291005545, -6.418715298488868, -5.532354351604477, -6.803710890313149, -6.889653320113874, -7.354541848692839, -7.502757793000282, -6.938924369120657, -6.214291571987144, -6.749334119435077, -7.237514956739779, -7.394443065445024, -6.457482478746571, -7.947443614261728, -6.86121538479334, -5.698583920623846, -6.845759126556648, -7.683896617545187, -7.180188461548061, -6.757725777071325, -7.8704825731256, -7.881944016644606, -7.107203568791572], [-6.7847523397462, -4.956382101946025, -5.55168468117927, -6.54694976738503, -6.679027997805541, -6.920449307304025, -6.651631877885628, -7.028532297007316, -6.819873006155468, -6.9668156836069475, -5.07824502580267, -3.6978375281608122, -3.2158001218369128, -3.221520357616586, -3.5888259593254497, -4.18118917224718, -3.0226572253435795, -3.971027081545084, -3.562365839815367, -6.241860395850777, -3.738063429007306, -2.286143322088719, -2.8154088123469325, -1.9366972011829824, -6.402763696295053, -4.09673956284193, -6.7632686126933175, -2.217315348390325, -2.7898081670861736, -2.2970370063901697, -2.7943042308650328, -4.45525252224197, -5.6170558845712195, -5.634999368405547, -4.204524838877218, -5.247540598697068, -7.679731693977602, -7.496500850432476, -7.694782629326121, -7.660949127015419, -8.444625153562612, -8.02242837580043, -7.9107342568656245, -7.844229700045118, -8.496568431723452, -8.90528981102539, -7.744710798608082, -7.796294560812227, -7.4084137546993, -8.54315037261458, -8.96995880381818, -7.702981796781846, -9.818649422090688, -7.982756374523151, -7.087136791691802, -7.726484291615801, -9.302472289566692, -8.264363329731667, -8.520253749464962, -9.573314897467998, -10.041792973404897, -9.03352354731742], [-5.530908158078372, -5.12660572425173, -5.285205739999349, -5.724471900365845, -5.873312125055024, -6.16534879225732, -6.148934708170209, -6.3610443530793574, -6.311033932504697, -6.085846837289822, -1.9318371570117603, -3.9897678769234868, -4.264721024425275, -4.508665668126993, -1.397989588361542, -5.165043688633805, -5.1354703161064155, -4.710076673495516, -2.2812710196631696, -5.744808587095395, -6.153180999051661, -2.7741841902487896, -4.878049696777151, -5.0409257316634655, -2.3898944653854226, -5.13595610674021, -7.841822106027579, -2.8471437524600627, -3.4340095911141377, -4.619316513108799, -2.3549200090866647, -5.921518406898233, -5.302848234969303, -6.786138406296854, -4.207750888408638, -6.039012800612939, -7.841822106027579, -7.704962923310382, -7.065002449743519, -7.589296290560376, -8.260096172428325, -8.035112694192108, -7.871344545293901, -7.140827013169332, -8.545938317958099, -8.836893715754849, -7.993349965830133, -7.27788665694764, -7.741565502287929, -7.888977243944857, -8.111485672976682, -7.875096894912451, -10.091297373707675, -7.392485113590403, -6.863268166852988, -7.610229500226842, -8.901213681542407, -8.92241588919301, -8.313038591698524, -9.803615301255896, -9.310181420201774, -9.00812940746704], [-5.262771475279199, -4.784242541566078, -5.27970172852834, -5.684918512820975, -5.945292208575153, -6.218979090786592, -6.228233983716081, -6.354509212158639, -6.204315840281487, -6.442281485719593, -2.526983467749325, -5.114296740462473, -4.530006859098648, -5.422625873895182, -2.9651924820539826, -5.615416770210166, -5.759915004501759, -0.738654569892248, -3.9951800832331146, -7.496424744983864, -2.271649654828947, -3.6688863968618732, -4.479075949205008, -6.406971077861438, -2.7704548687771466, -5.064362048924221, -7.65209604855975, -4.10990241837356, -4.3379100438872245, -3.144791234860585, -4.257400003418622, -6.678114421469913, -6.866487433947222, -7.697728752690109, -5.8045538215356425, -6.443871311173693, -8.079096309219214, -8.153204281372934, -8.17767713789166, -8.353264662504271, -8.864090286270262, -8.820319623214933, -8.693771874337664, -8.686253041923635, -8.948055666387596, -9.842684901480572, -8.909864867655205, -8.569719225667685, -8.285794202093616, -9.08611589448313, -9.094531091408415, -7.571541334321345, -10.622843459030147, -8.818180584966184, -7.823031278179709, -8.265917771336019, -8.826764328657575, -9.481851947393176, -9.0917181500318, -8.558120089824332, -9.233307603986084, -10.138335143581529], [-6.352576341887475, -4.375482811098582, -5.097247890129581, -5.253867383491415, -5.300921501088017, -5.719261065235271, -5.695077377144448, -4.664604980788103, -6.0509261540753565, -6.1211148128546835, -2.6955840583648807, -4.0885439717448415, -4.601612684872268, -4.3272866151564005, -0.8335121453767717, -4.368698782518719, -4.841722524158204, -3.6648788033138118, -2.5172646408008483, -5.8838060715240985, -5.03345350892465, -4.275021730152804, -4.6405776530763765, -5.25643229403688, -3.133213396702524, -3.881602832644344, -7.333792367288475, -3.6375800962202542, -3.280343679384232, -4.168603006615407, -3.0158762344692915, -4.957870530657444, -4.673357005846336, -7.071279326505898, -4.732292586433726, -6.102987428262127, -6.308187629826426, -6.692855118141101, -6.912844895031934, -6.811275201992611, -7.387021770702892, -6.866694309094154, -6.971677700053924, -7.120054614261141, -7.672123196561181, -8.47105299909571, -7.06564216575394, -7.303313817631059, -6.877175818825198, -7.588372692026248, -7.710844848156796, -7.012073355807918, -9.487490679574803, -7.203069557208429, -6.086293297912648, -6.450936411500557, -8.098699438256325, -7.677018501414594, -7.313496820953744, -9.611543328244782, -8.931691690166081, -8.112931574673333], [-6.278777370845214, -4.701340011999448, -5.078865738235898, -5.8830885176004895, -6.165050673343658, -6.274964098411205, -6.212416394644692, -6.400778791567437, -6.741692863598321, -6.816576094912807, -3.3928904712107735, -3.566247086795751, -3.646180662937257, -3.523693462975528, -4.243767227003855, -4.080529251966207, -3.795135190780714, -3.596123418433819, -2.7581828008765474, -6.336206897605354, -4.412794119233608, -2.7790238541384125, -3.7937492847610685, -1.8714655708089012, -4.039189587014222, -3.917817104421266, -7.57147815542886, -1.7211826197805813, -2.454732555101985, -3.258178620019967, -3.831607640543702, -4.745787338078659, -3.8173867609921763, -3.6132042458306612, -5.788388393117924, -5.078865738235898, -6.509442541403006, -6.358094961721853, -6.982655840165303, -5.955633264954192, -7.203135532877834, -6.519814112808369, -6.508748303549079, -6.5805671825510945, -7.521390752829464, -8.307422019239775, -6.987204748691672, -6.90870144071007, -6.578332663638785, -7.263487211998134, -7.281252691733005, -6.635300404511988, -9.747550691415777, -6.939413018359865, -5.8716986926591, -6.717215468733563, -8.061948046761787, -7.4055504084105825, -7.052394044113555, -9.471951436870118, -9.440772718621666, -8.392393495069179], [-6.019277337678054, -4.809610930457335, -5.353723551448762, -5.676382755872694, -5.70371603264992, -5.986064759126338, -5.8139317591377235, -5.978144349530662, -6.042881895447475, -6.198360420377549, -2.2921047448703056, -4.34095023979577, -4.345648109454055, -4.367598309292761, -2.0155741702111323, -3.0132780221373907, -4.923111941566577, -4.563679111444075, -2.223531060167143, -6.691102686207805, -5.506010794027101, -3.0594865563826104, -4.849221407410736, -4.504665385487454, -1.9294270482862954, -4.87704758753489, -7.8125932274537675, -2.476262613634102, -3.9821539473427885, -2.8193399945915623, -2.274864811699418, -5.85738068573614, -5.424700477670406, -7.189965927648669, -6.389275800912233, -5.638934468614081, -7.03745392086323, -7.341255375194895, -6.260863632640313, -7.279063354938447, -7.679061834829245, -7.811259004440631, -7.236144890792278, -7.5275849533863886, -7.494018129747305, -9.1181262319292, -7.907769904821474, -7.576977708715964, -7.061731490444577, -8.083943001314976, -8.105182737815888, -7.709906510180343, -8.65367988774833, -7.560240916360441, -6.676421938949557, -6.335428678009885, -9.006382193489584, -8.143473650809202, -7.5376758563683515, -10.154666091954931, -9.349927845986523, -8.656780665426579], [-6.45948206141902, -4.541389017548509, -5.0469147172405115, -5.714893702071352, -5.93765911006474, -6.159588168280474, -6.484829040506417, -6.54643138634695, -6.777423978144398, -6.472536180151478, -2.0992585177991696, -4.351360197286945, -4.810779680772453, -4.478347221038041, -1.057979923268165, -4.503803429976553, -4.369600328148692, -3.1761274441594316, -2.5925706700973334, -6.132316774171815, -5.0527495258256305, -3.548319221739584, -4.665298532220545, -4.239092161682817, -2.8823289365699227, -3.9799558061803935, -7.613527691788449, -2.9149052754992715, -3.0856952412535055, -3.988705873974733, -3.519423658731669, -5.380012776918168, -5.236242140888087, -7.294418265937878, -5.378236234528395, -5.653900468202214, -6.843339306558724, -6.1506932535462235, -7.23734725322546, -6.366553451469326, -7.364460455464448, -6.868708500684183, -6.652923233094938, -6.285920796794185, -7.434940854111734, -8.332992180665768, -7.045904469629325, -7.3935637933916185, -6.8666551118723405, -7.771896007559911, -7.6868589648739984, -6.982133113853507, -9.79784886428876, -7.098382406877942, -6.05045863483477, -6.643853499354418, -7.8988076492249855, -7.62952803313489, -7.2191872449632415, -9.030593711575094, -9.78510983851133, -8.512650620040239], [-7.312010848930209, -5.9704021836204655, -6.3878250976208735, -6.964626238636961, -6.946983784483292, -7.4124339237241115, -7.3181741966378775, -7.680384414313297, -7.712878247789743, -7.554150900935073, -2.446039713635401, -5.2426041270797, -5.947299500896444, -4.663409565765878, -1.9484553212571183, -5.736061416021277, -5.712018325729332, -4.9933347157551395, -3.124667578606162, -6.881965531986077, -5.837272445819192, -3.4071965596735665, -4.466780007835062, -3.7439634287862087, -2.4245394005655605, -2.7481043832730636, -7.96925132102691, -3.001717011027925, -4.3613498567188325, -1.0557054893757776, -3.70597885137566, -6.306264213716016, -4.571775536818785, -8.174476930790583, -5.896948417633875, -6.225037976827956, -7.408704473732679, -7.482057817244962, -7.278478028822127, -7.6772808871048746, -8.27951403118039, -7.874318428531343, -7.723948471543991, -7.700568997316141, -8.91379761462879, -8.048158534515101, -7.944081716081054, -7.980577820654186, -7.290733450482923, -8.159308709317411, -8.547715834521, -7.821812931153412, -10.179602578494553, -7.600205085285631, -6.935111692528849, -7.541317747952087, -8.80008790436004, -8.364689804291782, -8.08874001915325, -10.681459532277783, -10.11675467483489, -9.167510387022356], [-7.888212115175222, -6.644858293408979, -6.899926832152373, -7.644887076927795, -7.6845832447435445, -7.532598742461448, -8.35541557609491, -8.047237610771814, -8.039272966898704, -8.151069111852731, -3.668932909455333, -4.782855975285661, -2.6659061839419613, -2.8967154778709228, -2.0223904901109755, -4.446771913881909, -3.000219123980515, -4.8870757508472185, -5.930891109758962, -6.823383785692898, -3.2455278628878323, -2.6715253337166462, -3.4540042231373063, -1.613293110190054, -3.32378892967727, -4.249589417392827, -7.22956773062339, -3.479217249500404, -2.6394006861426296, -2.505331420610824, -5.608431497999796, -3.9128781321838506, -6.817651617834854, -5.899129031817539, -6.838124773860243, -4.798082958969038, -6.707198867504101, -6.862144714309829, -6.946702102337892, -6.119321735858142, -7.3106242225785305, -7.047703884629593, -7.1735103563413185, -7.1142937780684905, -7.8727953781736435, -7.684274173500164, -7.105593531784784, -7.043298950514618, -6.705339482889372, -7.153500510194026, -7.545964351744817, -6.429130146514012, -9.208169648866278, -7.146978747947639, -6.381569697965211, -6.863775814304458, -8.274721978073412, -7.87653370028425, -7.678419897035876, -9.611509357662127, -9.195484489338961, -8.64028017837942], [-5.479179331884548, -5.058052917950845, -5.462604366790335, -6.042749073104725, -5.955329948912708, -5.958323963125313, -6.182592584875092, -6.134964535885837, -5.987223312688662, -6.092926822690444, -1.4103668710151773, -5.3350267357418115, -4.970584606980225, -5.289457186272422, -1.770010358993832, -4.901554513033236, -5.37688758694302, -5.089624713826659, -3.3520314691389803, -5.392942008652894, -4.734460344193294, -5.4364649785873, -5.373540303482963, -5.098468738623112, -1.787587681664556, -3.2734283276295595, -6.018573842572738, -5.287155689284143, -2.4508369623232906, -5.268931950327691, -2.1042922431924556, -5.627005270227098, -5.60465202444635, -5.974953220096848, -5.917202163476144, -5.89303609762896, -7.4048682036926285, -7.92851451588768, -7.027392005486696, -7.609083745121319, -7.38594019380711, -7.373517673808553, -7.797674914928869, -7.69894007424318, -6.9721762828842495, -7.92851451588768, -7.9072371174403955, -7.498269378781029, -7.24617825143195, -7.836141195756666, -7.972477639308797, -8.030297210197622, -7.424161406627308, -8.130380668754604, -5.985681292336846, -7.548459123304884, -7.876146530370365, -7.886403030537553, -8.383576565135217, -8.509870290459508, -8.104405182351345, -7.3612475812167375], [-4.615704248273826, -5.059874046144977, -5.605352624766998, -6.125932548297868, -6.262849675716521, -6.79239877505316, -6.990929985784472, -6.841047905444769, -7.1584533214794765, -6.9886390340379165, -2.1061188583709898, -4.694960766195293, -5.294767434713961, -4.877455498156362, -1.7372915500183557, -4.54913156517679, -5.204152324138941, -4.129384627657894, -2.037945671503165, -6.073264053143508, -4.589028985169175, -2.990753289694604, -4.778969963958759, -4.689721987339014, -2.425686266522293, -4.746481928875957, -7.836127606736178, -3.351158263461062, -3.2418172438356554, -2.1018059989954563, -2.7036646563447677, -4.616611132654885, -4.89196974360223, -7.672674534246606, -4.594182160042578, -5.702759391924034, -6.928687676907251, -6.888038111045066, -6.879281938695605, -7.0247524516915965, -7.464610089759517, -7.314413547152238, -7.29014621148154, -7.293244587314209, -7.623992355607218, -8.372647679876952, -7.108856380340105, -7.38925322665568, -6.931385277679899, -7.915280634635728, -8.074744053353632, -6.836124212582984, -10.164407149105006, -7.372318020898097, -6.283267109907934, -7.0071153102054895, -7.702596214111655, -7.848216404055182, -7.332806708624255, -9.549591811814969, -9.295811291038868, -8.128717117098297], [-6.86596902684244, -5.052082122851932, -5.150933488599681, -5.292208274141252, -6.214682697483703, -6.745331904339627, -6.68552490120206, -6.760045968476806, -7.227305715576431, -6.950863382929711, -2.1215717821722215, -3.9228106947364623, -4.840908762978201, -3.0664686008437925, -1.6983460154315366, -4.061987675375524, -4.329017422576037, -4.468146834761754, -2.0559567932823986, -6.284656471404593, -4.495653917304999, -2.239702601986903, -4.059167724800704, -4.675038779649125, -2.554078005507218, -4.208324039380938, -7.846798279854166, -4.952711862629422, -3.1510988852700317, -2.942373372199169, -3.3895078376634635, -4.604176621456193, -5.1138376420493366, -7.9144140567925305, -4.640693910453494, -5.102920313867999, -6.555881088992486, -6.826369644359554, -6.799686147757116, -6.863359893400418, -7.558312671802701, -7.067443904093005, -7.193279183729173, -7.1899393531824245, -7.705592764618944, -8.457679157012771, -7.256276285899237, -7.186610639983997, -7.0503828029559665, -7.591798854832744, -8.06246715675167, -7.178062400973902, -9.768878866076495, -6.917516203514602, -5.665418470446385, -7.260143611855673, -6.937390775844807, -7.687607497719584, -7.3485921408206325, -9.2384661090352, -9.464847655623185, -7.897922355187247], [-6.774920173861653, -5.204290861824316, -5.43566671911157, -6.296930641076284, -6.565626027570306, -6.7373590105822805, -6.991252066666452, -7.063233360250443, -7.2676834795551315, -7.1966044322333405, -2.083909994196951, -4.067585265323549, -5.182321460673428, -4.835732283228186, -1.980079058156674, -5.0283663440075825, -5.072641431371989, -4.6392056435133515, -2.7086094993416667, -6.770304135451181, -5.008181972697398, -1.1019554288598505, -3.2594369265598493, -5.6765883311043766, -2.736784393019736, -3.364824539297133, -7.918041582638827, -5.789527217909832, -3.4282422713486094, -4.498435097687583, -3.6322465613937016, -5.355734234640129, -5.0804532195051415, -7.17597860187687, -4.9759313787901425, -6.718299350460867, -6.452973839496266, -6.266241964550636, -6.756169624516277, -6.43275589071587, -7.149632742116933, -6.722481373881418, -6.632646871978412, -6.5830057991087605, -7.441299494170494, -7.5325638154552745, -7.295254585596671, -7.196283868037615, -6.683920964952739, -7.4835386813502325, -7.587428922371031, -6.992558402580507, -9.54477866233574, -7.0691430439756715, -6.0008086042221205, -6.0079129112663265, -8.273962947039259, -7.457398913104675, -7.183861348039058, -9.235518989239411, -9.247910721534573, -8.527701618931673], [-6.725963719695806, -4.710012269685353, -4.981543035572536, -5.7693914443296395, -5.741238652017211, -6.34104225410854, -6.461701921452123, -6.616580803658851, -7.050213688876461, -6.932988782385651, -2.7442254377569215, -3.902092293780571, -4.081087684032349, -1.8459531867558991, -2.260196124051401, -3.8113055093318327, -2.1904142607255017, -3.5077174805325915, -3.070625904070985, -5.688404814159279, -3.676818033328958, -3.705969932950152, -4.175966474127823, -3.555777846153588, -3.6599169276373167, -4.233380646142029, -7.565625193952616, -4.758247211588438, -2.668505149650637, -2.593317887541856, -3.982238613532973, -4.922572014698711, -4.457568202222798, -6.922768742308341, -4.874097132395777, -3.7366670217502587, -6.099051002533345, -5.950541812233132, -6.754321355464054, -6.000835251677606, -6.818602573551164, -6.538250446625281, -6.5136752899401404, -5.7389354920585625, -6.837502673710269, -7.922940958163197, -6.4879709082904045, -6.559235554477551, -6.415308634074, -6.963830229106114, -7.464928001069955, -6.466545899251641, -9.489635645899149, -6.3635319575989815, -5.527598474760337, -6.407003965795449, -7.463914829757348, -7.418626873140783, -6.4607730713017455, -8.850715648219035, -9.544876513609825, -7.984492491801492], [-7.251231594323855, -5.21374696908488, -5.698223064924641, -6.1375976604763816, -6.322511629217009, -6.768362468268195, -6.494106816635844, -6.457670861321346, -6.860741199932812, -6.755693520691695, -4.0580333694338515, -3.9411669781297105, -3.7467312772977017, -3.0227837077197126, -3.61529958808048, -3.574469853485688, -3.2007714947116837, -4.080705573827712, -4.670471138732819, -5.7966968651863855, -4.155602316322404, -2.9527040483599185, -2.862751521261682, -2.037725742388079, -3.7818488691194445, -3.113228390392548, -8.164264994327704, -1.8458475827404217, -2.5467378386179824, -2.7020967605017185, -3.188124830525815, -3.6799312814828857, -3.4909044267832696, -5.197365837878935, -5.202188998354364, -5.870321468395694, -7.4721345906728445, -7.508740887837122, -7.308294052687071, -7.524695844262695, -8.528908107915612, -7.92499206108361, -7.684820971302881, -8.246116020758024, -8.52404191826444, -9.16793672064164, -7.835273241415741, -7.178620119286367, -7.559608480967977, -8.499112535221915, -8.875184344633446, -7.775193695995215, -10.198665810583236, -8.01769231637253, -6.730988787795164, -7.673144441641045, -9.323628945401888, -8.330457169191774, -8.013022138590646, -9.77099654516406, -9.908396414566504, -9.416350685452109], [-6.383306695601549, -4.903894061372386, -5.294037540593016, -5.450500954330478, -5.333223053427145, -6.308171725769449, -6.710407117707228, -6.887542628987831, -6.847572675445971, -7.035457528925254, -2.2105034037003235, -5.0202198363584465, -4.228806005807516, -4.2610399361561155, -2.630714925229401, -3.899363165669206, -4.911168632565807, -1.9598915311908123, -2.357698156549484, -6.874278408705108, -5.4856719078268386, -2.9453898821076563, -4.76865138002776, -5.849682782165729, -2.1582691520417923, -3.13711767038794, -7.932068702852963, -2.1341546758789423, -3.3759594869296796, -3.525124340216131, -3.4812218611704324, -5.083766155674364, -5.7936328268294535, -4.353576622920111, -5.907706290260866, -6.135534326871947, -6.846532994187875, -4.609063889576182, -7.503287789431182, -7.483445868632177, -7.2287173521187755, -7.853210127940644, -7.155334806184752, -7.407188463381393, -8.090673733029602, -9.081164896953064, -7.857960730699241, -7.896800564015505, -7.262268886290009, -8.128645226782451, -8.177248142566354, -7.485412437604219, -9.720755973044048, -7.9739596445621235, -6.796539470357211, -6.907958941482774, -8.646989682962468, -8.283628546124925, -7.797896489956591, -6.6686665609139695, -9.771081056931564, -8.750720965163202], [-4.685181991517679, -3.1394221383869034, -4.505733999919456, -4.7128201143378075, -4.721709061755053, -4.800636320344724, -4.867216645426291, -4.85864761270119, -4.820149134568306, -4.881081685563463, -3.6610088191091923, -4.523883318425133, -4.405422178721609, -4.511747247142908, -4.4973757423052545, -4.410859930021738, -4.692381446660533, -3.970040309575452, -4.061847858828575, -4.724689689893191, -4.678033998252392, -4.350556098635668, -4.167208374486401, -4.450909776342023, -4.644417387453407, -3.724277324458299, -4.630737283549326, -4.509337607422754, -3.787718752781497, -4.309323510802315, -0.6823718671879156, -4.449775346271411, -4.306372205167735, -4.410859930021738, -4.920232593125288, -4.3914208729895154, -6.478377211171838, -6.487035273914953, -6.801150603434889, -6.531487036485786, -6.875258575588611, -6.039010551387992, -6.9553012832621475, -5.362670489144429, -7.042312660251777, -7.027274782887236, -6.983472160228843, -6.58757650313683, -6.732157731947938, -5.820321350423163, -6.504579583565862, -6.710178825229162, -6.395685495326725, -6.710178825229162, -6.559138567816296, -5.802621773323762, -6.789316145787886, -6.8131267944816045, -6.29749346914751, -7.1889161344436525, -7.012459697102096, -6.888161980424519], [-6.973841065843466, -5.090695336189662, -5.341561221116127, -6.0115634498929555, -6.202945132409144, -6.628849553394376, -6.911669409716161, -7.0009992844957765, -6.886081347969349, -6.711294966663227, -2.260331825727248, -3.839334077735597, -3.9667628422059718, -3.420241689905642, -1.8735787833427253, -4.188499103605226, -3.691171678101274, -3.8593630820739957, -2.3524981891706576, -6.231341006564324, -3.7662373591100775, -3.6465568924129386, -3.4963200738799767, -3.4321214183238204, -2.593050419274572, -4.346868167488547, -7.8364379197408365, -3.990422275679548, -2.904818352752614, -2.3714612152432646, -3.1378637759089556, -4.357037863124861, -4.651314292504279, -6.760867254793699, -4.107313187039242, -4.3976920060255535, -6.440759053046417, -6.465477098310146, -6.582182413654962, -6.610599862587676, -6.978263755093254, -6.719449335464573, -6.730879435939417, -6.514035039942995, -7.843934206718764, -7.66305587798569, -6.780807453646412, -6.810345466083276, -6.347157031161333, -7.321885120582923, -7.779113558339937, -6.602957069625781, -9.658525717989464, -6.893034874465941, -5.640541915214837, -6.61808064554899, -7.825297811959789, -7.623352248005446, -6.729273803531229, -8.625119353004427, -9.394209194493895, -8.033505455437194], [-5.604855175474509, -4.462285490693087, -4.571350651692403, -5.056493889833258, -5.072440143885144, -5.681064638387167, -6.280536665394942, -5.74786868753166, -6.300402104581103, -6.443541857970619, -3.08781816164275, -3.6971443171127936, -2.396573090959646, -4.216785762291264, -2.055069601332964, -4.2400357108180975, -4.510301966657801, -2.8725223515174108, -2.985980079879496, -6.151031964062443, -4.17853136290412, -3.739735408911567, -4.177743865892257, -4.758224336321508, -3.406169247786832, -2.7407038589797272, -6.9336687884910315, -4.884332887441078, -2.4403221799860293, -1.8263508274951405, -3.4054696763623764, -4.562592776233452, -4.420217843796468, -6.782722037918705, -4.9372619598181355, -5.457485531187882, -6.256287610141627, -6.611108985431037, -6.565907361549403, -6.49534726380481, -7.061567128112425, -6.599672447239019, -6.7013251912547664, -6.80746927442344, -7.516987128232732, -8.06324629910308, -6.809992050813993, -6.860502751231873, -6.646463246903641, -7.480934599971211, -7.54322475844959, -6.575185095427639, -8.985698479921282, -6.994899040702809, -5.767168821265772, -6.530694737719788, -7.839611467995152, -7.435063610819411, -7.091053743823944, -8.707954377459634, -9.10781723001913, -8.370947238445352], [-6.890344120648169, -5.02940421086846, -5.408867487897182, -5.967857793669728, -6.047653312106489, -6.46237897058548, -6.749954722267399, -6.888388607925276, -6.98955227182629, -7.21719797180092, -2.4868466816371, -4.771074471877903, -3.6481121610473157, -4.664656454469179, -1.6817234068168874, -4.586866350603287, -4.845552155508254, -3.154508356873059, -2.57905575846229, -6.254276196451527, -4.921506816489567, -4.4705470709445265, -1.6402347840968394, -5.1280286653720175, -2.882844123251368, -4.595878517780293, -8.231899529453244, -3.131328577959143, -2.841674626039629, -3.3059284308174086, -3.1776616460738145, -4.478455963747706, -4.881954395151054, -7.651052083171771, -4.569886902888001, -4.27044926534142, -6.859624364774909, -6.620591583748609, -7.035724951332805, -7.0226528697654516, -7.663432064808081, -6.880725735179707, -7.096836038369936, -7.001238775261635, -7.94609099384926, -8.742881147291776, -7.2506609077408095, -7.3153280695559095, -7.053568548019118, -7.728589206441718, -8.033869830584925, -6.95515998225557, -10.250083546611185, -7.317576103473792, -6.34314386192865, -6.83930777142871, -8.443181097280567, -7.8493114938202, -7.476614929901965, -9.60010853791645, -8.54213954241637, -8.677952686589895], [-7.97530600380511, -6.415691397743754, -6.861539964259856, -7.354178676035024, -7.5544322703441, -7.956843940965374, -8.00285595557334, -7.941715059369074, -7.997284910523884, -7.042078762400056, -4.593656623473984, -3.4358427231129607, -2.813804032113097, -3.8443096208464707, -2.253236786554918, -3.523372060796657, -3.771274119941123, -4.213005093082092, -4.26676474950711, -6.604661414352346, -4.071781743539046, -3.0636392294004997, -2.748766550188664, -1.5979085155684218, -6.096125476162904, -3.8319741165296572, -8.437883856353995, -2.362166563210972, -2.079080404566457, -2.423752746988421, -6.1373971409737065, -6.112261605870961, -6.033619491127531, -6.187046440930159, -5.880551407088361, -5.508879098944167, -8.771401153617497, -8.582218506443597, -8.081702252336468, -7.820268405959624, -9.01805471242806, -8.991966276343762, -8.037776271878622, -8.339328192194554, -9.20125771484982, -9.932145223392613, -8.711282886564021, -8.633361864553784, -8.75433589306331, -8.724333642759511, -9.49070501282617, -8.37573721938598, -10.194509487860104, -8.552742688310644, -7.968342726152964, -7.956843940965374, -9.863152351905661, -9.598817643450872, -9.08877973142313, -9.278218755985948, -10.729432663205156, -9.20125771484982], [-5.1527222081970026, -5.012025127950751, -5.39388426501389, -6.217687012129963, -6.355002880531065, -5.6351154829069525, -6.363993950090922, -6.903492332891861, -6.853964796193012, -7.129436704737862, -2.71361594954537, -4.917813116205636, -4.909342477322127, -4.834757647401019, -1.3606202815267094, -5.049186428683174, -4.831934627067998, -4.02878090419557, -0.9857025802248848, -6.9583248115172545, -5.928056254423655, -5.1548164498001166, -5.532591229650075, -5.575217341325883, -2.1159432194090155, -4.2897086796604516, -7.362881241689201, -5.200780422733714, -4.007732553855012, -4.899317914996637, -5.732412626945404, -5.409409253401815, -4.570479874572386, -6.004563148213606, -4.806034206660294, -6.602559371961363, -8.008146936015933, -8.264296783798311, -8.114951495245993, -7.067733915701082, -8.653122572902513, -8.40456220860418, -8.35558820400772, -8.362968311305343, -9.101577865942044, -9.242891399008622, -8.693532111240389, -8.468328826963168, -8.348262163915647, -8.71436619814323, -8.123659823137777, -7.872345394856871, -9.1093600063841, -8.693532111240389, -7.279281672853908, -7.536521246398329, -8.924437667890087, -8.99854564004381, -8.061492916721225, -8.91796515338447, -7.579574252897618, -8.90514446495541], [-6.625065979493934, -4.788580474517243, -5.256407824340383, -5.37832984858721, -5.2038978790684185, -6.079823689638168, -6.317086566978479, -6.646320764762257, -6.72855886299923, -6.886116701148103, -1.505097000684665, -4.82588581184553, -5.253602398681012, -4.989577089424305, -1.5758741362397704, -5.185784176269271, -4.916501452893276, -3.8159863951228723, -1.6041923545980379, -5.633841418592661, -5.7199839934992545, -5.2161585690547705, -5.369003667873634, -3.3721189713516244, -2.661522576130661, -4.670108701549678, -7.227790153984301, -5.183916064386551, -2.494756625753895, -3.869077211017653, -4.27104038948766, -6.0954898063825675, -4.71167529774246, -7.136943681210728, -5.987593808088792, -6.590446143960144, -7.403791666759668, -6.814987319540992, -7.030531087602099, -7.451894781778061, -8.262224216961, -7.8389014914528286, -7.199379152151521, -7.871278420177289, -7.002535480112671, -8.630549778119708, -8.06204504276704, -7.169938203345767, -7.199379152151521, -8.169374062997537, -8.264930578558742, -7.090662928646722, -9.550754409315, -8.033689817011913, -7.078178371984477, -7.674437552058881, -7.921898411023797, -8.116604027017473, -8.273093889197904, -9.39660372948774, -7.831846788472939, -9.063739434242727], [-5.33959692367457, -3.901875433911095, -3.3263523840890916, -4.682859878096868, -5.0569946532038434, -4.833678156338227, -5.439237690659609, -5.4278954140556746, -5.662170026427569, -5.462315637942154, -3.751806551579704, -4.660118258928943, -3.8333548860836233, -3.8811153084238335, -3.4958051088141735, -4.553750303632283, -4.534932311997922, -1.8458046740483725, -3.0281790617647277, -4.436160927561686, -4.943995470762378, -4.549402848736927, -3.9218705969950047, -5.328568099202016, -4.452091297246621, -1.4302390343932077, -6.080922600826194, -5.1148485422425125, -3.3219189169449392, -2.621905648275213, -4.3789508674983555, -4.262632175345694, -5.088920047216344, -3.3849839728869298, -3.307281062205608, -5.974439120423745, -4.379829216259262, -6.791884017661267, -6.8166991867809905, -6.068955964208674, -7.1565271312491765, -6.683586642330131, -6.801736314104279, -6.532793078538745, -7.518256845849533, -8.152370494825284, -7.391505140210389, -6.7194113688482755, -6.416271872702784, -7.279027156783699, -6.4871905993829495, -6.66039719741122, -8.204669994228134, -6.878895394650897, -6.094660190607558, -6.427558674237414, -8.084652320770335, -7.339319387039819, -7.347830076707727, -6.950948712290955, -8.303341521735163, -7.468823387994359], [-5.431835629645527, -3.900968951139421, -4.151004765492447, -4.646528675532386, -4.521804796114552, -5.283128043243618, -5.29007437093391, -5.66335521876192, -5.740643416211578, -5.366660147370001, -2.670098748915231, -3.286242237340954, -3.3585141055181613, -3.755602707715123, -2.5586532961090027, -3.6926656242369105, -3.8330192649391925, -3.254475467118251, -3.8290307266838655, -5.019290301102854, -4.363753678748955, -2.9684754808494547, -3.151618081678543, -3.50657051212598, -2.895083629045828, -2.91665360144011, -6.285849431998168, -3.6228229122699127, -2.152751777383897, -3.3162167836347605, -4.2488413058475105, -4.652746019284857, -3.9288228848114928, -5.349158624766704, -5.572231075081914, -5.261791793324819, -5.624778734167247, -5.278347157643276, -5.3043854022704995, -5.210237912616575, -6.346428699255328, -5.93048158286466, -5.615084400228674, -5.89255288484716, -5.916841563358977, -6.777270925569916, -6.237681480853062, -5.483828321441287, -5.599254877723378, -5.735607736245962, -6.332645938959907, -5.443956990177872, -7.787390135984574, -5.974679054270741, -4.487700164671342, -5.484479363130949, -7.787390135984574, -6.624646575217306, -5.915838554247808, -7.256570295689133, -7.872330812633658, -7.3300889926342885], [-5.8962959205599565, -5.022482812723318, -5.262967235210922, -5.638331471894197, -5.778457744433211, -6.3109845765456285, -6.29163873510587, -6.56832765910669, -6.49255510063436, -6.3650100589758205, -2.863492599093995, -4.089024629780227, -5.154949557468251, -4.484159695248834, -0.9343644420039627, -4.498161263246394, -4.6279862283262885, -3.982370058053253, -2.2035282537140866, -6.199482179547703, -4.571665803362194, -4.084960905072187, -4.755413554835734, -4.800997636581884, -3.6971982708382995, -4.566795129611478, -6.871351961660363, -5.099526707333125, -4.135788994836005, -3.1706024468180507, -2.212641008744242, -5.078715683958774, -3.5811841558979225, -6.741940888008861, -4.856098125468122, -3.980907107020626, -6.712325967469931, -6.644006723492454, -7.153784678226001, -6.945055157447498, -6.898020208742524, -7.021563043483541, -7.173168545047049, -6.854391346353318, -7.669859657878135, -8.284314569862415, -6.895321151773359, -7.513205847832758, -6.795366054682441, -7.556635405760094, -7.827916592672458, -7.241272132349116, -8.35137880044296, -7.187505708193456, -6.1188193168155305, -7.139919638088829, -8.075324765986299, -7.379987004046535, -7.364785842714484, -8.480789874094464, -8.527936652520165, -7.717773013574032], [-5.1596391409467905, -4.446625879300866, -4.792661465824471, -5.092676376197713, -3.486026851337387, -5.293817992318437, -5.4207097441730605, -5.031624556083574, -5.433394903700377, -5.123410202650332, -4.715036303793935, -3.7021536789683007, -3.7870302329121537, -4.088809455118969, -4.812132309601868, -4.898021981482298, -4.704305701448848, -5.045386241156256, -4.694501701352228, -7.439823297251669, -3.3979941838940024, -2.7387677229303073, -3.4751598040428164, -2.0375451283648167, -7.179408351140228, -3.811103778633784, -5.971540702552751, -2.9729698958508197, -3.825149191015764, -4.478865937874288, -2.6641628726801234, -5.3193066285353545, -5.996634270150407, -6.375549146164451, -6.785390628872414, -6.572829186888741, -4.124487206195906, -4.535138151336436, -3.7135200209337587, -3.96463850525401, -4.554386339641352, -4.765674647825141, -4.429638997109137, -5.277664924510665, -5.02086262836912, -5.937153360243275, -4.7565501004863675, -3.693750268171921, -3.749678820831146, -3.176252392796863, -5.722496422806384, -4.546642548744449, -5.417354036326088, -3.1803577086920454, -3.749362815140414, -4.066652137558128, -4.660532401294013, -5.1570534898617675, -5.430844966067104, -5.284241491875228, -5.5748604240003194, -5.63236491848051], [-2.1543734650260515, -4.349659150265929, -4.518901599180162, -4.334150339570116, -5.146931928181808, -4.726677895862385, -3.8738862022041083, -5.547738126032732, -4.965611539500847, -5.329386916158828, -2.0510193107742145, -7.017067715762613, -6.5514024881154755, -6.782731624016124, -2.1387030138865364, -6.887331394106641, -7.26098839501858, -5.915573580719614, -2.792353319034718, -6.475549145597215, -8.196805549243091, -2.7290425937620233, -6.57338139483425, -7.482751683863832, -2.7069370493135643, -6.6217549791184664, -7.959675755954141, -2.6929063910157374, -6.402701413055781, -7.378081061299943, -3.038722067403394, -7.146279447242619, -7.442205589469482, -7.777354199160186, -5.436617340201765, -7.9268859331311505, -4.21410095816366, -3.7159537926574395, -3.274052690014408, -3.8709022387814955, -3.903841048493916, -4.907248138723605, -5.564815386850924, -5.634025390029099, -4.794669640998671, -6.37044055083756, -5.773624479216047, -4.984713243886536, -4.705003878814023, -5.884409585684786, -4.869643914101749, -5.8105008762596295, -6.426399204485604, -4.552569937605819, -4.688840192155229, -5.1496039404596194, -5.387063525747035, -5.940338138344011, -5.40158748887255, -6.8282736444630405, -6.302506440698851, -6.49077786629904], [-5.042027972011844, -4.299718672656642, -4.243324982365076, -1.9825142998026875, -4.725280240738409, -4.751810055033314, -5.528415080508104, -5.372269658882421, -5.58057814379269, -5.634199235231788, -2.092585085927561, -5.359310514239916, -4.924244936355461, -6.182928172882705, -4.358300013368391, -5.644135441890917, -5.3193472574280785, -2.4216711986248063, -3.5454438576615357, -6.6553797837670805, -6.406683158028254, -3.42063060974259, -6.230496589801815, -5.313180742512414, -2.1141313157147192, -6.38251279710044, -7.199431055161392, -3.210947634389801, -4.823896962521179, -5.599497627038855, -3.7500872171087103, -6.903338504201014, -6.960201366095558, -7.542682978767983, -5.613234106766742, -6.36476085164199, -3.6714819678600925, -4.875454772186028, -4.20471280907303, -3.884017303250544, -3.4828969984685907, -4.20471280907303, -5.449056802096191, -3.411103801364231, -4.476181634325785, -6.299489461288767, -4.386290645201218, -5.042027972011844, -4.789932598747867, -5.333885415874105, -4.1803254304220925, -4.950988261330972, -6.251222720318932, -4.73156059252596, -4.355150404465494, -4.750641830601388, -5.174666905853347, -5.78646844695934, -5.852762894799176, -6.180486169227153, -6.353098911894147, -6.096324376905443], [-3.621153540136533, -3.927545190670303, -4.761298021301781, -5.2327070159671, -5.390836480884121, -5.489330016767771, -5.419938023343771, -5.638503085521277, -4.333907884333748, -4.5925613698517225, -2.3774699118392917, -7.240027999198052, -5.856199730573355, -5.309006462636489, -2.1188535037876086, -6.988713570917146, -7.259929153515347, -5.733217613565709, -2.1988937611323722, -6.914879308906094, -7.285375819176511, -7.471829613255376, -6.297483936486472, -6.369085087302357, -3.0144535413397078, -6.131611710433321, -8.510182710405289, -3.290870578196505, -5.79050447130796, -7.191941812530414, -3.661034886200146, -5.933160771709483, -6.992552347224311, -6.430741168725453, -6.006448567479316, -7.6405790925037875, -3.820123280601132, -4.60874181430768, -4.552214016916188, -4.152125265844404, -1.8440074384079603, -4.33229280995845, -4.748982594711727, -5.590043992369601, -4.083551769494654, -5.885005729822436, -5.570371226770897, -4.35431953325285, -5.037889407059404, -5.600505333614971, -5.2234842891983275, -5.239347146606377, -6.012203979049936, -3.2506791222758458, -3.493367719618491, -4.386449689183688, -4.5929103490909995, -3.6635121789617022, -5.259536257638402, -6.071656848963408, -5.913358144413303, -6.637031915632404], [-2.697794466436189, -3.604662393389068, -4.011263957500705, -4.728080882146555, -4.781372179196779, -5.33155887684335, -5.439355447299057, -5.586922158039964, -5.370633615250003, -5.513150772897161, -4.398260316869362, -5.11207163673225, -4.190082883743458, -3.4501177344839546, -6.597640067453566, -5.490434815062624, -5.150911470048515, -4.837107457567751, -2.85030853825846, -7.320346050255055, -5.997701716211621, -3.4753179136511876, -4.2866759373606715, -3.397053452568678, -6.703360230660674, -4.113003141200578, -6.444877312901155, -3.074043686998346, -3.452623303723489, -5.193702907090925, -3.9327969395172757, -4.131242383720524, -5.418238523858134, -3.7478131640115406, -6.184652086682414, -7.063123185287982, -3.7394230709797665, -3.4179694218787144, -4.267288710081795, -3.0709940324857143, -4.263518677341253, -4.701005637323521, -4.417094629495229, -4.569236359692398, -4.473887479448818, -5.8432973258667005, -4.728080882146555, -3.754953367214181, -4.1263095154005, -3.415012032977706, -4.81618158386264, -4.691820172692138, -5.666572324000903, -2.795735372125381, -3.2437129699651224, -3.9019638438387987, -3.582993942690341, -5.112804505947904, -4.894364079117085, -4.722110715160052, -6.056219323109372, -5.5022692727096265], [-4.619141531251806, -4.242478414047348, -4.655112011634125, -5.260049492712331, -5.018630839221239, -5.434973330528597, -5.291682300504695, -5.652426966065128, -4.133100006910822, -4.834429582625707, -1.998550183209591, -5.7433143168994505, -6.845817661060527, -6.562531200081784, -2.4549848852751466, -6.626189051853761, -6.290387731057782, -5.567486289416045, -2.141146196531367, -7.242549089199937, -7.104679294976816, -2.8517566389677937, -5.192894636686687, -7.082206439124757, -2.3092018462234853, -6.71448165899944, -8.017189304011065, -2.1705324264442973, -5.663122255181876, -6.392769837961348, -2.584044215625091, -7.28700085177077, -6.816997222525035, -7.454881724409931, -7.498844847831047, -6.3210545294507545, -3.9234738704665255, -5.210974225191004, -4.156261002958894, -4.841496749848799, -4.597299789336757, -4.421170202790724, -4.975365923256807, -5.672123216040852, -4.474317672329112, -6.090327349926446, -5.367408011032831, -4.491383872387231, -5.280618880960439, -5.815780620152271, -4.640826054386648, -5.541761398177608, -6.307308608546119, -4.360911011272527, -3.968855752517438, -5.060904184340462, -5.092811178129705, -5.9935197902566175, -5.119506808592894, -5.086750553518014, -6.052587021943599, -5.38489332537385], [-5.261046281621696, -4.9148297535258045, -5.093521542269181, -5.160962823064713, -5.492794680129424, -5.583178741597693, -5.608843109973599, -5.828958322395465, -5.818217080564052, -5.6351835265880075, -1.9431063436877503, -5.762127613913009, -6.364566555601998, -6.310693565661846, -1.465488331318766, -6.653479397439911, -6.983078754773362, -4.701955198551832, -2.920630198813372, -7.381717897811127, -7.114655112562081, -3.628618850962897, -4.741931970713727, -6.050483313874263, -2.961525855445553, -6.683481647743711, -7.823550650090166, -2.1208799042866553, -6.236585593508124, -6.735576759627112, -3.3232120191551315, -6.891511561247872, -6.08336298799342, -7.9413336857465495, -5.554867108771802, -7.122498290023107, -4.193090909121505, -3.901105949199606, -4.152689701478908, -4.935827900365578, -3.224590180736388, -5.347946392964479, -5.205112607677643, -4.550306897288771, -4.485129674572183, -6.303724896345753, -5.654950567328784, -4.928779413034516, -5.0770940681272325, -5.30867433061095, -4.570963767879653, -4.7737511552658995, -6.146602446129337, -3.6472116840395126, -4.583317229489316, -4.247999881283233, -5.050961927850385, -5.554867108771802, -5.239253651630843, -5.752077278059507, -6.0889495947020595, -6.114482896707225], [-4.8357984388643835, -4.657872493543467, -5.438431611883497, -5.481947249319659, -5.480513557317811, -5.739017089171186, -5.670890393572083, -5.859121927588727, -6.047318448825703, -5.930658377913943, -1.7234534048021994, -6.808376835001858, -7.018097365983928, -6.515303913915201, -2.0800327647225068, -6.985307543160936, -6.434021280478344, -5.690138581877, -3.0546210789668655, -7.433612809945593, -7.609892052283585, -6.846949109788098, -7.145930737493813, -6.771237288052402, -1.3242305561151027, -6.88123818326673, -7.934388097858083, -6.599387031125742, -6.8029859863669815, -6.569147145936024, -2.3927324727103167, -7.200418922777882, -7.33738477785104, -7.62201341281593, -4.893045313052039, -5.366877919534872, -4.1899951004809015, -5.453656819480563, -4.694403309779647, -3.431804500645092, -4.0124147615767685, -5.525942614164559, -5.309719505694924, -5.568828205702649, -4.124028746206436, -6.415604553692113, -5.4275025413513065, -4.984699762805498, -5.237511197353998, -5.157641365469638, -4.101047614464405, -5.085610237260755, -5.550223017871613, -4.869440755282619, -4.489173830779152, -4.234321454353582, -5.041242413079193, -5.780689785571754, -5.231908941805328, -5.964947451392574, -5.752089170738539, -6.139854701747302], [-5.508465007700513, -4.839258034686576, -5.109472414097966, -5.519275923804729, -5.707703038161169, -5.439902336263216, -5.7235763873174585, -5.821556795677663, -5.734300350680434, -5.610247702010455, -5.910709501108635, -5.7235763873174585, -4.435262434557772, -3.8610478472011964, -5.758858201462099, -6.0632537548876195, -6.044700346991872, -4.117878794499082, -6.575328598054043, -7.243783166023616, -6.403478341127383, -5.083829983484629, -3.317712944869911, -1.7163886672099804, -5.622223893057171, -5.543479525545615, -7.373836294271814, -4.235327303674574, -3.4757995308184078, -3.8216678856997803, -6.863010670505823, -5.619817153026606, -5.966922645949188, -7.096625521687328, -7.1845942946332855, -6.5629060780554855, -3.9611313057581787, -4.164449795424729, -3.6745940531223313, -2.809054169189011, -3.303860323066671, -4.356804692686432, -4.075943929085389, -4.993775889109478, -3.5055329961999946, -5.4956443192714515, -3.896020865817486, -3.604273312212492, -3.5622476861654384, -2.789152302491872, -3.9102728885246876, -3.0660244750660355, -5.631909198791635, -4.404134333185408, -2.9033092297609358, -3.155554830818952, -5.387104150696246, -4.341290047595106, -5.258585299895267, -5.109472414097966, -5.821556795677663, -5.14024407276472], [-4.364789554126129, -4.918384854279969, -4.918384854279969, -5.1613310328903585, -4.587530609962979, -4.784853461655446, -5.006160465162354, -5.0403784224726955, -4.994002260682545, -5.402493089707247, -1.4796200682550344, -6.181000119218348, -6.701776073837507, -6.252825853789604, -2.1508860947624866, -6.086590434747274, -5.772240115213331, -5.482914657017821, -3.9652464014134288, -6.426672783592586, -6.401671481387169, -6.285261129542758, -5.890845857621178, -6.3772800282630095, -1.9660163181458141, -6.770768945324459, -7.107241181945672, -5.362398632505369, -5.920923312858457, -6.7530693682250575, -2.0597919147266994, -6.701776073837507, -6.505661194911217, -6.5476253940102485, -6.637237552699936, -6.7530693682250575, -3.9354569646113213, -4.092932522818745, -4.814706424805127, -4.961309898997003, -4.355174095426687, -4.949681861001884, -4.869194610089197, -5.046727650151355, -4.8507000274530325, -5.115948027042541, -4.814706424805127, -4.689097748317353, -4.863875448611597, -5.043547997233975, -3.6732539774605253, -3.9481153614832447, -5.340799520701906, -4.835115296436334, -4.068688911208753, -5.024679512929592, -4.753487802377227, -4.994002260682545, -5.247044551164388, -4.698046353893368, -5.307182909902921, -5.227776132298511], [-5.115394338583977, -4.7561420736309445, -5.007842676853897, -5.66123448019507, -5.033665707964376, -5.645764558422938, -5.535809351386801, -5.907471144471458, -5.828086115216893, -6.105296887801378, -1.6832402388074508, -7.437524027650993, -5.590313253886724, -6.862982589498895, -3.0491034044747836, -5.586172461220692, -6.632458930887062, -5.218606625703514, -2.4710973040085813, -7.398809515470303, -7.268447697607059, -2.7116510959031834, -6.686853002952861, -3.7188303108266303, -1.9217106058640063, -6.098376444956805, -7.548749662761217, -2.711068175455316, -6.088084758920257, -7.1630871819492326, -2.4953178542046213, -6.5371487510827375, -6.531815405107375, -6.592107635363495, -5.218606625703514, -7.464192274733154, -4.025855346121542, -5.098759278079445, -4.812664759541027, -5.369662433402117, -3.845996309584077, -5.031290409935469, -4.627804324836165, -5.549657248245595, -3.554062953084761, -6.151486270270753, -4.8426621708268565, -4.640563668589926, -5.069989313821911, -5.128381534110789, -4.6191257693799175, -5.164313543336852, -5.9721996171719525, -5.146851716173189, -4.482065653234942, -4.222339216728002, -5.190396010216277, -5.978297197040071, -4.681519433426748, -6.057830350562455, -5.619789072019677, -5.749664153441089], [-4.867406343870729, -4.4533740975545415, -4.953434176237852, -5.384635998990067, -5.328952912513263, -5.6593669190947695, -5.597013688377717, -5.757806991908022, -5.710273291442284, -6.117674508577314, -2.0752354231922467, -5.330274789771179, -7.239817294655618, -4.653127950557675, -1.9861891376978962, -6.498730651806554, -6.748830013109291, -5.799220777458779, -1.8880861157511444, -7.060476365999801, -6.592340150928861, -6.615981913985901, -5.98705432616025, -6.055682832549346, -2.2283490142554703, -6.244583360441343, -7.060476365999801, -6.9543698600052535, -6.005072831662928, -6.675112437212724, -3.324941220245065, -6.737960340872387, -6.846328377594169, -7.276184938826493, -5.466076330930241, -7.601607339261121, -3.2284338811241238, -4.7571452826690335, -4.2772171644143535, -3.794375211195705, -2.9321390456478555, -4.655146114713912, -4.556474587206883, -4.658518799192551, -3.9133691163329902, -6.159223511490186, -4.874928318325417, -3.9851607325065794, -4.593769151695071, -4.865742449976033, -4.192747084853396, -4.158562847663465, -5.583291346122616, -4.6687056444995445, -4.403457644910037, -4.135470875670939, -4.338591952168042, -5.484425469598437, -5.10713123845697, -4.666659956776251, -5.507845743806536, -5.56639905255811], [-5.367738120634485, -4.778432275797196, -5.153988612512787, -4.9880684755260525, -5.022992659762126, -5.315533041364984, -5.937669186281741, -6.10637419308453, -5.985149724514735, -6.0666028696576335, -1.4917500304139726, -5.449683054485573, -5.538947192058466, -5.596026010244295, -2.1356770840200325, -7.153812526227932, -7.094389105757131, -6.013018623523528, -2.2775252846627545, -6.010847070010019, -7.246405313055757, -6.755173383190167, -6.697507741340358, -5.530860894627108, -1.9981251565724831, -5.698525293621089, -8.083802102460249, -4.860110306811463, -5.882753428685626, -6.945748081740843, -2.8863534391722627, -7.100819996087421, -6.962461562714583, -7.231590227270615, -4.4224533361891325, -6.0688990819179836, -3.292438554454217, -4.782234561746934, -4.459461169483883, -4.359771877270199, -3.9163352754091854, -5.15214868559078, -5.0863471755948115, -5.622152314836516, -4.041658687615878, -6.321199217523649, -5.22160122153078, -5.082053480720112, -4.742403534262838, -4.896452531238822, -4.577416633798337, -4.325814840936012, -5.22258692885554, -5.570564946046022, -3.7753199382595333, -4.751597592285044, -5.351900685535859, -5.538947192058466, -3.6838339356894148, -5.717756655548977, -5.9396873504379775, -6.193602560418941], [-4.999514188504179, -4.700587505853424, -4.67022803135639, -4.81576740495976, -4.92872993147181, -5.418517964945142, -5.4525477135314535, -5.471965799388554, -5.601583292691941, -5.294558257397029, -2.0675057720915833, -6.606510097282728, -6.66295140818768, -6.268297216183731, -1.8472066532219233, -6.801101746668497, -6.464629510047284, -5.518112402024337, -2.6153829107633095, -5.40926720517299, -7.042937430994859, -6.122880216325227, -4.984950824316282, -6.978988706394586, -2.5341534752438712, -6.335738496979263, -7.22903971062872, -5.047082605423289, -6.606510097282728, -7.195517018590076, -3.3881221963080814, -6.764734102497622, -6.935691900641261, -6.06802185577318, -5.518112402024337, -6.21743879895024, -3.306166689176508, -4.5835098665078435, -4.096018906794089, -2.7778687007936327, -3.149770331333854, -4.522505810044474, -3.6065186143693406, -4.579485716208118, -3.9009018998360694, -5.695109350702764, -4.592420415540891, -4.39850374434322, -4.858371250530563, -4.687974726037726, -4.284600731462279, -4.828050056815601, -5.7714823294873385, -4.760362534007699, -3.6910492160852075, -3.1337486268025883, -5.114702793098268, -5.044521784561615, -4.990993138772245, -5.21262268525524, -5.1495981689488834, -4.939033399841534], [-5.573013579156251, -4.7557456323553735, -4.835888625843566, -5.4310032481645445, -5.375816832597254, -5.748275644237248, -5.617589203744957, -5.426286258286405, -5.527732874623095, -6.027336973645122, -5.771265162461947, -3.4363190171520683, -4.863649827007062, -5.342810536129083, -3.9603112459777443, -3.1741279621323484, -6.522548369609561, -4.089429329592385, -5.667234434234122, -6.573100648772392, -5.74180312973163, -3.6814950844272993, -4.555731535734934, -2.60141208694855, -6.057840427938536, -2.6538446056843625, -7.3352407008192895, -2.4633202784204524, -3.0508881488790776, -5.069846636151324, -3.6495408996227328, -4.548858656447172, -5.7613477258046, -6.031638055544512, -6.61856302284915, -5.764642621701453, -4.906092884516815, -3.945751808422034, -4.075923906392578, -4.145352412824341, -4.3284585910787134, -4.1375398730875474, -4.386633312115669, -4.864991208831262, -5.2459274106602996, -5.389330551763976, -4.366047982499911, -3.5236520927129136, -3.6216686341149815, -3.0467401241562313, -4.4457511661828155, -3.28056139498962, -5.728982441302569, -3.2282776746379773, -3.3483163872180257, -3.6120365041642786, -4.280940073114231, -4.240597235287379, -4.439591885822281, -4.571620648562269, -5.13801612348307, -5.583972592945972], [-5.648871675811755, -4.741030782968909, -5.070478143556586, -5.038332518583169, -4.984450311189462, -5.652113169735926, -5.878916005474925, -6.018576120040821, -5.698633185370818, -5.924757364823166, -1.8468585970227798, -6.844251516414859, -7.126600242891277, -6.081398705962876, -2.934905888757429, -3.6595354362414967, -6.817864761241664, -3.5921374614243615, -2.9623018840998165, -7.15537920744132, -5.837035508229937, -2.956367800752568, -5.44072677540776, -6.135188880549014, -2.35874447462609, -6.817864761241664, -7.744626792983114, -1.829808862763535, -4.656579250518651, -7.012765100242478, -3.346772137573546, -6.156466278996299, -6.772055225210369, -5.459294948136628, -6.2464145156592386, -6.772055225210369, -3.9054574859103, -5.259720143195114, -3.484916480113615, -4.59888775210554, -4.26703708287839, -5.280773552392946, -4.568219053541838, -5.236954932422101, -4.227597595795837, -6.429913235626207, -5.14878310004672, -4.299244656170703, -5.066854951187166, -5.722730736949879, -4.343868104023761, -4.895290258966649, -5.924757364823166, -4.2366225736701635, -3.439140235431567, -5.290903070631802, -4.913738130330258, -5.32777360644013, -5.700335313441349, -4.118883335232663, -6.285399962372192, -5.825384891009962], [-4.134798672989471, -4.089847285127205, -4.1376763728170864, -4.175860159787245, -3.5867437074551245, -4.073498147125676, -4.247208240584712, -4.485742942219219, -4.161001045383496, -4.76664532768562, -4.782994465687151, -5.224827217966189, -5.79734641073752, -5.371880635922686, -5.362028339479674, -4.805217602471861, -5.696541711615555, -5.1428140663053545, -4.243997964954463, -5.696541711615555, -6.074978147335799, -4.388579193765571, -4.887134724939747, -5.433124261163405, -5.127188748402274, -5.465212575714905, -5.5681605449673475, -3.871109027280912, -4.3522115495946965, -4.887134724939747, -1.4812734394569864, -4.9697213159490214, -5.42265296129611, -5.5681605449673475, -5.696541711615555, -5.592551998091507, -3.9025017396887742, -4.303421385425264, -4.377529357578986, -3.2036759521599856, -4.414847120586181, -4.370230055097374, -4.283218678107745, -4.293269013961246, -4.570900750559526, -4.639893622046477, -4.473572406598963, -3.944368319081564, -3.9659778034144195, -4.663991173625537, -3.393956618621509, -4.489832927470744, -3.9232159440763374, -4.221810049979101, -3.677082874537429, -4.095356940938174, -3.0285527594662907, -4.570900750559526, -4.348646483430199, -4.449666885745409, -4.481669616831582, -4.21556002963393], [-5.468543074118657, -4.3409301123745765, -4.939132602788878, -5.030955774234888, -5.486636817561356, -5.444018684255573, -5.476853371252285, -5.808607963491769, -5.908385471304435, -5.666153323427702, -2.5849374499856705, -5.739356727450997, -6.452595908093631, -6.391754248837386, -1.3630653142736813, -7.218432822021854, -6.741508749931544, -4.321907198551311, -2.3295168721177877, -7.554905058643067, -7.600367432719824, -7.660265574300893, -6.068262723688894, -7.064282142194595, -2.4089621725210324, -6.261701120589188, -7.218432822021854, -7.400754378815808, -6.405740490812126, -6.66108718262097, -3.202908170420881, -7.345184527660997, -7.318516280578836, -7.17951740577218, -6.2346317986209705, -7.685266876506311, -3.523815890500982, -5.001129001779655, -4.254551658580372, -3.1791480369827805, -3.272238460048792, -4.979751531007288, -4.696520976240299, -4.996816791561475, -3.7821441205484283, -6.068262723688894, -4.767838317222632, -4.961310103104565, -3.3169412686460493, -4.254142074302682, -3.686717843919221, -4.781622132973809, -6.395232513213711, -4.882290151848487, -4.035101934573064, -4.142657840794326, -4.886134831709296, -5.154991621688459, -5.261248596393322, -5.763145589415012, -5.307128202144016, -5.4187681732866855], [-3.7153348719791315, -4.23835637018879, -5.401341822408315, -5.655381048015406, -5.3230099791384085, -5.723838105173554, -5.787159933132683, -6.191069830974041, -6.118044695959151, -6.512733837722823, -2.85919837658876, -7.902248813542757, -2.4061387894065027, -6.693770047296114, -2.250325724429897, -7.350962550860164, -6.005451461342365, -3.2057646919026945, -3.1269276063809657, -8.282234836349936, -4.929356031958588, -4.893377430923357, -4.350779231381914, -5.039126698093967, -3.0950598675329433, -2.702149075140166, -7.065140944061303, -6.4656293685353505, -6.6671525062961585, -1.5532677412410396, -3.407685858090993, -6.917395410231305, -4.818860209735889, -8.330553413620743, -4.697144805661785, -7.44034239452964, -4.201345166462018, -5.036551747250591, -4.447052124631202, -4.271306064673767, -4.3875307527168115, -5.68024420720805, -5.463555006909888, -4.6680077696105, -4.729585831829687, -6.877248342217893, -5.7894972929813875, -4.744562286019245, -4.863271676535759, -5.996776708148691, -5.115645847928164, -4.782587554878788, -6.69569868820252, -5.6198664278662305, -4.372289459396023, -3.8304137647360905, -5.143464614214886, -5.692911512408815, -5.442384299786921, -6.144390895990383, -6.553756817069401, -6.771887826560066], [-5.439192275092616, -4.781176186054329, -4.766861598657258, -5.578787193555129, -5.543563426336324, -5.577507602849504, -5.742587353208953, -6.240425781448133, -6.11031213333427, -6.461710020172159, -2.6665339561656087, -5.7471225083743445, -6.597455272456702, -6.655866034613116, -2.1677366785812078, -7.063212610820987, -6.615344837207477, -1.8542823223629723, -2.8203019407701473, -7.921874229858505, -5.964600322152877, -7.4190807777897785, -7.115398363991557, -7.50316389500032, -2.434370037607787, -6.75042461723813, -8.501692725111447, -2.436966652419961, -5.508344242912733, -7.262628600974241, -3.3318853658660776, -6.320468489121669, -4.662752576421778, -7.96269622437876, -4.3951216252530045, -7.773454224740232, -3.4248833433841024, -4.6171877485771935, -4.856615893655904, -5.148452521314832, -3.262215858586344, -5.067705520626301, -4.995134827791466, -4.808655260848847, -4.056803079326151, -6.372894286461519, -5.283769734998518, -4.869131701587735, -4.356292992089062, -5.580068423711178, -4.205055407055328, -5.300108397353307, -6.413362236014366, -4.081705328198343, -3.5651654031689133, -4.4607321960690065, -4.914213385439241, -3.3562768189902368, -5.301077859377855, -5.2191716350828905, -5.66568217643417, -5.148452521314832], [-4.692472771166889, -4.3008934499719516, -4.549181741142988, -4.955265087292482, -5.083773336506403, -5.36724487642184, -5.286555965171697, -5.606625988867734, -5.3561026998685985, -5.6766361544403825, -6.310506725070788, -5.345083304618988, -6.354631529979725, -5.436838168221035, -2.778870257014558, -5.888394383529467, -5.518132910614282, -3.971194340356585, -5.541198183545278, -7.353160360090853, -5.418980550821029, -3.590537431731367, -3.3241709789128793, -1.8944436557320132, -5.91226186493611, -3.390599447549114, -7.2559966116372046, -3.6807421830309885, -3.742794171598555, -5.40143624117012, -6.914247317915148, -6.268246915780905, -5.966865998970962, -6.802329401711162, -7.18453764765506, -6.063197107909394, -4.040088147996806, -3.4233375319614976, -2.7095546985241823, -4.374927246403353, -3.562959147374605, -4.404434128912793, -4.579296940419071, -4.391681565015401, -4.438151297809303, -5.606625988867734, -4.3282656378478706, -4.1449534915722195, -4.0950638220693705, -3.3913758451036427, -5.0164696546102965, -4.409796072054178, -6.030044900592494, -3.187603169643488, -2.500811269450856, -3.4121731416766146, -5.192891489277063, -4.804034117568558, -4.273312053475689, -5.056716535118889, -5.514880875227905, -5.219114684376165], [-5.262101982971463, -4.751704716461968, -4.4750082149096455, -5.006514043913175, -5.07515161250067, -5.251452255054804, -5.285938431125974, -5.392274080942006, -5.414469813333791, -5.840086736879263, -2.606359904105076, -6.823463761969788, -6.89757173412351, -5.234645136738424, -1.8016665052081617, -5.931967688708849, -5.7146991018376605, -5.437169400849897, -1.672652532530341, -6.72630001351614, -6.773702252410724, -5.308112288620296, -6.00575385876808, -6.6810434219280195, -2.3521748840468923, -5.96169386897405, -7.419447194076086, -6.095225261598572, -5.008173795331539, -6.397795946544104, -5.514028406375509, -5.907173030095633, -6.070893160939042, -4.572171963363293, -6.91979487090822, -6.823463761969788, -3.666029218824578, -4.169934399745995, -4.315857524665681, -2.6071122929751187, -3.822642418908326, -5.326212330263914, -4.3600448319184775, -4.917707706137853, -3.8075287810982776, -5.953110125282659, -5.187841070540621, -4.792366055507543, -4.8403533292283845, -5.164264339976624, -4.320022461964966, -3.8952676954062846, -5.745470760504414, -4.036941945426083, -4.228361366337563, -4.708930900478682, -5.247223918945283, -5.006514043913175, -3.8952676954062846, -4.574322501826521, -5.886970322778113, -5.748949024880739], [-5.49714261021104, -4.668772759709356, -4.7793028170607235, -5.269011517385791, -5.403728165571614, -5.600190986110487, -5.607143505425369, -5.736034518493389, -5.749350294469161, -5.980569259788917, -1.8463761496649909, -6.7957313909118255, -6.842614976810675, -6.277301167760615, -1.6016725181239624, -6.834646807161499, -6.483542908272222, -3.3781298559126016, -1.9147238137262874, -7.264209466848723, -6.97044834832056, -6.2503937148406905, -6.323821183395508, -6.466922027036182, -2.304012035094697, -6.232849405189781, -7.182292344380837, -4.500040235600472, -6.194465162181466, -7.045955900828706, -3.7335540179496816, -6.633976111699347, -6.224191342446667, -7.560054849939665, -6.541029999189903, -7.7391030813886506, -3.9953827115751284, -5.225208894727398, -4.2623981472183505, -4.983820106246462, -3.5900722944719905, -4.880831478599072, -4.625152137233465, -5.350160419862394, -4.464551422640724, -6.03257404477923, -5.25911044640308, -4.290672925686517, -4.802954305142684, -5.287422079228971, -4.798800612773991, -4.757191363453292, -5.977196575310277, -4.844383490556383, -2.946883968159507, -4.887603513509017, -5.795997983260946, -5.6188395451885595, -4.719156549714065, -5.921535728204724, -6.1778672707724285, -5.694577804815043], [-4.469097220436953, -3.6430767584970343, -3.7844067498952314, -3.931680914923784, -4.272838155799217, -4.129621978555971, -4.297357772973536, -4.384369149963165, -4.338992745625113, -4.178787025578902, -4.976420213651742, -4.292052720743842, -5.461928029433443, -5.887479896108345, -4.57562801124297, -5.526466550571014, -6.262964172004573, -4.4462034005711, -5.238784478119233, -6.017291507630335, -6.355745905455539, -5.968972930359527, -5.520424236115051, -6.328346931267425, -5.70515833931439, -4.218953067304238, -6.6434279779073195, -5.648805402763258, -5.5635078222513625, -4.870360641691417, -6.412904319295488, -1.949196049230905, -5.968972930359527, -4.4990139814749455, -6.027241838483503, -6.442757282445169, -4.057336183871974, -4.448263133534111, -4.000410247075965, -3.7940068236242506, -4.150834895195163, -4.166267300233975, -4.566303934367847, -4.3901887590164295, -4.028468199871122, -4.376661982718227, -4.915384323065373, -3.014652447863089, -3.369082406994758, -4.61381880997536, -4.55706599338291, -3.5814225774035497, -4.577972678202224, -4.001728638829222, -3.270401473211861, -3.264703452097223, -3.9718368746319084, -3.132988733879366, -4.361423592613295, -2.595191993338583, -4.516519898988561, -4.912100247864183], [-3.9699474093476774, -4.189042372135887, -4.058989243887689, -4.102030995146257, -4.171520021443684, -4.318254103615893, -4.424253392850214, -4.758810531535327, -4.504029895277935, -4.414714369803455, -3.0879080140350674, -5.469976217597951, -3.790334046490602, -5.366435538657111, -3.235086137375257, -5.727805326900051, -5.408282648592611, -4.971959552124542, -5.070590155566169, -5.076706382583605, -6.181472536826093, -5.921961341341008, -4.9999725883522155, -5.5070174892783, -1.5568628976878147, -5.5070174892783, -6.110013572843948, -4.0678979887767985, -5.089052218405905, -5.200643283814367, -3.6661684098537215, -4.557328477002297, -5.479108701161223, -5.159821289294111, -5.479108701161223, -5.616579691789827, -3.7152580200502454, -4.23556238777078, -4.212031890360586, -3.866007780968386, -4.0678979887767985, -4.4633463186414915, -4.142178063159669, -4.318254103615893, -4.276053749125516, -4.772293881872614, -4.332726136224427, -4.11133338780857, -3.9089648990693253, -4.309670359924501, -3.9127889955077286, -4.338574106106851, -4.528549512452254, -4.149433234040841, -3.5292744136497762, -3.6009751629760713, -4.350373653038005, -4.828122331425556, -4.047963773875981, -4.665125047458004, -4.673288358097166, -4.633122316371831], [-4.806719860900339, -3.6507473420444714, -4.429814962058902, -4.347330952102998, -4.3949170222076255, -4.698366552912107, -4.809448375553543, -4.35423944244681, -4.534137594904308, -4.3611959922401695, -2.871768725634745, -4.598399529122678, -5.564587232141456, -5.519124858064699, -2.4250566837898457, -5.688201188108633, -5.967149580591659, -5.01834957015221, -2.505498094634157, -5.3939617151106924, -6.3813483686685775, -5.360223575478843, -5.821732580733155, -5.742268409378909, -2.5427453130265225, -6.0299504818306895, -5.949907774157153, -6.048642614842842, -6.106911522966818, -6.522426966928483, -1.9368412502165815, -5.384205540165328, -3.2469989725553616, -5.166904264475346, -4.662348253722951, -5.93295821584338, -4.052787789271272, -3.8964417188805776, -4.725629703670472, -3.994277013931218, -3.570836586604418, -4.883136091344702, -4.20182336843176, -4.894970548991704, -4.1342203512338624, -5.159122124033291, -4.442985425248647, -4.740819869164446, -4.583031498894364, -4.589588899440523, -4.4058064220068935, -4.87144005158151, -4.8541485544714496, -4.806719860900339, -4.241282205172307, -4.937530163410501, -4.382360803432213, -5.0116381355642226, -4.368201274828579, -4.782492565565015, -4.544565218066568, -4.4563316672015425]],
    probHashThreshold: 0.015,
    probHashChars: {'1': 1, '0': 0, '3': 3, '2': 2, '5': 5, '4': 4, '7': 7, '6': 6, '9': 9, '8': 8, 'A': 36, 'C': 38, 'B': 37, 'E': 40, 'D': 39, 'G': 42, 'F': 41, 'I': 44, 'H': 43, 'K': 46, 'J': 45, 'M': 48, 'L': 47, 'O': 50, 'N': 49, 'Q': 52, 'P': 51, 'S': 54, 'R': 53, 'U': 56, 'T': 55, 'W': 58, 'V': 57, 'Y': 60, 'X': 59, 'Z': 61, 'a': 10, 'c': 12, 'b': 11, 'e': 14, 'd': 13, 'g': 16, 'f': 15, 'i': 18, 'h': 17, 'k': 20, 'j': 19, 'm': 22, 'l': 21, 'o': 24, 'n': 23, 'q': 26, 'p': 25, 's': 28, 'r': 27, 'u': 30, 't': 29, 'w': 32, 'v': 31, 'y': 34, 'x': 33, 'z': 35},
    userTransitions: {
        search: {}
    },

    patternsLoader: new ContentExtractionPatternsLoader((patternsConfig, ruleset) => {
      CliqzHumanWeb.contentExtractor.updatePatterns(patternsConfig, ruleset);
    }),

    ts : "",
    mRefresh : {},
    can_url_match :{},
    ismRefresh : false,
    userTransitionsSearchSession: 5*60,
    key: ["source","protocol","authority","userInfo","user","password","host","port","relative","path","directory","file","query","anchor"],
    q:   {
        name:   "queryKey",
        parser: /(?:^|&)([^&=]*)=?([^&]*)/g
    },
    parser: {
        strict: /^(?:([^:\/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?))?((((?:[^?#\/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
        loose:  /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/
    },
    activeUsage : null,
    activeUsageThreshold : 2,
    pageStatusCountSent: null,
    actionStats: null,
    actionStatsLastSent: null,
    bloomFilter: null,
    bf:null,
    strictQueries:[],
    oc: null,
    SAFE_QUORUM_ENDPOINT: config.settings.ENDPOINT_SAFE_QUORUM_ENDPOINT,
    SAFE_QUORUM_PROVIDER: config.settings.ENDPOINT_SAFE_QUORUM_PROVIDER,
    safebrowsingEndpoint: new SafebrowsingEndpoint(),
    quorumBloomFilters: {},
    safeQuorumProvider: null,
    getAllOpenPages: getAllOpenPages,
    prefs: {
        humanWebOptOut: false,
        config_ts: null,
        config_location: null,
    },
    _md5: function(str) {
        return md5(str);
    },

    // There should be no reason for these URLs to show up, but if they do
    // we should never send them to the backend. Especially, "moz-extension"
    // is problematic, as it includes an id that is unique per user and
    // can be used to link messages.
    urlLeaksExtensionId(url) {
      return url.startsWith('moz-extension://') || url.startsWith('chrome-extension://');
    },
    maskURL(url) {
        if (CliqzHumanWeb.urlLeaksExtensionId(url)) {
          _log('Dropping URL with extension id:', url);
          return '';
        }
        var url_parts = null;
        var masked_url = null;
        url_parts = parseURL(url);

        // TO BE FIXED
        if (!url_parts) return '';

        // Fix
        if (CliqzHumanWeb.dropLongURL(url)) {
           //Explicit check for google search url.
            if(url_parts['hostname'].indexOf('google') > 0){
                if(url_parts['query_string']){
                    var query_stringsParts = parseQueryString(url_parts['query_string']);
                    if(query_stringsParts['url']){
                        masked_url = query_stringsParts['url'][0];
                        masked_url = CliqzHumanWeb.maskURL(decodeURIComponent(''+masked_url));
                        return masked_url;
                    }
                }
            }
            masked_url = url_parts.protocol + "://"  + url_parts.hostname + "/ (PROTECTED)" ;
            return masked_url;
        }
        return url;
    },
    maskURLStrict(url) {
        if (CliqzHumanWeb.urlLeaksExtensionId(url)) {
          _log('Dropping URL with extension id:', url);
          return '';
        }
        var url_parts = null;
        var masked_url = null;
        // Fix
        // url_parts = CliqzHumanWeb.parseUri(url);
        url_parts = parseURL(url);

        // TO BE FIXED
        if (!url_parts) return '';

        masked_url = url_parts.protocol + "://"  + url_parts.hostname + "/ (PROTECTED)" ;
        return masked_url;
    },
    isShortenerURL: function(url) {
        try {
            var url_parts = parseURL(url);
            if (!url_parts) return true;

            if ((url_parts.hostname.length < 8) && (url_parts.path.length > 4)) {
                var v = url_parts.path.split('/');
                for(var i=0;i<v.length;i++) if (CliqzHumanWeb.isHash(v[i])) return true;
            }
            return false;
        } catch(ee) {
            return true;
        }
    },
    getTime() {
      const now = new Date();
      const utcHours = now.getUTCHours();
      const ts = prefs.get('config_ts', null);
      if (!ts) {
        const date = now.getDate();
        const month = now.getMonth();
        const d = (date < 10 ? "0" : "") + date;
        const m = (month < 9 ? "0" : "") + (month + 1);
        const h = (utcHours < 10 ? "0" : "") + utcHours;
        const y = now.getFullYear();
        return y + "" + m + "" + d + "" + h;
      } else {
        const h = (utcHours < 10 ? "0" : "") + utcHours;
        return ts + "" + h;
      }
    },
    isSuspiciousDomainName: function(hostname) {
        /*
         We need to identify is a hostname looks suspicious.
        */

        let splitDomain = hostname.split('.');

        if (splitDomain.length > 6) {
            return true;
        }

        if (CliqzHumanWeb.checkForLongNumber(hostname, 5)) {
            return true;
        }

        let splitHyphen = hostname.split('-');

        if (splitHyphen.length > 4) {
            return true;
        }

        return false;
    },
    isSuspiciousURL: function(aURI) {
        try {
            var url_parts = {};
            url_parts = parseURL(aURI);

            if (!url_parts) return true;

            _log(JSON.stringify(url_parts));
            if (aURI.indexOf('about:') == 0) return true;

            if (isIpAddress(url_parts.hostname)) {
                return true;
            }

            if (url_parts.username!="" || url_parts.password!="" ) {
                return true;
            }

            if (url_parts.port != "" & (url_parts.port!="80" && url_parts.port!="443")) {
                return true;
            }

            if ( url_parts.protocol != "http"  & url_parts.protocol!="https"  & url_parts.protocol != "") {
                return true;
            }

            var pos_hash_char = aURI.indexOf('#');

            if (pos_hash_char > -1) {
                if (!CliqzHumanWeb.contentExtractor.isSearchEngineUrl(aURI) && aURI.length - pos_hash_char >= 10) {
                    _log("Dropped because of # in url: " + decodeURIComponent(aURI));
                    return true;
                }
            }

            if ( url_parts.hostname.indexOf('localhost') > -1){
                return true;
            }

            if (CliqzHumanWeb.isSuspiciousDomainName(url_parts.hostname)) {
                return true;
            }

            _log("Sanitize: URL is ok: " + aURI);

            return false;

        } catch(ee) {
            // if there were any exception, we return true for safety
            _log("Exception in isSuspiciousURL: " + ee);
            return true;
        }

    },
    calculateStrictness: function(url, page_doc) {
        var strict_value = true;
        if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
            // there is canonical,
            var can_url_parts = parseURL(page_doc['x']['canonical_url']);
            var url_parts = parseURL(url);

            if (!url_parts || !can_url_parts) return true;

            if (url_parts.hostname!=null && url_parts.hostname!='' && url_parts.hostname==can_url_parts.hostname) {
                // both canonical and url have a hostname and is the same,

                if ((page_doc['x']['canonical_url'] != url)  && (page_doc['x']['canonical_url'].length < url.length)) {
                    // the page has a canonical of same domain, which usually is a sign that is public,
                    // and the canonical is not the same url, which comes out of automatic generation
                    // of canonicals
                    strict_value = false;
                }
            }
        }
        return strict_value;
    },
    dropLongURL: function(url, options) {
        try {
            if (options==null) options = {'strict': false};

            if (CliqzHumanWeb.checkForEmail(url)) return true;

            var url_parts = parseURL(url);
            if (!url_parts) return true;

            if (options.strict == true) {
                if (url_parts.query_string && url_parts.query_string.length > CliqzHumanWeb.qs_len*0.75) return true;
                // check the number of parameters too,

                if (url_parts.query_string) {
                    var v = url_parts.query_string.split(/[&;]/);
                    if (v.length > 1) {
                        // that means that there is a least one &; hence 2 params
                        return true;
                    }

                    for (var i=0; i<v.length; i++) {
                        if (v[i].length>3 && CliqzHumanWeb.isHash(v[i])) return true;
                    }

                    if (CliqzHumanWeb.checkForLongNumber(url_parts.query_string, 8)!=null) return true;
                }

                if (CliqzHumanWeb.checkForLongNumber(url_parts.path, 8)!=null) return true;

            }
            else {
                if (url_parts.query_string && url_parts.query_string.length > CliqzHumanWeb.qs_len) return true;

                if (url_parts.query_string) {

                    var v = url_parts.query_string.split(/[&;]/);
                    if (v.length > 4) {
                        // that means that there is a least one &; hence 5 params
                        return true;
                    }
                    if (CliqzHumanWeb.checkForLongNumber(url_parts.query_string, 12)!=null) return true;
                }

                if (CliqzHumanWeb.checkForLongNumber(url_parts.path, 12)!=null) return true;
            }

            var vpath = url_parts.path.split(/[\/\._ \-:\+;]/);
            for (var i=0; i<vpath.length; i++) {
                if (vpath[i].length > CliqzHumanWeb.rel_part_len) {
                    return true;
                }

                if (options.strict == true) {
                    // if strict, check the no token in path looks like a hash
                    if (vpath[i].length>5 && CliqzHumanWeb.isHash(vpath[i])) return true;
                }
                else {
                    if (vpath[i].length>12 && CliqzHumanWeb.isHash(vpath[i])) return true;
                }

            }

            var vpath = url_parts.path.split('/');
            for (var i=0; i<vpath.length; i++) {
                var cstr = vpath[i].replace(/[^A-Za-z0-9]/g,'');
                var mult = 1.0;
                if (options.strict == true) mult = 0.5;
                if (cstr.length > CliqzHumanWeb.rel_segment_len * mult) {
                    if (CliqzHumanWeb.isHash(cstr)) return true;
                }
            }

            var v = [/\/admin([\/\?#=]|$)/i, /\/wp-admin([\/\?#=]|$)/i,
                     /\/edit([\/\?#=]|$)/i, /[&\?#\/]share([\/\?#=]|$)/i,
                     /[&\?#\/;]sharing([\/\?#=]|$)/i, /[&\?#\/;]logout([\/\?#=]|$)/i,
                     /WebLogic/i, /[&\?#\/;]token([\/\?#=_;]|$)/i,
                     /[&\?#\/;]trk([\/\?#=_]|$)/i, /[&\?#\/=;](http|https)(:\/|\%3A\%2F)/];

            // url_rel contains path and query_string
            //
            var url_rel = (url_parts.path || '/') + '?' + (url_parts.query_string || '');
            for (var i=0;i<v.length;i++) if (v[i].test(url_rel)) return true;

            // checks specific to the query string
            //
            // real query string (?) or a 'fake' one with a sharp on the path, they should be treated the same way
            //
            var path_query_string = null;
            var ind_pos = url_parts.path.indexOf('#');
            if (ind_pos!=-1) path_query_string = url_parts.path.slice(ind_pos, url_parts.path.length);

            if ((url_parts.query_string && url_parts.query_string.length > 0) || (path_query_string && path_query_string.length > 0)) {

                var v = [/[&\?#_\-;]user/i, /[&\?#_\-;]token/i, /[&\?#_\-;]auth/i, /[&\?#_\-;]uid/i, /[&\?#_\-;]email/i,
                         /[&\?#_\-;]usr/i, /[&\?#_\-;]pin/i, /[&\?#_\-;]pwd/i, /[&\?#_\-;]password/i, /[&\?#;]u[=#]/i, /[&\?#;]url[=#]/i,
                         /[&\?#_\-;]http/i, /[&\?#_\-;]ref[=#]/i, /[&\?#_\-;]red[=#]/i, /[&\?#_\-;]trk/i, /[&\?#_\-;]track/i,
                         /[&\?#_\-;]shar/i, /[&\?#_\-;]login/i, /[&\?#_\-;]logout/i, /[&\?#_\-;]session/i,
                         ];

                if (url_parts.query_string && url_parts.query_string.length > 0) {
                    for (var i=0; i<v.length; i++) if (v[i].test('?' + url_parts.query_string)) return true;
                }

                if (path_query_string && path_query_string.length > 0) {
                    for (var i=0; i<v.length; i++) if (v[i].test(path_query_string)) return true;
                }
            }

            return false;

        } catch(ee) {
            // if there were any exception, we return true for safety
            _log("Exception in dropLongURL: " + ee);
            return true;
        }

    },
    cleanDocCache: function() {
      for(var key in CliqzHumanWeb.docCache) {
        if ((CliqzHumanWeb.counter - CliqzHumanWeb.docCache[key]['time']) > 3600*CliqzHumanWeb.tmult) {
          delete CliqzHumanWeb.docCache[key];
        }
      }
    },
    cleanHttpCache: function() {
      for(var key in CliqzHumanWeb.httpCache) {
        if ((CliqzHumanWeb.counter - CliqzHumanWeb.httpCache[key]['time']) > 60*CliqzHumanWeb.tmult) {
         delete CliqzHumanWeb.httpCache[key];
        }
      }
      for(var key in CliqzHumanWeb.httpCache401) {
        if ((CliqzHumanWeb.counter - CliqzHumanWeb.httpCache401[key]['time']) > 60*CliqzHumanWeb.tmult) {
          delete CliqzHumanWeb.httpCache401[key];
        }
      }
    },
    getHeaders: function(strData) {
      var o = {};
      let _status = strData.split(" ")[1];

      if(parseInt(_status)) {
        _status = parseInt(_status);
      } else {
        _status = null;
      }

      o['status'] = _status;

      var l = strData.split("\n");
      for(var i=0;i<l.length;i++) {
        if (l[i].indexOf('Location: ') == 0) {
          o['loc'] = decodeURIComponent(l[i].split(" ")[1].trim());
        }
        if (l[i].indexOf('WWW-Authenticate: ') == 0) {
          var tmp = l[i].split(" ");
          var tmp = tmp.slice(1, tmp.length).join(" ");
          o['auth'] = tmp;
        }
      }

      return o;
    },
    httpObserver: {
        // check the non 2xx page and report if this is one of the cliqz result
        observeActivity: function({ url: aUrl, type, responseStatus, statusCode, responseHeaders, isPrivate, tabId }) {
          // If isPrivate true, then drop it.
          if (isPrivate) {
            return;
          }

          // But we are only concerned, with main_frame tabs,
          // hence we can return all the others, from this point.

          if (type !== 6 && type !== 'main_frame') {
            return;
          }

          // Detect ad click
          try {
            CliqzHumanWeb.detectAdClick(aUrl);

            const url = decodeURIComponent(aUrl);
            const status = responseStatus || statusCode;
            const headers = responseHeaders;

            if (status === 301 || status === 302) {
              let location;
              headers.forEach( eachHeader => {
                if (eachHeader.name && eachHeader.name.toLowerCase() === 'location') {
                  location = eachHeader.value;
                }
              });
              CliqzHumanWeb.httpCache[url] = {
                status: status,
                time: CliqzHumanWeb.counter,
                location,
              };
            } else if (status === 401) {
              CliqzHumanWeb.httpCache401[url] = {
                time: CliqzHumanWeb.counter
              };
            } else if(status) {
              CliqzHumanWeb.httpCache[url] = {
                status: status,
                time: CliqzHumanWeb.counter
              };
            }
          } catch (e) {
            _log(e);
          }
        }
    },
    onVisitRemoved({ urls, allHistory }) {
      if (allHistory) {
        CliqzHumanWeb.db.clearHistory();
      } else {
        for (const url of urls) {
          CliqzHumanWeb.db.deleteVisit(url);
        }
      }
    },
    linkCache: {},
    cleanLinkCache: function() {
      for(var key in CliqzHumanWeb.linkCache) {
        if ((CliqzHumanWeb.counter - CliqzHumanWeb.linkCache[key]['time']) > 60 * CliqzHumanWeb.tmult) {
          delete CliqzHumanWeb.linkCache[key];
        }
      }
    },
    // maxLength is the maximum length of the redirect chain
    // (once a cycle is found, it will always stop, independent of maxLength)
    getRedirects(url, _res, maxLength = 6) {
      const res = _res || [];
      if (maxLength <= 0) {
        return res;
      }

      const isRedirectToUrl = (status, location) => {
        if (status !== 301 && status !== 302) {
          return false;
        }

        return location === url || decodeURIComponent(location) === url || location + '/' === url;
      };

      // Note: O(n*c) complexity, where n is the size of CliqzHumanWeb.httpCache
      // and c is the redirect chain length. In practice, it should not be an
      // issue for two reasons:
      // 1) "c" is limited by maxLength, which typically is a small constant
      // 2) CliqzHumanWeb.httpCache is reguarly cleaned up, thus it should stay quite small
      //
      // (TODO: verify that the cleanup of CliqzHumanWeb.httpCache works)
      //
      try {
        for (const [key, { status, location }] of Object.entries(CliqzHumanWeb.httpCache)) {
          if (key !== url && isRedirectToUrl(status, location) && res.indexOf(key) < 0) {
            res.unshift(key);
            return CliqzHumanWeb.getRedirects(key, res, maxLength - 1);
          }
        }
      } catch (ee) {
        _log(">>>> REDIRECT ERROR >>> " + ee);
      };
      return res;
    },
    userSearchTransition: function(rq){
        // now let's manage the userTransitions.search
        if (rq!=null) {
            var source = rq['qurl'];
            var query = rq['q'].replace(/\s+/g, " ").trim().toLowerCase();

            if (source && query && query.length>0) {
                // we have both the source and the query,
                // let's see if we have done the query

                if (CliqzHumanWeb.userTransitions['search'][query] == null) {
                    CliqzHumanWeb.userTransitions['search'][query] = {'time': CliqzHumanWeb.counter, 'data': []}
                }
                CliqzHumanWeb.userTransitions['search'][query]['data'].push([source, CliqzHumanWeb.counter - CliqzHumanWeb.userTransitions['search'][query]['time']]);
                }
            }

    },
    getParametersQS: function(url) {
        var res = {};
        var KeysValues = url.split(/[\?&]+/);
        for (var i = 0; i < KeysValues.length; i++) {
            var kv = KeysValues[i].split("=");
            if (kv.length==2) res[kv[0]] = kv[1];
        }
        return res;
    },
    getEmbeddedURL: function(targetURL) {
        var ihttps = targetURL.lastIndexOf('https://')
        var ihttp = targetURL.lastIndexOf('http://')
        if (ihttps>0 || ihttp>0) {
            // contains either http or https not ont he query string, very suspicious
            try {
                const parqs = new URL(targetURL);
                const urlParam = parqs.searchParams.params.find(kv => kv[0] === 'url');
                if (urlParam) {
                    return decodeURIComponent(urlParam[1]);
                }
            } catch (e) {}
        }
        else return null;
    },
    auxIsAlive: function() {
        return true;
    },

    auxGetPageData: function(url, page_data, original_url, onsuccess, onerror) {
      CliqzHumanWeb.doublefetchHandler.anonymousHttpGet(url)
        .then(parseHtml)
        .then((doc) => {
          const x = CliqzHumanWeb.getPageData(url, doc);

          CliqzHumanWeb.docCache[url] = {
            time: CliqzHumanWeb.counter,
            doc,
          };

          onsuccess(url, page_data, original_url, x);
        })
        .catch((error_message) => {
          _log(`Error on doublefetch: ${error_message}`);
          onerror(url, page_data, original_url, error_message);
        });
    },
    auxIntersection: function(a, b) {
        var ai=0, bi=0;
        var result = new Array();
        while( ai < a.length && bi < b.length ) {
            if      (a[ai] < b[bi] ){ ai++; }
            else if (a[ai] > b[bi] ){ bi++; }
            else {
                result.push(a[ai]);
                ai++;
                bi++;
            }
        }
        return result;
    },
    auxUnion: function(a, b) {
        var h = {};
        for (var i = a.length-1; i >= 0; -- i) h[a[i]] = a[i];
        for (var i = b.length-1; i >= 0; -- i) h[b[i]] = b[i];
        var res = [];
        for (var k in h) {
            if (h.hasOwnProperty(k)) res.push(h[k]);
        }
        return res;
    },
    validDoubleFetch: function(struct_bef, struct_aft, options) {
        // compares the structure of the page when rendered in Firefox with the structure of
        // the page after.

        _log("xbef: " + JSON.stringify(struct_bef));
        _log("xaft: " + JSON.stringify(struct_aft));

        // Check if struct_bef or struct_aft is not null, in case anyone is then we mark it as private.

        // if any of the titles is null (false), then decline (discard)
        if (!(struct_bef && struct_aft)) {
            _log("fovalidDoubleFetch: found an empty structure");
            return false;
        }

        if (!(struct_bef['t'] && struct_aft['t'])) {
            _log("fovalidDoubleFetch: found an empty title");
            return false;
        }

        // if any of the two struct has a iall to false decline
        if (!(struct_bef['iall'] && struct_aft['iall'])) {
            _log("fovalidDoubleFetch: found a noindex");
            return false;
        }

        // check that there are not different number of frames (or iframes) with an internal
        // link on the two different loads (with and with session), if so, the frame (iframe)
        // might contain a password field. Cannot afford to fetch all iframes on page. Only
        // internal are considered, externals are likely to vary a lot due to advertisement.
        //

        /*
        Adding key to check how many pages will we loose if frame check is turned on
        if (struct_bef['nfsh']==null || struct_aft['nfsh']==null || struct_bef['nfsh']!=struct_aft['nfsh']) {
            _log("fovalidDoubleFetch: number of internal frames does not match");
            return false;
        }

        if (struct_bef['nifsh']==null || struct_aft['nifsh']==null || struct_bef['nifsh']!=struct_aft['nifsh']) {
            _log("fovalidDoubleFetch: number of internal iframes does not match");
            return false;
        }
        */
        if (struct_bef['canonical_url'] != struct_aft['canonical_url']) {
            // if canonicals are different, in principle are different pages,

            if (struct_aft['canonical_url']!=null && struct_aft['canonical_url'].length>0 && struct_bef['canonical_url']==null) {
                // unless in the case that struct_aft (after) has a canonical different than null
                // and struct_bef has no canonical,
            }
            else {
                return false;
            }
        }

        if (options.structure_strict==true) {

            // if there is enough html length, do the ratio, if below or above 10% then very imbalance, discard

            var length_html_ok = true;
            var length_text_ok = true;

            var ratio_lh = (struct_bef['lh'] || 0) / ((struct_bef['lh'] || 0) + (struct_aft['lh'] || 0));
            if ((struct_bef['lh'] || 0) > 10*1024) {
                var ratio_lh = (struct_bef['lh'] || 0) / ((struct_bef['lh'] || 0) + (struct_aft['lh'] || 0));
                if (ratio_lh < 0.10 || ratio_lh > 0.90) {
                    _log("fovalidDoubleFetch: lh is not balanced");
                    length_html_ok = false;
                }
            }

            // if there is enough html length, do the ratio, if below or above 10% then very imbalance, discard
            var ratio_nl = (struct_bef['nl'] || 0) / ((struct_bef['nl'] || 0) + (struct_aft['nl'] || 0));
            if ((struct_bef['lh'] || 0) > 30) {
                var ratio_nl = (struct_bef['nl'] || 0) / ((struct_bef['nl'] || 0) + (struct_aft['nl'] || 0));
                if (ratio_nl < 0.10 || ratio_nl > 0.90) {
                    _log("fovalidDoubleFetch: nl is not balanced");
                    length_text_ok = false;
                }
            }

            if (!length_text_ok && !length_html_ok) return false;

        }

        // check for passwords and forms if there is no canonical on the after (double fetched with no session)
        if (struct_aft['canonical_url']==null || struct_aft['canonical_url']=='') {
            // if had no password inputs before and it has after, decline
            if ((struct_bef['nip'] == null || struct_aft['nip'] == null) || (struct_bef['nip'] == 0 && struct_aft['nip'] > 0)) {
                _log("validDoubleFetch: fail nip");
                return false;
            }

            // if had no forms before and it has after, decline
            if ((struct_bef['nf'] == null || struct_aft['nf'] == null) || (struct_bef['nf'] == 0 && struct_aft['nf'] > 0)) {
                _log("validDoubleFetch: fail text nf");
                return false;
            }
        }

        // compare that titles are equal, if not equal, use the jaccard coefficient, decline if <=0.5
        var t1 = struct_bef['t'] || '';
        var t2 = struct_aft['t'] || '';
        var jc = 1.0;

        if (t1!=t2) {

            var vt1 = t1.split(' ').filter(function(el) {return el.length>1});
            var vt2 = t2.split(' ').filter(function(el) {return el.length>1});

            jc = CliqzHumanWeb.auxIntersection(vt1,vt2).length / CliqzHumanWeb.auxUnion(vt1,vt2).length;

            if (Math.max(vt1.length, vt2.length) <= 4) {
                // the longest titles is 4 tokens long, the, we are a bit flexible on title differences
                if (jc >= 0.5) return true;
                else {
                    _log("short title fail title overlap");
                    return false;
                }
            }
            else {
                // the longest titles has 4 or more tokens, be more restrictive
                if (jc <= 0.5) {
                    // one last check, perhaps it's an encoding issue

                    var tt1 = t1.replace(/[^A-Za-z 0-9 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]*/g, '');
                    var tt2 = t2.replace(/[^A-Za-z 0-9 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]*/g, '');

                    if ((tt1.length > t1.length*0.5) && ((tt2.length > t2.length*0.5))) {
                        // if we have not decreased the titles by more than 50%
                        var vtt1 = tt1.split(' ').filter(function(el) {return el.length>1});
                        var vtt2 = tt2.split(' ').filter(function(el) {return el.length>1});
                        jc = CliqzHumanWeb.auxIntersection(vtt1,vtt2).length / CliqzHumanWeb.auxUnion(vtt1,vtt2).length;
                        // we are more demanding on the title overlap now
                        if (jc <= 0.80) {
                            _log("validDoubleFetch: fail title overlap after ascii");
                            return false;
                        }
                    }
                    else {
                      _log("validDoubleFetch: fail title overlap");
                      return false;
                    }
                }

                // if the titles are not a perfect match then check for more structural things like number of inputs
                // that are type password and number of forms. This is prone to false positives because when not logged in
                // legitimate sites something prompt you to register

                // if had no password inputs before and it has after, decline
                if ((struct_bef['nip'] == null || struct_aft['nip'] == null) || (struct_bef['nip'] == 0 && struct_aft['nip'] > 0)) {
                    _log("validDoubleFetch: fail nip");
                    return false;
                }

                // if had no forms before and it has after, decline
                if ((struct_bef['nf'] == null || struct_aft['nf'] == null) || (struct_bef['nf'] == 0 && struct_aft['nf'] > 0)) {
                    _log("validDoubleFetch: fail text nf");
                    return false;
                }

                return true;
            }
        }
        else {
            // exactly same title
            return true;
        }

        _log("validDoubleFetch: default option");

        return false;

    },
    getCleanerURL: function(url) {

        var clean_url = url;
        // check first if there is a query string,

        var url_parts = parseURL(url);
        if (!url_parts) return null;

        if (url_parts && url_parts.query_string && url_parts.query_string != '') {
            // it has a query string, either by ? # or ;
            clean_url = url_parts.protocol + '://' + url_parts.hostname + url_parts.path;
        }
        else {
            // it has neither query_string or hash or semicolon, so let's try to remove the last segment

            if (url_parts && url_parts.path && url_parts.path != '') {
                var qs = url_parts.path.split('/')
                var cqs = [];
                for (let i=0;i<qs.length;i++) if (qs[i]!='') cqs.push(qs[i]);

                if (cqs.length >= 3) {
                    // more than 3, /a/b/c, let's remove the last one

                    var new_path =  cqs.slice(0, cqs.length-1).join('/');
                    clean_url = url_parts.protocol + '://' + url_parts.hostname + '/' + new_path + '/';
                }
            }

        }

        if (clean_url != url) {
            // they are different, sanity checks
            if (CliqzHumanWeb.isSuspiciousURL(clean_url) || CliqzHumanWeb.dropLongURL(clean_url)) return url;
            else return clean_url;

        }
        else return url;


    },
    fetchReferral: function(referral_url, callback) {

        _log("PPP in fetchReferral: " + referral_url);

        if (referral_url && referral_url!='') {
            if (CliqzHumanWeb.docCache[referral_url]==null) {
                CliqzHumanWeb.auxGetPageData(referral_url, null, null, function(referral_url) {
                    _log("PPP in fetchReferral success auxGetPageData: " + referral_url);
                    callback();
                },
                function(referral_url) {
                    _log("PPP in fetchReferral failure auxGetPageData: " + referral_url);
                    callback();
                });
            }
            else {
                _log("PPP in fetchReferral already in docCache: " + referral_url);
                callback();
            }
        }
        else callback();
    },
    doubleFetch: function(url, page_doc) {
        // Need to add check for page MU.
        // one last validation whether should be fetchable or not. If we cannot send that URL because it's
        // private/suspicious/search_result page/etc. we can mark it as private directly

        var isok = true;

        if (page_doc ==null || page_doc['x'] == null) {
            // this should not happen, but it does. Need to debug why the 'x' field gets lost
            // right now, let's set is a private to avoid any risk
            //
            isok = false
        }

        if (page_doc && page_doc['x'] && page_doc['x']['iall'] == false) {
                // the url is marked as noindex
                isok = false;
        }

        // the url is suspicious, this should never be the case here but better safe
        //
        if (CliqzHumanWeb.isSuspiciousURL(url) == true) isok = false;

        if (CliqzHumanWeb.dropLongURL(url)) {
            _log("The url: " + url + " is long");

            if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
                // the url is to be drop, but it has a canonical URL so it should be public
                if (CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'])) {
                    // wops, the canonical is also bad, therefore mark as private
                    _log("The canonical url: " + page_doc['x']['canonical_url'] + " is also long");
                    isok = false;
                }
                else {
                    // there we are in the good scenario in which canonical looks ok although
                    // url did not
                    isok = true;
                }
            }
            else {
                if(page_doc['isMU']){
                    isok = true;
                }
                else{
                    isok = false;
                }
            }
        }

        if (isok) {
            _log("going to double-fetch: " + url);

            CliqzHumanWeb.auxGetPageData(url, page_doc, url, function(url, page_doc, original_url, data) {

                // data contains the public data of the url double-fetch,

                _log("success on doubleFetch, need further validation" + url);

                if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': false})) {


                    //
                    // If the double fetch is validated, we will now check for the iFrame and frameSets
                    // Since we only need the telemetry for now, this seems to be the right place
                    // we need to inject the page structure. The final event will have an extra key
                    // nifshmatch : true / false , nfshmatch: true / false.,
                    // nifshbf : Iframe count in struct_bef, nifshbf : framsetcount in before.
                    // This key is added to both page_doc and data.
                    //

                    let nifshmatch = CliqzHumanWeb.validFrameCount(page_doc['x'], data);
                    let nfshmatch = CliqzHumanWeb.validFrameSetCount(page_doc['x'], data);

                    data.nifshmatch = nifshmatch;
                    data.nfshmatch = nfshmatch;
                    data.nifshbf = page_doc.x.nifsh;
                    data.nfshbf = page_doc.x.nifsh;

                    //
                    // url, we should have the data of the double for the referral in CliqzHumanWeb.docCache
                    //
                    CliqzHumanWeb.fetchReferral(page_doc['ref'], function() {

                        var strict_value = CliqzHumanWeb.calculateStrictness(url, page_doc);

                        if (page_doc['ref'] && page_doc['ref']!='') {
                            // the page has a referral
                            _log("PPP: page has a referral, " + url + " < " + page_doc['ref']);
                            var hasurl = CliqzHumanWeb.hasURL(page_doc['ref'], url);
                            _log("PPP: page has a referral, " + url + " < " + page_doc['ref'] + ">>>> " + hasurl);

                            // overwrite strict value because the link exists on a public fetchable page
                            if (hasurl) strict_value = false;
                        }
                        else {
                            // page has no referral
                            _log("PPP: page has NO referral, " + url);

                            // we do not know the origin of the page, run the dropLongURL strict version, if
                            // there is no canonical or if there is canonical and is the same as the url,
                        }

                        _log("strict URL:" + url + " > " + strict_value);

                        if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': strict_value})) {

                            // we do not know the origin of the page, run the dropLongURL strict version

                            if (CliqzHumanWeb.dropLongURL(url, {'strict': strict_value})) {
                                if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
                                    if (CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'], {'strict': strict_value})) {
                                        _log("doubleFetch failed on dropLongURL strict=true:" + url);
                                        CliqzHumanWeb.setAsPrivate(url);
                                        return;
                                    }
                                }
                                else {
                                    _log("doubleFetch failed on dropLongURL strict=true:" + url);
                                    CliqzHumanWeb.setAsPrivate(url);
                                    return;
                                }
                            }
                        }
                        else {
                            // the strict version of validDoubleFetch fails for a page with no referral,
                            // since we do not know the origin mark as private
                            _log("doubleFetch failed on structure_strict=true: " + url);
                            CliqzHumanWeb.setAsPrivate(url);
                            return;
                        }



                        _log("success on doubleFetch, need further validation");

                        //
                        // we need to modify the 'x' field of page_doc to substitute any structural information about
                        // the page content by the data coming from the doubleFetch (no session)
                        //

                        var first_url_double_fetched = url;

                        //
                        // we need to modify the 'x' field of page_doc to substitute any structural information about
                        // the page content by the data coming from the doubleFetch (no session)
                        //

                        // at this point we might have 3 different urls: url, page_doc['x']['canonical_url'] (either
                        // one of them has to have passed the dropLongURL test), and finally we also have data['canonical_url']
                        // (which has not passed the dropLongURL test). In principle, data['canonical_url']==page_doc['x']['canonical_url']
                        // but is not always the case, different calls can give different urls

                        if (data['canonical_url']!=null && data['canonical_url'] != '' && CliqzHumanWeb.dropLongURL(data['canonical_url'])==false) {
                            page_doc['url'] = data['canonical_url'];
                            page_doc['x'] = data;
                        }
                        else {
                            if (page_doc['x']['canonical_url']!=null && page_doc['x']['canonical_url'] != '' && CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'])==false) {
                                page_doc['url'] = page_doc['x']['canonical_url'];
                                page_doc['x'] = data;
                                page_doc['x']['canonical_url'] = page_doc['url'];
                            }
                            else {
                                // there was no canonical either on page_doc['x'] or in data or it was droppable

                                if (CliqzHumanWeb.dropLongURL(url)==false) {
                                    page_doc['url'] = url;
                                    page_doc['x'] = data;

                                    if (page_doc['x']['canonical_url']!=null && page_doc['x']['canonical_url'] != '') {
                                        page_doc['x']['canonical_url'] = url;
                                    }
                                }
                                else {
                                    // this should not happen since it would be covered by the isok checks, but better safe,
                                    CliqzHumanWeb.setAsPrivate(url);
                                    return;
                                }
                            }
                        }

                        var clean_url = CliqzHumanWeb.getCleanerURL(page_doc['url']);

                        if (clean_url != page_doc['url']) {
                            // we have a candidate for a cleaner url (without query_string) or without the last segment
                            // of the path, we want to double fetch it and if successful, then, we could replace the url
                            // because it would be cleaner hence safer
                            //

                            _log("going to clean_url double-fetch: " + clean_url);

                            CliqzHumanWeb.auxGetPageData(clean_url, page_doc, first_url_double_fetched, function(url, page_doc, original_url, data) {

                                _log("success on clean_url doubleFetch, need further validation");

                                if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': false})) {
                                    // if it the second double fetch is valid, that means that the clean_url is (url parameter) is
                                    // equivalent, so we can replace
                                    page_doc['url'] = url;
                                    page_doc['x'] = data;
                                    CliqzHumanWeb.setAsPublic(original_url);
                                    CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});

                                }
                                else {
                                    // the page with the clean_urls does not return the same, it can be two cases here, one is that
                                    // the content is just totally different, in this case, we should send the page with the unclean_url,
                                    // the other case is that the content is different but now we have passwords where we did not have before,
                                    // in such a case, it's safer to assume that the fragments cleaned were identifiying a user, and the
                                    // website is redirecting to the login page, in such a case, we should not send the page at all, in fact, we
                                    // should mark it as private just to be sure,

                                    if (CliqzHumanWeb.debug) {
                                        _log("checking clean_url, page_doc: " + JSON.stringify(page_doc));
                                        _log("checking clean_url, data: " + JSON.stringify(data));
                                    }


                                    if (page_doc['x']['nip'] < data['nip']) {
                                        // the page with url_clean have more input password fields or more forms, this is dangerous,
                                        _log("failure on checking the password and forms for clean_url: " + original_url + " set as private");
                                        CliqzHumanWeb.setAsPrivate(original_url);
                                    }
                                    else {
                                        // safe, here we will send the url before clean_url
                                        CliqzHumanWeb.setAsPublic(original_url);
                                        CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
                                    }
                                }
                            },
                            function(url, page_doc, original_url, error_message) {
                                _log("failure on clean_url doubleFetch! structure did not match");
                                // there was a failure, the clean_url does not go to the same place, therefore it's better
                                // not to replace

                                CliqzHumanWeb.setAsPublic(original_url);
                                CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});

                            });
                        }
                        else {
                            CliqzHumanWeb.setAsPublic(original_url);
                            CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
                        }
                    });
                }
                else {
                    _log("failure on doubleFetch! " + "structure did not match");
                    CliqzHumanWeb.setAsPrivate(url);
                }
            },
            function(url, page_doc, original_url, error_message) {
                _log("failure on doubleFetch! " + error_message);
                CliqzHumanWeb.setAsPrivate(url);
            });

        }
        else {
            _log("doubleFetch refused to process this url: " + url);
            CliqzHumanWeb.setAsPrivate(url);
        }

    },
    hasURL: function(source_url, target_url) {
        // the target_url is in the source_url

        try {


            var tt = CliqzHumanWeb.docCache[source_url];
            if (tt) {
                var cd = tt['doc'];
                if (!cd) {
                    // fetch the content of the source_url,
                    //
                    _log("hasURL no CD!!! ");
                    return false;
                }
            }
            else return false;

            var target_url_no_protocol = target_url.replace(/^http(s?)\:\/\//, '');
            var target_url_relative = null;
            try {
                var source_hostname = parseURL(source_url).hostname;
                var target_hostname = parseURL(target_url).hostname;
            } catch(ee) {
                return false;
            }

            if (source_hostname == target_hostname) {
                // same domain, path could be relative
                target_url_relative = '/';
                var v = target_url_no_protocol.split('/');
                if (v.length>1) target_url_relative = '/' + v.slice(1, v.length).join('/')
            }

            //var html = cd.documentElement.innerHTML;
            //var ind1 = html.indexOf(target_url_no_protocol);
            var found = false;

            var links = cd.documentElement.getElementsByTagName('a');

            for(var i=0;i<links.length;i++) {
                try {
                    var link = links[i].href;
                    link = link.replace(/^http(s?)\:\/\//, '');

                    if (link == target_url_no_protocol) {
                        found = true;
                        break;
                    }
                    else {
                        if (target_url_relative && link == target_url_relative) {
                            found = true;
                            break;
                        }
                    }
                } catch(ee) {}
            }


            return found;

        } catch(ee) {
            _log("Error on hasURL: " + ee);
            return false;
        }
    },
    getPageData: function(url, cd) {

        var len_html = null;
        var len_text = null;
        var title = null;
        var numlinks = null;
        var inputs = null;
        var inputs_nh = null;
        var inputs_pwd = null;
        var frames_same_host = null;
        var iframes_same_host = null;
        var frames = null;
        var forms = null;
        var pg_l = null;
        var metas = null;
        var tag_html = null;
        var iall = true;
        var all = null;
        var canonical_url = null;

        var url_host = null;
        var frame_host = null;

        let ourl = parseURL(url);

        try {
            url_host = parseURL(url).hostname;
        } catch(ee) {
            url_host = null;
        }

        try { len_html = cd.documentElement.innerHTML.length; } catch(ee) {}
        try { len_text = cd.documentElement.textContent.length; } catch(ee) {}
        try { title = cd.getElementsByTagName('title')[0].textContent; } catch(ee) {}
        //title = unescape(encodeURIComponent(title));

        try { numlinks = cd.getElementsByTagName('a').length; } catch(ee) {}
        try {
            inputs = cd.getElementsByTagName('input') || [];
            inputs_nh = 0;
            inputs_pwd = 0;
            for(var i=0;i<inputs.length;i++) {
                if (inputs[i]['type'] && inputs[i]['type']!='hidden') inputs_nh+=1;
                if (inputs[i]['type'] && inputs[i]['type']=='password') inputs_pwd+=1;
            }
        } catch(ee) {}


        try {
            frames = cd.getElementsByTagName('frame') || [];
            frames_same_host = 0;
            for(var i=0;i<frames.length;i++) {
                if (frames[i]['src']) {
                    var tsrc = frames[i]['src'];
                    if (frames[i]['src'].startsWith('//')) tsrc = 'http:' + frames[i]['src'];
                    try {
                        frame_host = parseURL(tsrc).hostname;
                        if (frame_host===url_host || frame_host=='browser') frames_same_host++;
                    } catch(ee) {
                        frames_same_host++;
                    }
                }
            }
        }
        catch(ee) {}

        try {
            frames = cd.getElementsByTagName('iframe') || [];
            iframes_same_host = 0;
            for(var i=0;i<frames.length;i++) {
                if (frames[i]['src']) {
                    var tsrc = frames[i]['src'];
                    if (frames[i]['src'].startsWith('//')) tsrc = 'http:' + frames[i]['src'];
                    try {
                        frame_host = parseURL(tsrc).hostname;
                        if (frame_host===url_host || frame_host=='browser') iframes_same_host++;
                    } catch(ee) {
                        iframes_same_host++;
                    }
                }
            }
        }
        catch(ee) {}



        try { forms = cd.getElementsByTagName('form'); } catch(ee) {}

        var metas = cd.getElementsByTagName('meta');

        // extract the language of th
        try {
            for (let i=0;i<metas.length;i++) {if (metas[i].getAttribute("http-equiv") == "content-language" || metas[i].getAttribute("name") == "language") {
                pg_l = metas[i].getAttribute("content");
            }};

            if (pg_l == null) {
                tag_html = cd.getElementsByTagName('html');
                pg_l = tag_html[0].getAttribute("lang");
            };

            // Keep a tab on the length of pagel.
            if(pg_l) {
                if (pg_l.length > 10) {
                    pg_l = null;
                }
            }
        }catch(ee){}


        // Check if page is not allowed for indexing, by checking the no-index tag.
        let headTag = '';
        try {
            headTag = cd.querySelector('head');
            if (headTag) {
                let headContent = headTag.innerHTML.toLowerCase();
                if(headContent && headContent.indexOf('noindex') > -1){
                    iall = false;
                }
            }
        } catch (ee) {
            _log("no-index check failed " + ee);
        }

        // extract the canonical url if available
        var link_tag = cd.getElementsByTagName('link');
        for (var j=0;j<link_tag.length;j++) {
            if (link_tag[j].getAttribute("rel") == "canonical") {
                canonical_url = link_tag[j].href;

                // This check is done because of misplaces titles on sites like 500px, youtube etc.
                // Since could not find a proper fix, hence dropping canonical URL looks like a safe idea.

                if(CliqzHumanWeb.can_url_match[canonical_url] && CliqzHumanWeb.can_url_match[canonical_url] != url) canonical_url = null;
            }
        }

        if (canonical_url != null && canonical_url.length > 0) {
            // check that canonical url is not relative.
            // Possible variations
            // ourl:
            // http://www.liceubarcelona.cat/ca/properament_2017/opera
            // curl:
            // ca/properament_2017/opera,
            // /liceubarcelona.cat/ca/properament_2017/opera
            // chrome://www.liceubarcelona.cat/ca/properament_2017/opera
            // //www.ghacks.net/

            if (canonical_url.startsWith('//')) {
                canonical_url = canonical_url.replace('//', '');
            }

            let _urlDetails = getDetailsFromUrl(canonical_url);
            if (_urlDetails.scheme.startsWith('chrome') || _urlDetails.scheme === '' || (_urlDetails.name && !_urlDetails.tld)) {
                canonical_url =  `${ourl.protocol}:\/\/${ourl.hostname}${_urlDetails.path}`;
            }
        }

        // extract the location of the user (country level)
        try {var location = CliqzHumanWeb.getCountryCode();} catch(ee){}

        var x = {'lh': len_html, 'lt': len_text, 't': title, 'nl': numlinks, 'ni': (inputs || []).length, 'ninh': inputs_nh, 'nip': inputs_pwd, 'nf': (forms || []).length, 'pagel' : pg_l , 'ctry' : location, 'iall': iall, 'canonical_url': canonical_url, 'nfsh': frames_same_host, 'nifsh': iframes_same_host};

        return x;
    },
    listener: {
        tmpURL: undefined,

        onLocationChange: function({ isPrivate, isLoadingDocument, url, referrer, frameId }) {

            // New location, means a page loaded on the top window, visible tab
            // Return if it's a private tab.
            if(isPrivate) {
                return;
            }

            if (url === this.tmpURL) {
              return;
            }
            this.tmpURL = url;

            CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
            CliqzHumanWeb.lastActiveAll = CliqzHumanWeb.counter;

            // The "originalURL" is the URL as shown in the address bar of the browser.
            // Internally, human web uses a decoded version. In general, if something is
            // referred to as "url" in human-web, you should assume it is a decoded URL.
            //
            // The difference is important when accessing the browser API, where
            // you should prefer the "originalURL". But within human-web itself, you
            // should consistently use the internal representation. Otherwise,
            // you risk that it will fail to match identical URLs when they contain
            // special characters.
            //
            // (Note: Why not use the "originalURL" also for the internal representation?
            //  In principle, it should work and could potentially simplify the code a bit,
            //  as you do not have to worry less about the encoding, especially when
            //  communicating with other modules or with the browser API.)
            const originalURL = url;
            const activeURL = CliqzHumanWeb.cleanCurrentUrl(url);

            //Check if the URL is know to be bad: private, about:, odd ports, etc.
            if (CliqzHumanWeb.isSuspiciousURL(activeURL)) {
              return;
            }

            if (activeURL.indexOf('about:')!=1) {

                if (CliqzHumanWeb.state['v'][activeURL] == null) {

                    if (CliqzHumanWeb.contentExtractor.isSearchEngineUrl(activeURL)) {
                        pacemaker.setTimeout(function(url, originalURL) {
                          if (!CliqzHumanWeb) {
                            return;
                          }
                          getContentDocument(originalURL).then((doc) => {
                            CliqzHumanWeb.checkURL(doc, url, 'normal');
                          }).catch((e) => {
                            logger.info(`Failed to get content for originalURL=${originalURL} (internalURL=${url}, details=${e})`);
                          });
                        }, CliqzHumanWeb.WAIT_TIME, activeURL, originalURL);
                    }

                    var status = null;

                    if (CliqzHumanWeb.httpCache[activeURL]!=null) {
                        status = CliqzHumanWeb.httpCache[activeURL]['status'];
                    }

                    var referral = null;
                    var qreferral = null;


                    if (CliqzHumanWeb.linkCache[activeURL] != null) {
                        referral = CliqzHumanWeb.linkCache[activeURL]['s'];
                    }

                    //Get redirect chain
                    var red = [];
                    red = CliqzHumanWeb.getRedirects(activeURL, red);
                    if(red.length == 0){
                        red = null;
                    }

                    //Set referral for the first redirect in the chain.
                    if (red && referral == null) {
                            var redURL = red[0];
                            var refURL = CliqzHumanWeb.linkCache[redURL];
                            if(refURL){
                                referral = refURL['s'];
                            }

                            //Update query cache with the redirected URL

                            if (CliqzHumanWeb.queryCache[redURL]) {
                                CliqzHumanWeb.queryCache[activeURL] = CliqzHumanWeb.queryCache[redURL];
                            }
                    }
                    // Page details to be saved.
                    CliqzHumanWeb.state['v'][activeURL] = {
                      url: activeURL,
                      a: 0,
                      x: null,
                      tin: new Date().getTime(),
                      e: {
                        cp: 0,
                        mm: 0,
                        kp: 0,
                        sc: 0,
                        md: 0
                      },
                      st: status,
                      c: [],
                      ref: referral,
                      red: red
                    };

                    if (referral) {
                        // if there is a good referral, we must inherit the query if there is one
                        if (CliqzHumanWeb.state['v'][referral] && CliqzHumanWeb.state['v'][referral]['qr']) {
                            CliqzHumanWeb.state['v'][activeURL]['qr'] = {}
                            CliqzHumanWeb.state['v'][activeURL]['qr']['q'] = CliqzHumanWeb.state['v'][referral]['qr']['q'];
                            CliqzHumanWeb.state['v'][activeURL]['qr']['t'] = CliqzHumanWeb.state['v'][referral]['qr']['t'];
                            CliqzHumanWeb.state['v'][activeURL]['qr']['d'] = CliqzHumanWeb.state['v'][referral]['qr']['d']+1;

                           //If the depth is greater then two, we need to check if the ref. is of same domain.
                            //If not then drop the QR object, else keep it.
                            if(CliqzHumanWeb.state['v'][activeURL]['qr']['d'] > 2){
                                delete CliqzHumanWeb.state['v'][activeURL]['qr'];
                            }
                            else if(CliqzHumanWeb.state['v'][activeURL]['qr']['d'] == 2){
                                try {
                                    if (parseURL(activeURL).hostname !== parseURL(referral).hostname) {
                                        delete CliqzHumanWeb.state['v'][activeURL]['qr'];
                                    }
                                } catch(ee) {
                                    delete CliqzHumanWeb.state['v'][activeURL]['qr'];
                                }
                            }
                        }
                    }

                    pacemaker.setTimeout(function(currURL) {

                        // Extract info about the page, title, length of the page, number of links, hash signature,
                        // 404, soft-404, you name it
                        //


                            // we cannot get it directly via
                            // var cd = currWin.gBrowser.selectedBrowser.contentDocument;
                            // because during the time of the timeout there can be win or tab switching
                            //
                            //var activeURL = CliqzHumanWeb.currentURL();
                            //if (activeURL != currURL) {}

                            getContentDocument(originalURL).then(function (cd) {

                              if (!CliqzHumanWeb.contentExtractor.isSearchEngineUrl(currURL)) {
                                try {
                                  CliqzHumanWeb.checkURL(cd, currURL,"normal");
                                } catch (e) {
                                }
                                  //Check active usage...
                                  // CliqzHumanWeb.activeUsage += 1;
                                  CliqzHumanWeb.incrActiveUsage();

                              }

                              var x = CliqzHumanWeb.getPageData(currURL, cd);

                              if (x['canonical_url']) {
                                  CliqzHumanWeb.can_urls[currURL] = x['canonical_url'];
                                  CliqzHumanWeb.can_url_match[x['canonical_url']] = currURL;

                              }

                              if (CliqzHumanWeb.state['v'][currURL] != null) {
                                  CliqzHumanWeb.state['v'][currURL]['x'] = x;
                              }

                              if (CliqzHumanWeb.queryCache[currURL]) {
                                  CliqzHumanWeb.state['v'][currURL]['qr'] = CliqzHumanWeb.queryCache[currURL];
                                  delete CliqzHumanWeb.queryCache[currURL];
                              }

                              if (CliqzHumanWeb.state['v'][currURL] != null) {
                                  CliqzHumanWeb.addURLtoDB(currURL, CliqzHumanWeb.state['v'][currURL]['ref'], CliqzHumanWeb.state['v'][currURL]);
                                  CliqzHumanWeb.queryCache[currURL];
                              }

                            }, function () {
                              if (CliqzHumanWeb.debug) {
                                  _log("CANNOT GET THE CONTENT OF : " + currURL);
                              }
                            }).catch( ee => {
                              _log("Error fetching title and length of page: " + ee + " : " + currURL);
                            });

                  }, CliqzHumanWeb.PAGE_WAIT_TIME, activeURL, originalURL);

                }
                else {
                    // wops, it exists on the active page, probably it comes from a back button or back
                    // from tab navigation
                    CliqzHumanWeb.state['v'][activeURL]['tend'] = null;
                    CliqzEvents.pub('HW-activeURL:', {activeURL});
                }
            }
        },
    },
    startPacemaker: function() {
      const tasks = [];

      // TODO - replace with some event to detect when currentURL changes
      // Every second (used to be 250 ms)
      tasks.push(pacemaker.register(function hwCheckCurrentURL() {
        CliqzHumanWeb.currentURL()
        .then( (activeURL) => {
            if (activeURL && (activeURL).indexOf('about:') !== 0) {
                if ((CliqzHumanWeb.counter - CliqzHumanWeb.lastActive) < 5*CliqzHumanWeb.tmult) {
                    // if there has been an event on the last 5 seconds, if not do no count, the user must
                    // be doing something else,
                    //
                    try {
                        CliqzHumanWeb.state['v'][activeURL]['a'] += 1;
                    } catch(ee) {}
                }
            }
        })
        .catch((e) => {
            _log('Error fetching fetching the currentURL: ' + e);
        });

        CliqzHumanWeb.counter += 4;
      }, { timeout: 1000 })); // 1 second


      // TODO - could be replaced by tab events (closed, updated, new)
      // Every 5 seconds
      tasks.push(pacemaker.register(function hwCheckAllOpenPages() {
            CliqzHumanWeb.getAllOpenPages()
            .then((openPages) => {
                var tt = new Date().getTime();

                for (var url in CliqzHumanWeb.state['v']) {
                    if (CliqzHumanWeb.state['v'].hasOwnProperty(url)) {

                        if (openPages.indexOf(url) == -1) {
                            // not opened

                            if (CliqzHumanWeb.state['v'][url]['tend'] == null) {
                                CliqzHumanWeb.state['v'][url]['tend'] = tt;
                            }

                            if (tt - CliqzHumanWeb.state['v'][url]['tend'] > CliqzHumanWeb.deadFiveMts * 60 * 1000) {
                                // move to "dead pages" after 5 minutes
                                CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
                                CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
                                delete CliqzHumanWeb.state['v'][url];
                                delete CliqzHumanWeb.queryCache[url];
                            }
                        } else {
                            // stil opened, do nothing.
                            if (tt - CliqzHumanWeb.state['v'][url]['tin'] > CliqzHumanWeb.deadTwentyMts * 60 * 1000) {
                                // unless it was opened more than 20 minutes ago, if so, let's move it to dead pages

                                CliqzHumanWeb.state['v'][url]['tend'] = null;
                                CliqzHumanWeb.state['v'][url]['too_long'] = true;
                                CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
                                CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
                                delete CliqzHumanWeb.state['v'][url];
                                delete CliqzHumanWeb.queryCache[url];
                                //_log("Deleted: moved to dead pages after 20 mts.");
                                //_log("Deleted: moved to dead pages after 20 mts: " + CliqzHumanWeb.state['m'].length);
                            }
                        }
                    }
                }
            }).catch((ee) => {
              _log(ee);
            });
        }, { timeout: 5000 }));

      // Every 10 seconds
      tasks.push(pacemaker.register(function hwLoadBloomFilterCheck() {
            var ll = CliqzHumanWeb.state['m'].length;
            if (ll > 0) {
                var v = CliqzHumanWeb.state['m'].slice(0, ll);
                CliqzHumanWeb.state['m'] = CliqzHumanWeb.state['m'].slice(ll, CliqzHumanWeb.state['m'].length);

            }
            if(!CliqzHumanWeb.bloomFilter){
                CliqzHumanWeb.loadBloomFilter();
            }
        }, { timeout: 10 * 1000 })); // 10 seconds

      // Clean http cache (when >1 hour)
      tasks.push(pacemaker.register(function hwCleanHttpCache() {
            CliqzHumanWeb.cleanHttpCache();
        }, { timeout: 15 * 1000 })); // 15 seconds

      // Purge clean link cache (when >1 minute)
      tasks.push(pacemaker.register(function hwCleanLinkCache() {
            CliqzHumanWeb.cleanLinkCache();
        }, { timeout: 15 * 1000 })); // 15 seconds

      // Purge ad lookup (when >15 minutes)
      tasks.push(pacemaker.register(function hwPurgeAdLookUp() {
            CliqzHumanWeb.purgeAdLookUp();
        }, { timeout: 15 * 1000 })); // 15 seconds

      // Clean doc cache (>1 hour)
      tasks.push(pacemaker.register(function hwCleanDocCache() {
            CliqzHumanWeb.cleanDocCache();
        }, { timeout: 60 * 1000 })); // 1 minute

      // Every minute
      tasks.push(pacemaker.register(function hwProcessUnchecked() {
            CliqzHumanWeb.listOfUnchecked(1, CliqzHumanWeb.doubleFetchTimeInSec, null, CliqzHumanWeb.processUnchecks);
            CliqzHumanWeb.auxGetQuery();
        }, { timeout: 60 * 1000 })); // 1 minute

      // Every 5 minutes
      tasks.push(pacemaker.register(function hwFlushExpiredCacheEntries() {
          logger.debug('human-web: flush network cache');
          CliqzHumanWeb.network.flushExpiredCacheEntries();
        }, { timeout: 5 * 60 * 1000 })); // 5 minutes

      // Every 20 minutes
      tasks.push(pacemaker.register(function hwExpireQuorumBloomFilter() {
            if (CliqzHumanWeb.debug) {
                _log('Load ts config');
            }
            CliqzHumanWeb.expireQuorumBloomFilter();
        }, { timeout: 20 * 60 * 1000 })); // 20 minutes

      // Every 20 minutes
      tasks.push(pacemaker.register(function hwCheckActiveUsage() {
            if (CliqzHumanWeb.debug) {
                _log('Check if alive');
            }
            CliqzHumanWeb.checkActiveUsage();
        }, { timeout: 20 * 60 * 1000 })); // 20 minutes

      // Every 20 minutes
      tasks.push(pacemaker.register(function hwSaveActionStats() {
            if (CliqzHumanWeb.debug) {
                _log('Save actions stats');
            }
            CliqzHumanWeb.saveActionStats();
            CliqzHumanWeb.sendActionStatsIfNeeded();
        }, { timeout: 20 * 60 * 1000 })); // 20 minutes

      // Every 4 hours
      tasks.push(pacemaker.register(function hwCheckAllOpenPages() {
        CliqzHumanWeb.fetchSafeQuorumConfig();
      }, { timeout: 4 * 60 * 60 * 1000 })); // 4 hours

      return tasks;
    },
    cleanUserTransitions: function(force) {
        for(var query in CliqzHumanWeb.userTransitions['search']) {
            if ((force) || ((CliqzHumanWeb.counter - CliqzHumanWeb.userTransitions['search'][query]['time']) > CliqzHumanWeb.userTransitionsSearchSession*CliqzHumanWeb.tmult)) {

                // the query session is more than 5 minutes old or we are forcing the event,
                // if the condition is met and there are more than two elements in data we
                // must create the signal
                //
                if (CliqzHumanWeb.userTransitions['search'][query]['data'].length > 1) {
                    try {var location = CliqzHumanWeb.getCountryCode();} catch(ee){};
                    var doc = {'q': query, 'sources': CliqzHumanWeb.userTransitions['search'][query]['data'],'ctry': location};
                    if (CliqzHumanWeb.debug) {
                        _log(JSON.stringify(doc,undefined,2));
                    }
                    CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'userTransition.search', 'payload': doc});
                }
                delete CliqzHumanWeb.userTransitions['search'][query];
            }
        }

    },

    unload: function() {
      if (CliqzHumanWeb.rushaWorker) {
        delete CliqzHumanWeb.rushaCallbacks;
        delete CliqzHumanWeb.rushaWorker.onmessage;
        CliqzHumanWeb.rushaWorker.terminate();
        delete CliqzHumanWeb.rushaWorker;
      }
      //Check is active usage, was sent
      CliqzHumanWeb.checkActiveUsage();
      // send all the data
      CliqzHumanWeb.safebrowsingEndpoint.flushSendQueue();

      if (CliqzHumanWeb.pacemakerTasks) {
        CliqzHumanWeb.pacemakerTasks.forEach((task) => {
          task.stop();
        });
        CliqzHumanWeb.pacemakerTasks = null;
      }
      CliqzHumanWeb.safebrowsingEndpoint.unload();

      CliqzHumanWeb.patternsLoader.unload();
      CliqzHumanWeb.doublefetchHandler.unload();
    },
    currentURL: function() {
        return getActiveTab().then(({ url }) => CliqzHumanWeb.cleanCurrentUrl(url));
    },
    cleanCurrentUrl: function(url){
        try {
            url = decodeURIComponent(url.trim());
        } catch(ee) {}

        if (url!=null || url!=undefined) return url;
        else return null;
    },
    pacemakerTasks: null,
    // load from the about:config settings
    captureKeyPressPage: function(ev) {
        if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['keypresspage']|0)) > 1 * CliqzHumanWeb.tmult) {
            if (CliqzHumanWeb.debug) {
                //_log('captureKeyPressPage');
            }
            CliqzHumanWeb.lastEv['keypresspage'] = CliqzHumanWeb.counter;
            CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
            var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
            if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
                CliqzHumanWeb.state['v'][activeURL]['e']['kp'] += 1;
            }
        }
    },
    captureMouseMovePage: function(ev) {
        if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['mousemovepage']|0)) > 1 * CliqzHumanWeb.tmult) {
            if (CliqzHumanWeb.debug) {
                _log('captureMouseMovePage');
            }
            CliqzHumanWeb.lastEv['mousemovepage'] = CliqzHumanWeb.counter;
            CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
            var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
            if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
                CliqzHumanWeb.state['v'][activeURL]['e']['mm'] += 1;
            }
        }
    },
    getURLFromEvent: function(ev) {
        _log(">>>> Get url from event >>> " + ev.target.href);
        try {
            if (ev.target.href!=null || ev.target.href!=undefined) {
                return decodeURIComponent(''+ev.target.href);
            }
            else {
                if (ev.target.parentNode.href!=null || ev.target.parentNode.href!=undefined) {
                    return decodeURIComponent(''+ev.target.parentNode.href);
                }
            }
        }
        catch(ee) {
            if (CliqzHumanWeb.debug) {
                _log('Error in getURLFromEvent: ' + ee);
            }
        }
        return null;
    },
    contextFromEvent: null,

    captureMouseClickPage(ev, contextHTML, href) {
        // if the target is a link of type hash it does not work, it will create a new page without referral
        //

        var targetURL = CliqzHumanWeb.getURLFromEvent(ev) || href;

        _log('captureMouseClickPage>> ' + targetURL);
        if (contextHTML) {
           CliqzHumanWeb.contextFromEvent = {
             html: contextHTML,
             ts: Date.now()
           };
        } else {
           CliqzHumanWeb.contextFromEvent = null;
        }

        if (targetURL!=null) {

            var embURL = CliqzHumanWeb.getEmbeddedURL(targetURL);
            if (embURL!=null) targetURL = embURL;

            // Need to improve.
            var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
            if (CliqzHumanWeb.debug) {
                _log('captureMouseClickPage>> ' + CliqzHumanWeb.counter + ' ' + targetURL  + ' : ' + " active: " + activeURL + " " + (CliqzHumanWeb.state['v'][activeURL]!=null) + " " + ev.target + ' :: ' + ev.target.value  + ' >>' + JSON.stringify(CliqzHumanWeb.lastEv));
            }

            if (CliqzHumanWeb.state['v'][activeURL]!=null) {
                CliqzHumanWeb.linkCache[targetURL] = {'s': ''+activeURL, 'time': CliqzHumanWeb.counter};
                //Fix same link in 'l'
                //Only add if gur. that they are public and the link exists in the double fetch page(Public).it's available on the public page.Such
                //check is not done, therefore we do not push the links clicked on that page. - potential record linkage.
                //We need to check for redirections and use the final link for 'l' this is why the logic is here. This will
                //for sure miss the first time it's see, cause we don't know on mouse click where it redirects.

                var linkURL = targetURL;
                if(CliqzHumanWeb.httpCache[targetURL]){
                    if(CliqzHumanWeb.httpCache[targetURL]['status'] === 301){
                        linkURL = CliqzHumanWeb.httpCache[targetURL]['location'];
                    }
                }

                if(!CliqzHumanWeb.isSuspiciousURL(linkURL) && !CliqzHumanWeb.dropLongURL(linkURL)){
                    CliqzHumanWeb.isAlreadyMarkedPrivate(linkURL, function(_res) {
                        if (_res && (_res['private'] == 0)) {
                            CliqzHumanWeb.state['v'][activeURL]['c'].push({'l': ''+ CliqzHumanWeb.maskURL(linkURL), 't': CliqzHumanWeb.counter});
                        }
                        else if(!_res){
                            CliqzHumanWeb.state['v'][activeURL]['c'].push({'l': ''+ CliqzHumanWeb.maskURL(linkURL), 't': CliqzHumanWeb.counter});

                        }

                    })
                }

            }
        }

        if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['mouseclickpage']|0)) > 1 * CliqzHumanWeb.tmult) {
            if (CliqzHumanWeb.debug) {
                _log('captureMouseClickPage');
            }
            CliqzHumanWeb.lastEv['mouseclickpage'] = CliqzHumanWeb.counter;
            CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
            var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);;
            if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
                CliqzHumanWeb.state['v'][activeURL]['e']['md'] += 1;
            }
        }
    },
    captureScrollPage: function(ev) {
        if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['scrollpage']|0)) > 1 * CliqzHumanWeb.tmult) {
            if (CliqzHumanWeb.debug) {
                _log('captureScrollPage ');
            }

            CliqzHumanWeb.lastEv['scrollpage'] = CliqzHumanWeb.counter;
            CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
            var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
            if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
                CliqzHumanWeb.state['v'][activeURL]['e']['sc'] += 1;
            }
        }
    },
    captureCopyPage: function(ev) {
        if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['copypage']|0)) > 1 * CliqzHumanWeb.tmult) {
            if (CliqzHumanWeb.debug) {
                _log('captureCopyPage');
            }
            CliqzHumanWeb.lastEv['copypage'] = CliqzHumanWeb.counter;
            CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
            var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
            if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
                CliqzHumanWeb.state['v'][activeURL]['e']['cp'] += 1;
            }
        }
    },
    counter: 0,
    tmult: 4,
    tpace: 250,
    lastEv: {},
    lastActive: null,
    lastActiveAll: null,

    network: new Network(),
    doublefetchHandler: new DoublefetchHandler({
      onHostnameResolved: (domain, ip) => {
        // Remember the DNS mapping because not all platforms
        // provide an API for DNS resolution.
        CliqzHumanWeb.network.cacheDnsResolution(domain, ip);
      }
    }),

    init: function() {
      return Promise.resolve().then(() => {
        if(prefs.get("humanWebOptOut", false)) {
          return;
        }

        _log("Init function called:")
        CliqzHumanWeb.log = _log;
        return Promise.resolve()
            .then(() => {
              if (CliqzHumanWeb.db) {
                _log('Closing database connections...');
                return CliqzHumanWeb.db.asyncClose()
                  .then(() => {
                    CliqzHumanWeb.db = undefined;
                    _log('Closing database connections...done');
                  })
                  .catch(e => _log(e));
              } else {
                return Promise.resolve();
              }
            }).then(() => {
              const db = new Storage(CliqzHumanWeb);
              return db.init().then(() => {
                CliqzHumanWeb.db = db;
                _log('Successfully connected to database');
              });
            })
            .then(() => {
              if (CliqzHumanWeb.state == null) {
                  CliqzHumanWeb.state = {};
              }

              if (CliqzHumanWeb.actionStats == null) {
                CliqzHumanWeb.loadActionStats();
              }
              if (CliqzHumanWeb.actionStatsLastSent == null) {
                CliqzHumanWeb.loadActionStatsLastSent();
              }

              // Load bloom filter
              if(!CliqzHumanWeb.bloomFilter){
                CliqzHumanWeb.loadBloomFilter();
              }

              // Load strict queries
              CliqzHumanWeb.loadStrictQueries();

              const promises = [];

              promises.push(CliqzHumanWeb.patternsLoader.init());
              promises.push(CliqzHumanWeb.doublefetchHandler.init());

              // Load config from the backend
              promises.push(CliqzHumanWeb.fetchSafeQuorumConfig());

              // Load quorum bloom filter
              CliqzHumanWeb.loadQuorumBloomFilter();

              // Load active usage stats.
              if (CliqzHumanWeb.activeUsage === null) {
                  CliqzHumanWeb.db.loadRecordTelemetry('activeUsage', (data) => {
                      if (data === null) {
                          CliqzHumanWeb.activeUsage = {};
                      } else {
                          CliqzHumanWeb.activeUsage = JSON.parse(data);
                      }
                  });
              }

              // Check last alive signal sent.
              CliqzHumanWeb.db.loadRecordTelemetry('activeUsageLastSent', (data) => {
                  if (data === null) {
                      // Means we have never sent the signal.
                      CliqzHumanWeb.saveActiveUsageTime();
                  } else {
                      _log(`Active usage last sent  from db as ${data}`);
                      CliqzHumanWeb.activeUsageLastSent = parseInt(data);
                  }
              });

              return Promise.all(promises).then(() => {
                CliqzHumanWeb.safebrowsingEndpoint.init();
                if (CliqzHumanWeb.pacemakerTasks === null) {
                  CliqzHumanWeb.pacemakerTasks = CliqzHumanWeb.startPacemaker();
                }
              });
            });
      });
    },
    state: {'v': {}, 'm': [], '_id': Math.floor( random() * 1000 ) },
    hashCode: function(s) {
        return s.split("").reduce(function(a,b){a=((a<<5)-a)+b.charCodeAt(0);return a&a},0);
    },
    msgSanitize: function(msg){

        //Check and add time , else do not send the message
        msg.channel = CliqzHumanWeb.CHANNEL;

        // Applying '--' , instead of null;
        msg.ts = CliqzHumanWeb.getTS();

        if(!msg.ts || msg.ts == ''){
            return null;
        }

        // Adding anti-duplicate key, so to detect duplicate messages on the backend.
        msg['anti-duplicates'] = Math.floor(random() * 10000000);

        if (msg.action == 'page') {
            if(msg.payload.tend  && msg.payload.tin){
                var duration = msg.payload.tend - msg.payload.tin;
                _log("Duration spent: " +  msg.payload.tend + " : " +  msg.payload.tin + " : " + duration );
            }
            else{
                var duration = null;
                _log("Duration spent: " +  msg.payload.tend + " : " +  msg.payload.tin + " : " + duration );
            }

            msg.payload['dur'] = duration;

            delete msg.payload.tend;
            delete msg.payload.tin;

            // Check for fields which have urls like ref.
            // Check if they are suspicious.
            // Check if they are marked private.
            if(msg.payload.ref){
                if(CliqzHumanWeb.isSuspiciousURL(msg.payload['ref'])){
                    msg.payload['ref'] = null;
                }
                else{
                    msg.payload['ref'] = CliqzHumanWeb.maskURL(msg.payload['ref']);
                }

                // Check if ref. exists in bloom filter, then turn ref to null.
                CliqzHumanWeb.isAlreadyMarkedPrivate(msg.payload.ref, function(_res) {
                    if (_res) {
                        if(_res['private'] == 1 ){
                            msg.payload['ref'] = null;
                        }
                    }
                })
            }

            // Check for title.
            if(msg.payload.x.t){
                if(CliqzHumanWeb.isSuspiciousTitle(msg.payload.x.t)){
                    _log("Suspicious Title: " + msg.payload.x.t);
                    return null;
                }
            }
            else{
                _log("Missing Title: " + msg.payload.x.t);
                return null;
            }

            // Remove C
            if(msg.payload.c) {
                msg.payload.c = null;
            }

            if (CliqzHumanWeb.dropLongURL(msg.payload.url)==true) {
                // the url is not safe, replace by the canonical if exists and is safe
                var canonical_url = msg.payload.x.canonical_url;

                if (canonical_url != null && canonical_url != '' && CliqzHumanWeb.dropLongURL(canonical_url)==false) {
                    // the canonical exists and is ok
                    msg.payload.url = canonical_url;
                }
                else {
                    _log("Suspicious url with no/bad canonical: " + msg.payload.url);
                    return null;
                }
            }
            else {
                var canonical_url = msg.payload.x.canonical_url;
                if (canonical_url != null && canonical_url != '' && CliqzHumanWeb.dropLongURL(canonical_url)==true) {
                    // the canonical is not safe, but the url is, remove only the canonical
                    msg.payload.x.canonical_url = null;
                }
            }

            // validate that is not a shortener url, they are not useful anyway

            var short_url = CliqzHumanWeb.isShortenerURL(msg.payload.url);
            var short_canonical_url = false;
            if (msg.payload.x.canonical_url != null &&  msg.payload.x.canonical_url != '') {
                short_canonical_url = CliqzHumanWeb.isShortenerURL(msg.payload.x.canonical_url);
            }

            if (short_url || short_canonical_url) return null;

            // check if suspiciousURL
            if (CliqzHumanWeb.isSuspiciousURL(msg.payload.url)) return null;

            if (msg.payload.x.canonical_url != null &&  msg.payload.x.canonical_url != '') {
                if (CliqzHumanWeb.isSuspiciousURL(msg.payload.x.canonical_url)) return null;
            }

            //Mask the long ugly redirect URLs
            if(msg.payload.red){
                var cleanRed = [];
                msg.payload.red.forEach(function(e){
                    if(!CliqzHumanWeb.isSuspiciousURL(e)) {
                        cleanRed.push(CliqzHumanWeb.maskURL(e));
                    }
                })
                msg.payload.red = cleanRed;
            }

            // Check for canonical seen or not.
            if(msg.payload['x']['canonical_url']) {
                if(msg.payload['url'] == msg.payload['x']['canonical_url']){
                    _log("Canoncial is same: ");
                    // canonicalSeen = CliqzHumanWeb.canoincalUrlSeen(msg.payload['x']['canonical_url']);
                    if(msg.payload['csb'] && msg.payload['ft']) {
                        _log("Canoncial seen before: ");
                        delete msg.payload.csb;
                        delete msg.payload.ft;
                    }
                }

                // if the url is not replaces by canonical then also clear the csb key.
                if(msg.payload['csb']) delete msg.payload.csb;
            }

        }

        //Check the depth. Just to be extra sure.

        if(msg.payload.qr){
          if(msg.payload.qr.d > 2){
            delete msg.payload.qr;
          }
        }

        // Check if qr.q is suspicious.
        if(msg.payload.qr){
          if (CliqzHumanWeb.isSuspiciousQuery(msg.payload.qr.q)) {
            delete msg.payload.qr;
          }
        }

        //Check for doorway action durl
        if(msg.action=='doorwaypage') {
            if((CliqzHumanWeb.isSuspiciousURL(msg.payload['durl'])) || (CliqzHumanWeb.isSuspiciousURL(msg.payload['url']))){
                return null;
            }
            if((CliqzHumanWeb.dropLongURL(msg.payload['durl'])) || (CliqzHumanWeb.dropLongURL(msg.payload['url']))){
                return null;
            }
        }


        //Remove the msg if the query is too long,

        if(msg.action=='query' || msg.action == 'anon-query') {
            //Remove the msg if the query is too long,
            if ((msg.payload.q == null) || (msg.payload.q == '')) {
                return null;
            }
            else {
                if (CliqzHumanWeb.isSuspiciousQuery(msg.payload.q)) {
                    return null;
                }
            }

            // We need to check the URLs for suspicious patterns,
            // Remove the suspicious URLs and limit them to 8 results.
            // Ensure reordering is done.
            if (msg.payload.r) {
                let cleanR = [];
                let newR = {};

                Object.keys(msg.payload.r).forEach( eachResult => {
                    if (!CliqzHumanWeb.isSuspiciousURL(msg.payload.r[eachResult].u)) {
                        cleanR.push(msg.payload.r[eachResult]);
                    }
                });
                // If there are too few results, the search query tends to be very specific.
                // To avoid leaking personal information, drop them.
                //
                // Note: This limit only takes into account the results on the first page.
                // For navigational queries, like 'facebook', 'web', 'sueddeutsche', although
                // there are billions of results, only few of them are on the first page.
                // That it where we currently set the threshold.
                if (cleanR.length < 4) {
                  _log(`Dropping message for query ${msg.payload.q}, as there are too few search results.`);
                  return null;
                }
                cleanR.slice(0,8).forEach( (each, idx) => {
                    newR[idx] = each;
                });

                _log("Original: " + JSON.stringify(msg.payload.r));
                _log("New: " + JSON.stringify(newR));
                msg.payload.r = newR;
            }
        }

        return msg;


    },
    // ****************************
    // telemetry, PREFER NOT TO SHARE WITH utils for safety, blatant rip-off though
    // ****************************
    notification: function(payload){
      try {var location = CliqzHumanWeb.getCountryCode();} catch(ee){};

      if(payload && typeof(payload) === 'object'){
        payload['ctry'] = location;
        CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'telemetry', 'payload': payload});
      } else {
        _log("Not a valid object, not sent to notification");
      }
    },

    async telemetry(msg, instantPush) {
      const { accepted, rejectDetails } = await CliqzHumanWeb._telemetry(msg, instantPush);
      if (accepted) {
        _log('all checks passed: telemetry message added to the sent queue:', msg);
      } else {
        _log('telemetry message has been discarded:', rejectDetails, msg);
      }
    },

    // TODO: migrate actionsStats to Anolysis
    async _telemetry(msg, instantPush) {
      function discard(reason) {
        return { accepted: false, rejectDetails: reason };
      }
      function accept() {
        return { accepted: true };
      }

      if (!CliqzHumanWeb || //might be called after the module gets unloaded
          prefs.get('humanWebOptOut', false)) {
        return discard('human web disabled');
      }
      if (isPrivateMode(getWindow())) {
        return discard('private mode');
      }

      // Sanitize message before doing the quorum check.
      msg.ver = CliqzHumanWeb.VERSION;
      msg = CliqzHumanWeb.msgSanitize(msg);
      if (!msg) {
        return discard('rejected by sanitize checks');
      }

      // Check if host is private or not.
      msg = await CliqzHumanWeb.network.sanitizeUrlsWithPrivateDomains(msg);
      if (!msg) {
        return discard('failed local domain check');
      }

      let isSafe;
      try {
        isSafe = await CliqzHumanWeb.safeQuorumCheck(msg);
      } catch (e) {
        _log('Exception during safe quorum check', e);
        CliqzHumanWeb.incrActionStats(`dropped-${e}`);
        return discard('failed safe quorum check: ${e}');
      }

      _log('Quorum consent?', isSafe);
      if (!isSafe) {
        CliqzHumanWeb.incrActionStats('droppedQC');
        return discard('failed safe quorum check');
      }

      // Check and sanitize all other urls.
      try {
        msg = await CliqzHumanWeb.quorumCheckOtherUrls(msg);
      } catch (e) {
        _log('Error while checking other urls for quorum', e);
        return discard('failed safe quorum check for other urls');
      }

      // message passed all checks and is ready to be sent
      CliqzHumanWeb.incrActionStats(msg.action);

      // do not wait for the promise to complete (keeping the old fire-and-forget API)
      const start = Date.now();
      CliqzHumanWeb.safebrowsingEndpoint.send(msg)
        .then(() => {
          logger.debug('Successfully sent message after', (Date.now() - start) / 1000.0, 'sec', msg);
        })
        .catch((e) => {
          logger.info(`Finally giving up on sending message (reason: ${e}, elapsed: ${(Date.now() - start) / 1000.0} sec)`, msg);
        });

      return accept();
    },

    pushTelemetry() {
      CliqzHumanWeb.safebrowsingEndpoint.flushSendQueue();
    },

    // ************************ Database ***********************
    // source modules/CliqzHistory
    // *********************************************************
    auxSameDomain: function(url1, url2) {
        try {
            var d1 = parseURL(url1).hostname.replace('www.','');
            var d2 = parseURL(url2).hostname.replace('www.','');
            return d1==d2;
        } catch(ee) {
            return false;
        }
    },
    isAlreadyMarkedPrivate: function(url, callback) {
        var hash = (md5(url)).substring(0,16);
        var r = null;
        if(CliqzHumanWeb.bloomFilter){
            var sta = CliqzHumanWeb.bloomFilter.testSingle(hash);
            if(sta) {
                r = {"hash": hash, "private": 1}
            }
            else{
                r = {"hash": hash, "private": 0}
            }
        }
        callback(r);
        return r;
    },

  // to invoke in console: CliqzHumanWeb.listOfUnchecked(1000000000000, 0, null, function(x) {console.log(x)})
    forceDoubleFetch: function(url) {
        CliqzHumanWeb.listOfUnchecked(1000000000000, 0, url, CliqzHumanWeb.processUnchecks);
    },

    /**
     * Returns true if the given string contains any text that looks
     * like an email address. The check is conservative, that means
     * false positives are expected, but false negatives are not.
     */
    checkForEmail(str) {
      return /[a-z0-9\-_@]+(@|%40|%(25)+40)[a-z0-9\-_]+\.[a-z0-9\-_]/i.test(str);
    },

    checkForLongNumber: function(str, max_number_length) {

        var cstr = str.replace(/[^A-Za-z0-9]/g,'');

        var lcn = 0;
        var maxlcn = 0;
        var maxlcnpos = null;

        for(let i=0;i<cstr.length;i++) {
            if (cstr[i]>='0' && cstr[i]<='9') lcn+=1;
            else {
                if (lcn > maxlcn) {
                    maxlcn = lcn;
                    maxlcnpos = i;
                    lcn = 0;
                }
                else lcn = 0;
            }
        }

        if (lcn > maxlcn) {
            maxlcn = lcn;
            maxlcnpos = cstr.length;
            lcn = 0;
        }
        else lcn = 0;

        if (maxlcnpos!=null && maxlcn > max_number_length) return cstr.slice(maxlcnpos-maxlcn,maxlcnpos);
        else return null;

    },

    checkURL(pageContent, url, ruleset) {
      return CliqzHumanWeb.contentExtractor.checkURL(pageContent, url, ruleset);
    },

    /**
     * Used in context-search module
     *
     * TODO: A safer option would be to hard-code the list of
     * search engines. Otherwise, updating the patterns can potentially
     * change the search results that we show.
     */
    isSearchEngineUrl(url) {
      return CliqzHumanWeb.contentExtractor.isSearchEngineUrl(url);
    },

    aggregateMetrics:function (metricsBefore, metricsAfter){
        var aggregates =    {"cp": 0,"mm": 0,"kp": 0,"sc": 0,"md": 0};
        if (CliqzHumanWeb.debug) {
            _log("aggregates: " + JSON.stringify(metricsBefore) + JSON.stringify(metricsAfter));
        }

        var _keys = Object.keys(aggregates);
        for(var i=0;i<_keys.length;i++){
             aggregates[_keys[i]] = metricsBefore[_keys[i]] + metricsAfter[_keys[i]];
        }
        if (CliqzHumanWeb.debug) {
            _log("aggregates: " + JSON.stringify(aggregates));
        }


        return aggregates;
    },
    isSuspiciousTitle: function(title) {
        // 1. Need to check if the title is suspicious or not.
        // 2. Title should should not contain number greater than 8.
        // 3. Title should not contain html.

        if (title.length > 500) return true;
        var vt = title.split(' ');
        for(var i=0;i<vt.length;i++) {
            if (vt[i].length>CliqzHumanWeb.rel_segment_len) {
                var cstr = vt[i].replace(/[^A-Za-z0-9]/g,'');
                if (cstr.length > CliqzHumanWeb.rel_segment_len) {
                    if (CliqzHumanWeb.isHash(cstr)) return true;

                    var pp = CliqzHumanWeb.isHashProb(cstr.toLowerCase());
                    if (pp < CliqzHumanWeb.probHashThreshold*1.5) {
                            return true;
                    }
                }
            }
            var cstr = vt[i].replace(/[^A-Za-z0-9]/g,'');
            if (CliqzHumanWeb.checkForLongNumber(cstr, 8) != null) {
                return true;
            }

            if (CliqzHumanWeb.checkForEmail(cstr)) {
                return true;
            }

            if(/<[^<]+>/.test(cstr)){
                return true;
            }
        }

        if (CliqzHumanWeb.checkForLongNumber(title, 8) != null) {
            return true;
        }

        if (CliqzHumanWeb.checkForEmail(title)) {
            return true;
        }

        if(/<[^<]+>/.test(title)){
            return true;
        }

        return false;

    },
    auxProbString: function(h, p, s, c) {

        // h[i] = k : there are k bins (letter) with i balls (repetitions) on it,
        // h[2] = 1 : there is one letter that is repeated twice


        if (h[1] == s)  {
            // recursion stop condition
            return 1.0;
        }
        else {
            var tot_p = 0.0;
            var kkp = 1.0;

            for(let i=h.length-1; i>1; i--) {
                if (h[i]!=0) {
                    // it has repetitions,
                    var pi = (h[i-1]+1)*p;
                    var h2 = h.slice();

                    h2[i] -= 1;

                    if (false && (h2[i]==0 && h2[i-1]==0 && i>1)) {
                        // minimize number of recursions
                        h2[i-1] += 1;

                        kkp = kkp * p;

                    }
                    else {
                        h2[i-1] += 1;

                        var lab = h2.join('-');
                        if (!c[lab]) c[lab] = CliqzHumanWeb.auxProbString(h2, p, s, c);
                        tot_p += (kkp * pi * c[lab]);
                    }
                }
            }

            return tot_p;
        }
    },
    probString: function(str) {

        var bins = {}

        for(let i=0;i<str.length;i++) bins[str[i]] = (bins[str[i]] || 0) + 1

        var keys = Object.keys(bins);

        var p = 1.0 / keys.length;
        var k = str.length - keys.length;

        var h = []
        for(let i=0;i<str.length+1;i++) h[i] = 0;
        var s = 0;

        var max_freq = 0;
        for(let i=0;i<keys.length+1;i++) {
            var freq = bins[keys[i]];

            if (freq > 0) {
                h[bins[keys[i]]] += 1;
                s += 1;

                if (freq > max_freq) {
                    max_freq = freq;
                }
            }
        }

        h = h.slice(0, max_freq+1);

        var cache = {};
        var prob_conf = CliqzHumanWeb.auxProbString(h, p, s, cache);

        // probability of the string to be random, e.g. hash. Better to keep it for
        // strings > 15, lower probability means not random, > 0.25, random.

        return prob_conf;

    },
    isHashProb: function(str) {

        var log_prob = 0.0;
        var trans_c = 0;
        str = str.replace(/[^A-Za-z0-9]/g,'');

        for(var i=0;i<str.length-1;i++) {

            var pos1 = CliqzHumanWeb.probHashChars[str[i]]
            var pos2 = CliqzHumanWeb.probHashChars[str[i+1]];

            if (pos1 && pos2) {
                log_prob += CliqzHumanWeb.probHashLogM[pos1][pos2];
                trans_c += 1;
            }

        }

        if (trans_c > 0) return Math.exp(log_prob/trans_c);
        else return Math.exp(log_prob);

    },
    isHash: function(str) {
        var p = CliqzHumanWeb.isHashProb(str);
        return (p < CliqzHumanWeb.probHashThreshold);
    },
    incrActiveUsage: function() {
        let t = CliqzHumanWeb.getTime();

        if (!CliqzHumanWeb.activeUsage.hasOwnProperty(t)) {
            CliqzHumanWeb.activeUsage[t] = 0;
        }
        CliqzHumanWeb.activeUsage[t] += 1;

        // Persist active usage count.
        CliqzHumanWeb.db.saveRecordTelemetry('activeUsage', JSON.stringify(CliqzHumanWeb.activeUsage), (result) => {
            _log('Active usage stats saved:', result);
        });
    },
    checkActiveUsage: function(){
        /*
          This event is generated every 60 minutes to check a user
          was active or not. We treat the user to be active if in the
          last hour, the user visited two non search pages.
          Get the active usage from DB.
          hen generate & send payload.
          Update the time last sent, and reset the value to 0.

          Sample payload:
            {
            "action": "alive",
            "ver": "2.7",
            "type": "humanweb",
            "payload": {
            "status": true,
            "ctry": "de",
            "t": "2016110909"
            },
            "ts": "2016110909"
            }
        */

        const tDiff = parseInt((new Date().getTime() - CliqzHumanWeb.activeUsageLastSent) / 1000);
        if (tDiff > 3600) {
            const activeHours = Object.keys(CliqzHumanWeb.activeUsage);
            activeHours.forEach( (h) => {
                if ( (CliqzHumanWeb.activeUsage[h] > CliqzHumanWeb.activeUsageThreshold) &&
                     (h != CliqzHumanWeb.getTime())
                    ) {
                    CliqzHumanWeb.sendAliveMessage(h);
                    delete CliqzHumanWeb.activeUsage[h];
                    CliqzHumanWeb.db.saveRecordTelemetry('activeUsage', JSON.stringify(CliqzHumanWeb.activeUsage), (result) => {
                        _log('Active usage stats saved:', result);
                    });
                    CliqzHumanWeb.saveActiveUsageTime();
                }
            });
        }

    },
    sendAliveMessage: function(h) {
        const payload = {
            status: true,
            t: h,
            ctry: CliqzHumanWeb.getCountryCode() // Need to fix this.
        };

        _log(`Sending alive message for the hour: ${h} , ${JSON.stringify(payload)}`);

        CliqzHumanWeb.telemetry({
            type: CliqzHumanWeb.msgType,
            action: 'alive',
            payload:payload
        });
    },
    loadActionStats: function() {
        CliqzHumanWeb.db.loadRecordTelemetry('actionStats', function(data) {
            if (data==null) {
                _log("There was no data on action stats");
                CliqzHumanWeb.actionStats = {};
            }
            else {
                try {
                    CliqzHumanWeb.actionStats = JSON.parse(data);
                } catch(ee) {
                    CliqzHumanWeb.actionStats = {};
                }
            }
        });
    },
    loadActionStatsLastSent: function() {
        CliqzHumanWeb.db.loadRecordTelemetry('actionStats_last_send', function(data) {
            if (data==null) {
                _log("There was no data on CliqzHumanWeb.actionstats");
                CliqzHumanWeb.actionStatsLastSent = CliqzHumanWeb.getTime().slice(0,8);
                CliqzHumanWeb.saveActionStatsLastSent();
            }
            else CliqzHumanWeb.actionStatsLastSent = data;
        });
    },
    incrActionStats: function(action) {
        if(!CliqzHumanWeb.actionStats) return;
        if (CliqzHumanWeb.actionStats[action] == null) CliqzHumanWeb.actionStats[action] = 0;
        if (CliqzHumanWeb.actionStats['total'] == null) CliqzHumanWeb.actionStats['total'] = 0;
        CliqzHumanWeb.actionStats[action]++;
        CliqzHumanWeb.actionStats['total']++;

    },
    saveActionStats: function() {
        if (CliqzHumanWeb.actionStats && Object.keys(CliqzHumanWeb.actionStats).length > 0) {
            CliqzHumanWeb.db.saveRecordTelemetry('actionStats', JSON.stringify(CliqzHumanWeb.actionStats), (result) => {
                _log('actionStats saved');
            });
        }
    },
    saveActiveUsageTime: function() {
        let t = new Date().getTime();
        CliqzHumanWeb.db.saveRecordTelemetry('activeUsageLastSent', t, (result) => {
            CliqzHumanWeb.activeUsageLastSent = t;
            _log(`Active usage last sent as ${t}`);
        });
    },
    sendActionStats: function() {
        var payl;
        if (CliqzHumanWeb.actionStats && Object.keys(CliqzHumanWeb.actionStats).length > 0) {
            payl = {'data': CliqzHumanWeb.actionStats};

            CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'hw.telemetry.actionstats', 'payload': payl});

            // reset the state
            CliqzHumanWeb.actionStats = {};
            CliqzHumanWeb.saveActionStats();
        }
    },
    saveActionStatsLastSent: function() {
        CliqzHumanWeb.db.saveRecordTelemetry('actionStats_last_send', CliqzHumanWeb.actionStatsLastSent, (result) => {
            _log('actionStats_last_send saved:', result);
        });
    },
    saveStrictQueries: function() {
        _log("Saving local table");
        CliqzHumanWeb.db.saveRecordTelemetry('localStrictQueries', JSON.stringify(CliqzHumanWeb.strictQueries), (result) => {
            _log('localStrictQueries saved:', result);
        });
    },
    sendActionStatsIfNeeded: function() {
        // Send action stats once per day.
        // Day resolution.
        var timestamp = CliqzHumanWeb.getTime().slice(0,8);


        if (timestamp != CliqzHumanWeb.actionStatsLastSent) {
            // it's not the same timestamp (hour) of the last time that was sent
            // or the first install (defaults to current timestamp)

            CliqzHumanWeb.actionStatsLastSent = timestamp;
            CliqzHumanWeb.saveActionStatsLastSent();
            CliqzHumanWeb.sendActionStats();
        }
    },
    dumpBloomFilter: function(){
        var bf = [].slice.call(CliqzHumanWeb.bloomFilter.buckets);
        if(bf){
            CliqzHumanWeb.db.saveRecordTelemetry('bf', bf.join('|'), (result) => {
                _log('bloom filter saved:', result);
            });
        }

    },
    loadBloomFilter: function(){
        CliqzHumanWeb.db.loadRecordTelemetry('bf', function(data) {
            if (data==null) {
                _log("There was no data on CliqzHumanWeb.bf");
                CliqzHumanWeb.bloomFilter = new BloomFilter(Array(bloomFilterSize).join('0'),bloomFilterNHashes);
            }
            else {
                var _data = data.split("|").map(Number);
                CliqzHumanWeb.bloomFilter = new BloomFilter(_data,bloomFilterNHashes);
            }

        });

    },
    loadStrictQueries: function(){
        CliqzHumanWeb.db.loadRecordTelemetry('localStrictQueries', function(data) {
            if (data==null || data.length == 0) {
                _log("There was no data on CliqzHumanWeb.bf");
                CliqzHumanWeb.strictQueries = [];
            }
            else {
                CliqzHumanWeb.strictQueries = JSON.parse(data);
            }

        });

    },
    auxGetQuery: function(){
        CliqzHumanWeb.strictQueries.forEach( function(e, idx) {
            var t = Date.now();
            if((t - e.ts) > (e.tDiff * 60 * 1000)) {
                CliqzHumanWeb.auxGetPageData(e.qurl, null, e.qurl,function(url, page_data, ourl, x){
                    let cd = CliqzHumanWeb.docCache[url]['doc'];
                    CliqzHumanWeb.checkURL(cd, url, "strict");
                }, function(a,b,c,d){
                    _log("Error aux>>>> " + d)
                });
                CliqzHumanWeb.strictQueries.splice(idx, 1);
                CliqzHumanWeb.saveStrictQueries();
            }
        })
    },
    isSuspiciousQuery: function(query) {
        //Remove the msg if the query is too long,
        if (query.length > 50) return true;
        if (query.split(' ').length > 7) return true;

        // Remove the msg if the query contains a number longer than 7 digits
        // can be 666666 but also things like (090)90-2, 5555 3235
        // note that full dates will be removed 2014/12/12
        //
        var haslongnumber = CliqzHumanWeb.checkForLongNumber(query, 7);
        if (haslongnumber!=null) return true;


        //Remove if email (exact), even if not totally well formed
        if (CliqzHumanWeb.checkForEmail(query)) {
          return true;
        }
        //Remove if query looks like an http pass
        if (/[^:]+:[^@]+@/.test(query)) return true;

        var v = query.split(' ');
        for(let i=0;i<v.length;i++) {
            if (v[i].length > 20) return true;
            if (/[^:]+:[^@]+@/.test(v[i])) return true;
        }

        if (query.length > 12) {

            var cquery = query.replace(/[^A-Za-z0-9]/g,'');

            if (cquery.length > 12) {
                var pp = CliqzHumanWeb.isHashProb(cquery);
                // we are a bit more strict here because the query
                // can have parts well formed
                if (pp < CliqzHumanWeb.probHashThreshold*1.5) return true;
            }
        }
        return false;
    },
    sanitizeResultTelemetry: function(data) {
        /*
        Sanitize result telemetry. Does NOT send it, but returns sanitized telemetry.
        */
        // If there is a problem in initializing human-web, we should return.
        if(CliqzHumanWeb && CliqzHumanWeb.counter === 0) {
          return Promise.reject("HumanWeb not initialized");
        }

        const msg = data.msg;
        const msgType = data.type;

        let query = data.q;
        let sanitisedQuery = null;
        let url = msg.u;

        const hostNameDetails = getDetailsFromUrl(url);
        const hostName = hostNameDetails.host;

        // Check if there is a query.
        if (!query || query.length == 0) {
          _log("No Query");
          return Promise.reject("No Query");
        }

        // If suspicious query.
         if (CliqzHumanWeb.isSuspiciousQuery(query)) {
            _log("Query is suspicious");
            sanitisedQuery = "(PROTECTED)";
        }

        // Check if query is like a URL.
        let query_parts = parseURL(query);
        let queryLikeURL = false;
        if (query_parts && (query_parts.protocol === "http"  || query_parts.protocol === "https"  || query_parts.protocol === "www")) {
            queryLikeURL = true;
        }

        if (queryLikeURL &&
            (CliqzHumanWeb.isSuspiciousURL(query) || CliqzHumanWeb.dropLongURL(query))) {
            _log("Query is dangerous");
            sanitisedQuery = "(PROTECTED)"
        };

        // Queries also appear on the URL, in which
        // case we need to check whether it's a URL or not.
        if (hostName.length > 0 && url && url.length > 0) {
            // Check if the URL is marked as already private.

            // If there is a problem in initializing human-web bloom-filter, we should return.
            if(!CliqzHumanWeb.bloomFilter) {
              return Promise.reject("Bloom filter not initialized");
            }
            const urlPrivate = CliqzHumanWeb.bloomFilter.testSingle(md5(url));
            if (urlPrivate) {
              _log("Url is already marked private");
              return Promise.reject("Url is already marked private");
            }

            // Check URL is suspicious
            if (CliqzHumanWeb.isSuspiciousURL(url)) {
                _log("Url is suspicious");
                url = '(PROTECTED)';
            }

            // Check URL is dangerous, with strict DROPLONGURL.
            if (CliqzHumanWeb.dropLongURL(url, {strict: true})) {
                // If it's Google / Yahoo / Bing. Then mask and send them.
                if (CliqzHumanWeb.contentExtractor.isSearchEngineUrl(url)) {
                  url = CliqzHumanWeb.maskURL(url);
                } else {
                  url = '(PROTECTED)';
                }

            }

            // Check for DNS.
            // isHostNamePrivate can leak sensitive information while doing the DNS lookup
            // https://bugzilla.mozilla.org/show_bug.cgi?id=1469709
            // return CliqzHumanWeb.network.isHostNamePrivate(url).then((res) => {
            return Promise.resolve().then(() => {
              let maskedURL;
              let parsedUrl = parseURL(url);
              if(parsedUrl && parsedUrl.hostname) {
                maskedURL = CliqzHumanWeb.maskURL(url);
              } else {
                maskedURL = "(PROTECTED)";
              }

              // Cases when query and URL are same.
              if (url === query){
                  sanitisedQuery = "(PROTECTED)";
                  maskedURL = sanitisedQuery;
              }
              // Check if query failed any checks, then replace it with
              // a placeholder.
              if (sanitisedQuery) {
                  query = sanitisedQuery;
                  maskedURL = sanitisedQuery;
              }
              return {
                query,
                url: maskedURL,
                data,
              };
            });
        } else {
            // The URL was not a URL hence drop it.
            // Check if query failed any checks, then replace it with
            // a placeholder.

            // As a final check, if query is a single token, and can be a private domain.
            // Like my.adminportal.com
            if(query.indexOf(' ') === -1 && query.indexOf('.') > -1) {
                return CliqzHumanWeb.network.isHostNamePrivate(query).then((res) => {
                    if (res) {
                    _log("Private Domain");
                        sanitisedQuery = "(PROTECTED)";
                    }
                    if (sanitisedQuery) {
                        query = sanitisedQuery;
                    }
                    return {
                      query,
                      url: null,
                      data,
                    };
                });
            } else {
                if (sanitisedQuery) {
                    query = sanitisedQuery;
                }
                return Promise.resolve({
                  query,
                  url: null,
                  data,
                });
            }
        }
    },
    sendResultTelemetry: function(query, url, data){
        const params = encodeURIComponent(query) +
                        (data.msg.a ? '&a=' + encodeURIComponent(data.msg.a) : '') +
                        '&i=' + data.msg.i +
                        (url ? '&u=' + encodeURIComponent(url) : '') +
                        data.s +
                        data.msg.o +
                        (data.msg.e ? '&e=' + data.msg.e : '');
        const payLoadURL = data.endpoint + params;
        httpGet(payLoadURL);
    },
    validFrameCount: function(struct_bef, struct_aft) {
        //
        // To take into account, the transition state, when the extension is updated
        // Data saved in the DB will not have the key nifsh, hence we should return true
        // for those cases.
        //

        if (struct_bef.nifsh == null || struct_aft.nifsh == null || struct_bef.nifsh !=struct_aft.nifsh) {
            _log("fovalidDoubleFetch: number of internal iframes does not match");
            return false;
        }

        return true;
    },
    validFrameSetCount: function(struct_bef, struct_aft) {
        //
        // To take into account, the transition state, when the extension is updated
        // Data saved in the DB will not have the key nfsh, hence we should return true
        // for those cases.
        //

        if (struct_bef.nfsh ==null || struct_aft.nfsh==null || struct_bef.nfsh!=struct_aft.nfsh) {
            _log("fovalidDoubleFetch: number of internal frameset does not match");
            return false;
        }

        return true;

    },
    performQC: function(msg){
        if (msg.action === "page") {
            let parse_url = parseURL(msg.payload.url);

            if (msg.payload.qr && (msg.payload.qr.t === "cl" || msg.payload.qr.t === "othr")) {
                if (parse_url && parse_url.path.length > 1 || (parse_url.query_string && parse_url.path.length == 1 && parse_url.query_string.length > 1)) return true;
            } else if (!msg.payload.qr) {
                if (parse_url && parse_url.path.length > 1 || (parse_url.query_string && parse_url.path.length == 1 && parse_url.query_string.length > 1)) return true;
            }
        }
        return false;
    },
    safeQuorumCheck: function(msg) {
        //
        // Check for conditions.
        //

        let promise = new Promise( (resolve, reject) => {
            if (CliqzHumanWeb.performQC(msg)) {
                _log("Perform QC: true");
                let url = msg.payload.url;

                return CliqzHumanWeb.sha1(url)
                    .then( CliqzHumanWeb.sendQuorumIncrement )
                    .then( CliqzHumanWeb.getQuorumConsent )
                    .then( result => resolve(result))
                    .catch( err => {
                        _log("Error while safe quorum check: " + err);
                        reject("quorumcheck");
                    });

            } else {
                _log("Perform QC: false");
                resolve(true);
            }
        });
        return promise;
    },
    initWorker: function() {
      if (!CliqzHumanWeb.rushaWorker) {
        CliqzHumanWeb.rushaWorker = new Worker(`${config.baseURL}human-web/rusha.bundle.js`);
        CliqzHumanWeb.rushaCallbacks = new Map();
        CliqzHumanWeb.rushaWorker.onmessage = ({ data }) => {
          const { id, error, hash } = data;
          const { resolve, reject } = CliqzHumanWeb.rushaCallbacks.get(id);
          CliqzHumanWeb.rushaCallbacks.delete(id);
          if (error) {
            reject(error);
          } else {
            resolve(hash);
          }
        };
      }
    },
    sha1: function(s) {
      if (isEdge) {
        CliqzHumanWeb.initWorker();
        return new Promise((resolve, reject) => {
          const id = Math.random() * (2 ** 53);
          CliqzHumanWeb.rushaCallbacks.set(id, { resolve, reject });
          CliqzHumanWeb.rushaWorker.postMessage({ id, data: toUTF8(s) });
        });
      }
      return sha1(s);
    },
    sendQuorumIncrement: function(hashedUrl){
        let promise = new Promise( (resolve, reject) => {
          // Check for hashed URL in quorum bloom filter;
          CliqzHumanWeb.isPageVisitedQuorumBloomFilter(hashedUrl)
            .then( status => {
                _log("Page already visited: " + status);
                if (status) {
                    resolve(hashedUrl);
                } else {
                    let payload = `?hu=${hashedUrl}&oc=${CliqzHumanWeb.oc}`;
                    let _rp = `${CliqzHumanWeb.SAFE_QUORUM_ENDPOINT}incrquorum`;
                    return CliqzHumanWeb.hpn.action('sendInstantMessage', _rp, payload);
                }
             })
            .then( CliqzHumanWeb.setPageVisitQuorumBloomFilter(hashedUrl))
            .then( () => resolve(hashedUrl))
            .catch( err => {
                _log("Error while sending send quorum increment" + err);
                reject("quorumincr");
            });
        });
        return promise;
    },
    async getQuorumConsent(hashedUrl) {
      const payload = "?hu=" + hashedUrl;
      const _rp = CliqzHumanWeb.SAFE_QUORUM_ENDPOINT + "checkquorum";

      const quorumReached = await CliqzHumanWeb.hpn.action('sendInstantMessage', _rp, payload);
      if (quorumReached !== true && quorumReached !== false) {
        throw new Error(`Got unexpected result from quorum server: ${quorumReached}`);
      }
      return quorumReached;
    },
    registerQuorumBloomFilters: function(){
        let promise = new Promise( (resolve, reject) => {
            // Check for current date.
            let currentDay = CliqzHumanWeb.getTime().slice(0,8);
            // Check if quorumBF exists for current date.
            if (Object.keys(CliqzHumanWeb.quorumBloomFilters).indexOf(currentDay) === -1) {
                _log("Need to create quorum bloom filter for: " + currentDay);
                let bloomFilterSize = 10000; // User is unlikely to visit more than 10000 URLs in day. Size is approx. 12 KB. per Bloom filter.
                let bloomFilter = new BloomFilter(Array(bloomFilterSize).join('0'),bloomFilterNHashes);
                CliqzHumanWeb.quorumBloomFilters[currentDay] = bloomFilter;
            }

            let bf = CliqzHumanWeb.quorumBloomFilters[currentDay];
            resolve(bf);
        });

        return promise;
    },
    setPageVisitQuorumBloomFilter: function(hashedUrl) {
        CliqzHumanWeb.registerQuorumBloomFilters().then( bf => {
            if (bf) {
                bf.addSingle(hashedUrl);
                CliqzHumanWeb.dumpQuorumBloomFilter();
                Promise.resolve(true);
            }
        });
    },
    isPageVisitedQuorumBloomFilter: function(hashedUrl) {
        let promise = new Promise( (resolve, reject) => {
            Object.keys(CliqzHumanWeb.quorumBloomFilters).forEach( eachDay => {
                var status = CliqzHumanWeb.quorumBloomFilters[eachDay].testSingle(hashedUrl);
                if (status) {
                    resolve(true);
                }
            });
            resolve(false);
        });
        return promise;
    },
    dumpQuorumBloomFilter: function(){
        let quorumBF = {};

        Object.keys(CliqzHumanWeb.quorumBloomFilters).forEach( eachDay => {
            let _bf = [].slice.call(CliqzHumanWeb.quorumBloomFilters[eachDay].buckets);
            quorumBF[eachDay] = _bf.join("|");
        });
        CliqzHumanWeb.db.saveRecordTelemetry('quorumbf', JSON.stringify(quorumBF), (result) => {
            _log('quorum bloom filter saved:', result);
        });
    },
    loadQuorumBloomFilter: function(){
        CliqzHumanWeb.db.loadRecordTelemetry('quorumbf', function(data) {
            if (data) {
                let jData = JSON.parse(data);
                _log("Loading quorum bloom filter");
                Object.keys(jData).forEach( eachDay => {
                    let _data = jData[eachDay].split("|").map(Number);
                    let bloomFilter = new BloomFilter(_data,bloomFilterNHashes);
                    CliqzHumanWeb.quorumBloomFilters[eachDay] = bloomFilter;
                });
            }
        });

    },
    expireQuorumBloomFilter: function(){
        // Need to pass the string as YYYY, MM-1, DD to convert to date object.
        let currentDay = CliqzHumanWeb.getTime().slice(0,8);
        let dateToday = new Date(currentDay.slice(0,4), currentDay.slice(4,6)-1, currentDay.slice(6,8));

        Object.keys(CliqzHumanWeb.quorumBloomFilters).forEach( eachDay => {
            let registerDate = new Date(eachDay.slice(0,4), eachDay.slice(4,6)-1, eachDay.slice(6,8));
            let diff = (dateToday - registerDate);
            if (diff > CliqzHumanWeb.keyExpire) {
                delete CliqzHumanWeb.quorumBloomFilters[eachDay];
            }
        });
        CliqzHumanWeb.dumpQuorumBloomFilter();
    },
    async fetchSafeQuorumConfig() {
      try {
        const req = await fetch(CliqzHumanWeb.SAFE_QUORUM_PROVIDER);
        if (!req.ok) {
          throw new Error(req.statusText);
        }
        const json = await req.json();

        if (CliqzHumanWeb) {
          CliqzHumanWeb.keyExpire = json.expiry * (24 * 60 * 60 * 1000); // Backend sends it in days;
          CliqzHumanWeb.oc = json.oc;
          CliqzHumanWeb.quorumThreshold = json.threshold;
          CliqzHumanWeb.loadedQuorumConfig = true;
        }
      } catch (e) {
        _log('Error loading config.', e);
      }
    },
    getCountryCode: function() {
        let ctryCode = prefs.get('config_location', null);
        return CliqzHumanWeb.sanitizeCounrtyCode(ctryCode);
    },
    getTS: function() {
        try {
          let ts = prefs.get('config_ts', '--');
          return ts;
        } catch(ee) {
          return null;
        }
    },
    sanitizeCounrtyCode: function(ctryCode){
        let _countryCode = ctryCode;
        if(allowedCountryCodes.indexOf(_countryCode) === -1) {
            _countryCode = '--';
        }
        return _countryCode;
    },
    getQuorumCheckOtherUrls: function(msg){
        let urls = [];
        let urlPos = {};

        // Ony check canonical, when
        // It is not from qr.
        // If it is from qr then only type cl or othr.
        if ((msg.payload.x.canonical_url && !msg.payload.qr) ||
            (msg.payload.x.canonical_url && msg.payload.qr && (msg.payload.qr.t === 'cl' || msg.payload.qr.t === 'othr'))) {
            let canURL = msg.payload.x.canonical_url;
            let parse_url = parseURL(canURL);

            if ( parse_url && parse_url.path.length > 1 ||
                (parse_url.query_string && parse_url.path.length == 1 &&
                parse_url.query_string.length > 1) ) {

                urlPos[urls.length] = {t : 'canonical', url: canURL};
                urls.push(canURL);
            }
        }
        if (msg.payload.ref) {
            let refURL = msg.payload.ref;
            let parse_url = parseURL(refURL);

            if ( parse_url && parse_url.path.length > 1 ||
                (parse_url.query_string && parse_url.path.length == 1 &&
                parse_url.query_string.length > 1) ) {

                urlPos[urls.length] = {t : 'ref', url: refURL};
                urls.push(refURL);
            }
        }

        if (msg.payload.red) {
            msg.payload.red.forEach( (_redURL, idx) => {
              if (_redURL) {
                let redURL = _redURL;
                let parse_url = parseURL(redURL);

                if ( parse_url && parse_url.path.length > 1 ||
                    (parse_url.query_string && parse_url.path.length == 1 &&
                    parse_url.query_string.length > 1) ) {

                    urlPos[urls.length] = {t : 'red:' + idx, url: redURL};
                    urls.push(redURL);
                }
              }
            });
        }

        return {u: urls, up: urlPos};
    },
    quorumCheckOtherUrls: function(msg) {
        /*
        Before sending the action page, we need to ensure
        that the URLs carried in canonical_url field, ref-
        rre and redirect chain are safe.

        To do a final sanity, we are going to check for safety-quorum
        of these URLs, if returned false then we will mask the domain
        with (PROTECTED).

        Not all URLs require a sanity check , we only send the URLs
        which either have a path/ or query string.

        Example:
        https://example.com/ will not be sent.
        https://example.com/?q=something will be sent.

        We do not need to check canonical url for quorum if it comes out of a search engine.

        */
        let promise = new Promise( (resolve, reject) => {
            if (msg.action === 'page') {

                var urls;
                var urlPos;
                var allURLS = CliqzHumanWeb.getQuorumCheckOtherUrls(msg);
                urls = allURLS.u;
                urlPos = allURLS.up;
                _log('All urls in the message:' + JSON.stringify(urls));
                _log('All urls in the message:' + JSON.stringify(urlPos));

                Promise.all(urls.map(CliqzHumanWeb.sha1)).then( hashes => {
                    _log(JSON.stringify(hashes));
                    Promise.all(hashes.map(CliqzHumanWeb.getQuorumConsent)).then( results => {
                        _log(JSON.stringify(results));
                        results.forEach( (r, idx) => {
                            let original = urlPos[idx];
                            if(original.t === 'canonical') {
                                if(!r) msg.payload.x.canonical_url = CliqzHumanWeb.maskURLStrict(original.url);
                            }

                            if(original.t === 'ref') {
                                if(!r) msg.payload.ref = CliqzHumanWeb.maskURLStrict(original.url);
                            }

                            if(original.t.startsWith('red')) {
                                let redPos = original.t.split(':')[1];
                                if(!r) msg.payload.red[redPos] = CliqzHumanWeb.maskURLStrict(original.url);
                            }
                        });
                        _log('All urls in the message:' + JSON.stringify(msg));
                        resolve(msg);
                    }).catch( err => {
                      reject('Error in getQuorumConsent')
                    });
                }).catch( err => reject('Error in sha1'));
            } else {
                resolve(msg);
            }
        });

        return promise;
    },
    addURLtoDB: function (url, ref, paylobj) {
      /*
      1. Check if the given URL is a search URL,
      We do not want to save search URLs in the DB,
      for double-fetch.

      2. Then we want to check whether we have already marked the UpRL as private.
      If yes then we should return.

      3. Check if URL
        a. Not in the DB.
          i. Check if URL is fit to save
            x. paylobj['x'] == null
            y. CliqzHumanWeb.isSuspiciousURL(url)
            z. CliqzHumanWeb.httpCache401[url]

            if any of the above condition is met, it is set as private.
            else it is inserted in the DB

        b. Already in the DB.
          i. Update the stats, like engagement metrics.
      */

        var tt = new Date().getTime();
        if (CliqzHumanWeb.contentExtractor.isSearchEngineUrl(url)) {
            return;
        }

        //Check if url is in hashtable

      /*
      The key ft is obselete, should remove.
      */

        var ft = 1;
        var privateHash = false;

        CliqzHumanWeb.isAlreadyMarkedPrivate(url, function (_res) {
            if (_res) {
                if (_res['private'] == 1) {
                    privateHash = true;
                } else {
                    ft = 0;
                }
            } else {
                // we never seen it, let's add it
                paylobj['ft'] = true;
            }
        });

        // Need to add if canonical is seen before or not.
        // This is helpful, becuase now we replace the url with canonical incase of dropLongUrl(url) => true.
        // Hence, in the event log, lot of URL's look ft => true.

        CliqzHumanWeb.db.getURL(url, function(obj) {
          // If the url is already not in the DB or marked private, then we need save it.
          _log(">>>>> Add url to dbobj" + obj.length + privateHash);
          if (!privateHash && obj.length === 0) {
            // does not exist
            var setPrivate = false;

            var newObj = {};
            newObj.url = url;
            newObj.ref = ref;
            newObj.last_visit = tt;
            newObj.first_visit = tt;
            newObj.ft = ft;
            newObj.payload = paylobj || {}; // This needs to stringified before pushing to chrome storage.

            if (paylobj['x'] == null) {
                // page data structure is empty, so no need to double fetch, is private
                let reason = 'empty page data';
                setPrivate = true;
                _log("Setting private because empty page data");
            } else if (CliqzHumanWeb.isSuspiciousURL(url)) {
                // if the url looks private already add it already as checked and private
                let reason = 'susp. url';
                setPrivate = true;
                _log("Setting private because suspiciousURL");
            } else {
                if (CliqzHumanWeb.httpCache401[url]) {
                    let reason = '401';
                    setPrivate = true;
                    _log("Setting private because of 401");
                } else {
                    let reason = '';
                    setPrivate = false;
                }
            }
            _log(">>>>> lets save >>> " + JSON.stringify(newObj));

            // This needs to simplified, if it needs to set Private, why insert it in the first place.
            // Possibly because else the remove url would break in setAsPrivate.
            CliqzHumanWeb.db.saveURL(url, newObj, function() {

                if (CliqzHumanWeb.debug) {
                    _log("Insertion success add urltoDB");
                }

                if (setPrivate) CliqzHumanWeb.setAsPrivate(url);

            });
          } else if (obj.length === 1) {
            _log(">>>>> Add url to dbobj found record" + JSON.stringify(obj));
            let record = obj[0];
            // Looks like the URL is already there, we just need to update the stats.

            //Need to aggregate the engagement metrics.
            _log(record);
            let metricsBefore;
            if (typeof record.payload === 'string') {
              // (possibly only reachable on Bootstrapped extensions)
              metricsBefore = JSON.parse(record.payload).e;
            } else {
              metricsBefore = record.payload.e;
            }

            var metricsAfter = paylobj['e'];
            paylobj['e'] = CliqzHumanWeb.aggregateMetrics(metricsBefore, metricsAfter);

            var cloneObj = record;

            cloneObj.last_visit = tt;
            cloneObj.payload = paylobj || {};

            CliqzHumanWeb.db.updateURL(url, cloneObj, function() {
                _log("Record updated");
            });

            paylobj['e'] = { 'cp': 0, 'mm': 0, 'kp': 0, 'sc': 0, 'md': 0 };
          }
        });
    },
  setAsPrivate: function(url) {
    if(CliqzHumanWeb.bloomFilter){
      CliqzHumanWeb.bloomFilter.addSingle((md5(url)).substring(0,16));
    }

    CliqzHumanWeb.db.removeUnsafe(url, (result) => {
        _log(`Deleting ${url} : ${result}`);
    });

    if(CliqzHumanWeb.state['v'][url]){
      delete CliqzHumanWeb.state['v'][url];
    }
    CliqzHumanWeb.dumpBloomFilter();

    // incr stats for private page, to see public vs private pages marked.
    // mp : when double fetch failed or url was marked private.
    CliqzHumanWeb.incrActionStats("mp");
  },
  setAsPublic: function(url) {
    CliqzHumanWeb.db.removeUnsafe(url, (result) => {
        _log(`Deleting ${url} : ${result}`);
    });

    if(CliqzHumanWeb.state['v'][url]){
      delete CliqzHumanWeb.state['v'][url];
    }
  },
  listOfUnchecked: function(cap, sec_old, fixed_url, callback) {
    CliqzHumanWeb.db.getListOfUnchecked(cap, sec_old, fixed_url, (res, res2) => {
      callback(res);
    });
  },
  processUnchecks: function(listOfUncheckedUrls) {
    _log(">>> URLS UNPROCESSED >>> " + JSON.stringify(listOfUncheckedUrls))
    var url_pagedocPair = {};

    for(var i=0;i<listOfUncheckedUrls.length;i++) {
      var url = listOfUncheckedUrls[i][0];
      var page_doc = listOfUncheckedUrls[i][1];
      var page_struct_before = page_doc['x'];
      url_pagedocPair[url] = page_doc;

      CliqzHumanWeb.log('Going for double fetch (url:', url, ', page_doc:',
                        page_doc, ', page_struct_before:',
                        page_struct_before, ')');

      // only do doubleFetch for the same url 3 times in a row
      // (set up as this.humanWeb.MAX_NUMBER_DOUBLEFETCH_ATTEMPS).
      // If more attemps are tried then the url is marked as private.
      // Prevent infinite loop if the doubleFetch causes the browser
      // to crash (issue #2213)
      //
      CliqzHumanWeb.db.loadRecordTelemetry('last-double-fetch', function(data) {
        var obj = null;
        if (data==null) obj = {'url': url, 'count': 1};
        else {
          obj = JSON.parse(data);
          if (obj.url!=url) obj = {'url': url, 'count': 1};
          else {
            try {
              obj['count'] += 1;
            } catch(err) {
              obj['count'] = 1;
            }
          }
        }
        _log(">>>>> DOUBLE FETCH COUNT >>> " + JSON.stringify(obj));
        CliqzHumanWeb.db.saveRecordTelemetry('last-double-fetch', JSON.stringify(obj), (result) => {
            _log('last-double-fetch saved:', result);
        });

        if (obj.count > CliqzHumanWeb.MAX_NUMBER_DOUBLEFETCH_ATTEMPS) {
          CliqzHumanWeb.setAsPrivate(url);
        }
        else {
          CliqzHumanWeb.doubleFetch(url, url_pagedocPair[url]);
        }
      });
    }
  },
  detectAdClick(targetURL) {
    // The first URL observed after clicking the ad has the pattern,
    // google and aclk? in it.
    if (targetURL.includes('google') && targetURL.includes('aclk?')) {
      const clickedU = normalizeAclkUrl(targetURL);
      _log('ad-ctr: targetURL:', targetURL, 'normalized to', clickedU);

      if (CliqzHumanWeb.adDetails[clickedU]) {
        let query = CliqzHumanWeb.adDetails[clickedU].query;
        if (CliqzHumanWeb.isSuspiciousQuery(query)) {
          query = ' (PROTECTED) ';
        }
        const domain = cleanFinalUrl(
          CliqzHumanWeb.adDetails[clickedU].furl[0],
          CliqzHumanWeb.adDetails[clickedU].furl[1]);

        const payload = {
          action: 'ad-ctr',
          'anti-duplicates': Math.floor(random() * 10000000),
          type: 'humanweb',
          channel: CliqzHumanWeb.CHANNEL,
          payload: {
            query,
            domain,
            ctry: CliqzHumanWeb.getCountryCode(),
          }
        };

        _log('ad-ctr payload:', payload);
        CliqzHumanWeb.telemetry(payload);
      }
    }
  },
  purgeAdLookUp: function() {
    // We should clean the ads in lookup table, keep it from growing too huge.
    // Cleaning ads which are older than 15 minutes seems reasonable right now.

    let ts = Date.now();
    Object.keys(CliqzHumanWeb.adDetails).forEach( (item) => {
      let adTS = CliqzHumanWeb.adDetails[item]['ts'];
      let diff = (ts - adTS) / (1000 * 60);
      if (diff > 15) {
        delete CliqzHumanWeb.adDetails[item];
      }
    });

  },
  addStrictQueries(url, query) {
    // In some cases, we get query undefined.
    if (!query) {
        _log(">> Got an undefined query >>> " + url);
        return;
    }

    if (CliqzHumanWeb.isSuspiciousQuery(query)) {
      _log('Dropping suspicious query before double-fetch:', query);
      return;
    }

    const { isSearchEngineUrl, queryUrl } = CliqzHumanWeb.contentExtractor.checkAnonSearchURL(url, query);
    if (isSearchEngineUrl) {
        try {
            const qObj = {
              qurl: queryUrl,
              ts: Date.now(),
              tDiff: getRandomIntInclusive(1, 20),
            };
            CliqzHumanWeb.strictQueries.push(qObj);
            CliqzHumanWeb.saveStrictQueries();
        } catch (ee) {
          logger.error('Failed to add query:', ee);
        }
    }
  }
};
CliqzHumanWeb.contentExtractor = new ContentExtractor(CliqzHumanWeb);

export default CliqzHumanWeb;