token.py

import abc
import pickle
import re
import math
import pandas as pd
import jieba

from pygents.util import count_subelements, dictcount, calc_f1, counters_init, remove_all, dict_update, dict_compress_with_loss
from pygents.util import evaluate_compression, evaluate_anti_entropy 
from pygents.text import preprocess_text, grams_count_with_char_freedoms, grams_count_with_gram_freedoms
from pygents.text import url_lines, tokenize_with_sorted_lexicon, profile_freedoms, profile_probabilities, load_word_list_reverse


# Basic Tokenizer
class Tokenizer(abc.ABC):

    def __init__(self, debug=True):
        self.debug = debug

    def tokenize(self,text):
        return text.split()

assert str(Tokenizer().tokenize("ab c")) == "['ab', 'c']"

def tokenize_detaching_head(text,chars="'\"{[("):
    tokens = []
    for head in range(len(text)):
        found = chars.find(text[head])
        if found >= 0:
            tokens.append(chars[found])
        else:
            return tokens, text[head:]
    return tokens, None
assert str(tokenize_detaching_head("test")) == "([], 'test')"
assert str(tokenize_detaching_head("'\"")) == '(["\'", \'"\'], None)'
assert str(tokenize_detaching_head("\"'test")) == "(['\"', \"'\"], 'test')"


def tokenize_detaching_tail(text,chars="'\":,;.!?}])"):
    tokens = []
    length = len(text)
    for i in range(length):
        tail = length - i - 1
        found = chars.find(text[tail])
        if found >= 0:
            tokens.append(chars[found])
        else:
            tokens.reverse()
            return tokens, text[:tail + 1]
    tokens.reverse()
    return tokens, None
assert str(tokenize_detaching_tail("test")) == "([], 'test')"
assert str(tokenize_detaching_tail("test'")) == "([\"'\"], 'test')"
assert str(tokenize_detaching_tail("test.\"")) == "(['.', '\"'], 'test')"
assert str(tokenize_detaching_tail("test').\"")) == "([\"'\", ')', '.', '\"'], 'test')"

    
def tokenize_split_with_delimiters_and_quotes(text):
    tokens = []
    splits = text.split(' ') # TODO add ALL whitespaces like \n, \r \t, etc. 
    for split in splits:
        if len(tokens) > 0:
            tokens.append(' ')
        head, token = tokenize_detaching_head(split)
        tokens.extend(head)
        if token is not None and len(token) > 0: 
            tail, token = tokenize_detaching_tail(token)
            if token is not None and len(token) > 0:
                tokens.append(token)
            tokens.extend(tail)
    return tokens
assert str(tokenize_split_with_delimiters_and_quotes("man says hi")) == "['man', ' ', 'says', ' ', 'hi']"
assert str(tokenize_split_with_delimiters_and_quotes("man (tom) says 'hi there!' to me.")) == "['man', ' ', '(', 'tom', ')', ' ', 'says', ' ', \"'\", 'hi', ' ', 'there', '!', \"'\", ' ', 'to', ' ', 'me', '.']"


#TODO case sensitivity
class DelimiterTokenizer(Tokenizer):
    def __init__(self):
        Tokenizer.__init__(self,debug=False)
    def tokenize(self,text):
        return tokenize_split_with_delimiters_and_quotes(text)
assert str(DelimiterTokenizer().tokenize("man (tom) says 'hi there!' to me.")) == "['man', ' ', '(', 'tom', ')', ' ', 'says', ' ', \"'\", 'hi', ' ', 'there', '!', \"'\", ' ', 'to', ' ', 'me', '.']"
assert str(DelimiterTokenizer().tokenize("hi there, man!")) == "['hi', ' ', 'there', ',', ' ', 'man', '!']"
assert str(DelimiterTokenizer().tokenize("man says: hi there!, man.")) == "['man', ' ', 'says', ':', ' ', 'hi', ' ', 'there', '!', ',', ' ', 'man', '.']"


#https://github.com/fxsjy/jieba
class JiebaTokenizer(Tokenizer):
    def __init__(self):
        Tokenizer.__init__(self,debug=False)
    def tokenize(self,text):
        return [r[0] for r in jieba.tokenize(text)]


# Lexicon-based Tokenization

class LexiconTokenizer(Tokenizer): # very unefficient because of iterative search, use indexed LexiconIndexedTokenizer instead

    def __init__(self, name=None, lexicon=None, cased=False, url=None, debug=False):
        Tokenizer.__init__(self,debug=debug)
        self.name = name
        if not lexicon is None: 
            self.alex = list(lexicon) #copy
        else:
            lex_lines = url_lines(url)
            self.alex = [re.split('\t| |,|;|\n|\r',line)[0] for line in lex_lines] #load from url
            # TODO load from file
        self.compile()
        self.cased = cased

    def compile(self):
        self.alex.sort(key=len,reverse=True) #precompile

    def tokenize(self,text):
        return tokenize_with_sorted_lexicon(self.alex,text,cased=self.cased)

assert str(LexiconTokenizer(lexicon=['tuna','is','fish','cat','mammal']).tokenize("tunaisafish.catisamammal"))=="['tuna', 'is', 'a', 'fish', '.', 'cat', 'is', 'a', 'mammal']"    
assert str(LexiconTokenizer(lexicon=['tuna','is','fish','cat','mammal']).tokenize("Tunaisafish.Catisamammal"))=="['Tuna', 'is', 'a', 'fish', '.Cat', 'is', 'a', 'mammal']"
assert str(LexiconTokenizer(lexicon=['tuna','is','fish','cat','mammal'],cased=True).tokenize("Tunaisafish.Catisamammal"))=="['Tuna', 'is', 'a', 'fish', '.', 'Cat', 'is', 'a', 'mammal']"


def prefixed_match_from_list(lst,text):
    for item in lst:
        if text.startswith(item[0]):
            return item
    return None

def prefixed_match(prefixed_dict,text):
    letter = text[0]
    if not letter in prefixed_dict:
        return None
    return prefixed_match_from_list(prefixed_dict[letter],text)

def tokenize_with_prexied_sorted_lexicon(prefixed_dict,text,cased=False):
    original = text
    if cased: #if need to spend time on lowercasing non-lowercased text
        text = text.lower()
    tokens = []
    start = 0
    cur = 0
    length = len(text)
    sum_weight = 0
    while cur < length:
        subtext = text[cur:]
        word_weight = prefixed_match(prefixed_dict,subtext)
        #print(al)
        if not word_weight is None:
            word_len = len(word_weight[0])
            if start < cur:
                tokens.append(original[start:cur])
            tokens.append(original[cur:cur+word_len])
            sum_weight += word_weight[1]
            cur += word_len
            start = cur
        else:
            cur += 1
            #print('yo')
    if start < cur:
        tokens.append(original[start:cur])
        #print(original[start:cur])
    return tokens, sum_weight

def tabbed_line2tuple(line,log=True):
    lst = re.split('\t| |,|;|\n|\r',line)
    if len(lst) > 1:
        return (lst[0],float(lst[1]) if not log else math.log10(1+float(lst[1])))
    else:
        return (lst[0],1.0)

def weightedlist2dict(lst,lower=False): # (key,weight) -> sum weigts by keys, keys may be lowercased
    dic = {}
    for item in lst:
        dictcount(dic,item[0].lower() if lower else item[0],item[1])
    return dic

class LexiconIndexedTokenizer(Tokenizer):

    def __init__(self, name=None, lexicon=None, cased=False, debug=False, url=None, sortmode=0):
        Tokenizer.__init__(self,debug=debug)
        self.name = name
        if not lexicon is None: 
            self.freqlist = [(word,1.0) for word in lexicon] if type(lexicon[0]) is str else lexicon #copy
        elif not url is None:
            lex_lines = url_lines(url)
            self.freqlist = [tabbed_line2tuple(line) for line in lex_lines] #load from url
            # TODO load from file
        self.sortmode = sortmode
        self.cased = cased
        self.top_weight = None
        self.compile()

    def compile(self):
        self.dict = {}
        self.fulldict = weightedlist2dict(self.freqlist,lower=True) # save for debugging only!?
        for key in self.fulldict:
            value = self.fulldict[key]
            if self.top_weight is None or self.top_weight < value:
                self.top_weight = value
            if len(key) > 0:
                letter = key[0]
                if not letter in self.dict:
                    self.dict[letter] = set()
                self.dict[letter].add((key,value))
        #print(self.dict['f'])
        for key in self.dict:
            lst = list(self.dict[key])
            if self.sortmode == 0: # by len
                lst.sort(key=lambda s: len(s[0]), reverse=True)
            elif self.sortmode == 1: # by weight
                lst.sort(key=lambda s: s[1], reverse=True)
            else: # by len times logweight
                #TODO log separately for better performance
                lst.sort(key=lambda s: math.log10(s[1])*len(s[0]), reverse=True)
            self.dict[key] = lst
        self.freqlist = [(key,self.fulldict[key]) for key in self.fulldict] # save for extension
        #print(self.dict['f'])

    def extend(self,weightedlist):
        self.freqlist.extend(weightedlist)
        self.compile()

    def tokenize(self,text):
        tokens, weight = tokenize_with_prexied_sorted_lexicon(self.dict,text,cased=self.cased)
        return tokens

    def tokenize_weight(self,text):
        tokens, weight = tokenize_with_prexied_sorted_lexicon(self.dict,text,cased=self.cased)
        length = len(tokens)
        return tokens, 0 if length == 0 else weight / length 

    def count_params(self):
        return len(self.fulldict)

assert str(LexiconIndexedTokenizer(lexicon=['tuna','is','fish','cat','mammal']).tokenize("tunaisafish.catisamammal"))=="['tuna', 'is', 'a', 'fish', '.', 'cat', 'is', 'a', 'mammal']"    
assert str(LexiconIndexedTokenizer(lexicon=['tuna','is','fish','cat','mammal']).tokenize("Tunaisafish.Catisamammal"))=="['Tuna', 'is', 'a', 'fish', '.Cat', 'is', 'a', 'mammal']"
assert str(LexiconIndexedTokenizer(lexicon=['tuna','is','fish','cat','mammal'],cased=True).tokenize("Tunaisafish.Catisamammal"))=="['Tuna', 'is', 'a', 'fish', '.', 'Cat', 'is', 'a', 'mammal']"


# Extended Tokenizer based on "freedoms"
class FreedomTokenizer(Tokenizer):

    def __init__(self, name=None, max_n=7, mode='grams', debug=False):
        Tokenizer.__init__(self,debug=debug)
        self.max_n = max_n
        self.model = pickle.load(open(name, 'rb')) if name is not None else [{},{},{}]
        self.mode = mode

    def train(self,texts,max_n=None):
        if max_n is None:
            max_n = self.max_n
        model = counters_init(max_n) 
        
        for text in texts:
            cnt = texts[text] if type(texts) == dict else 1
            text = preprocess_text(text)
            if self.mode == 'grams':
                for n in range(max_n):
                    grams_count_with_gram_freedoms(model,text,n+1,cnt=cnt,debug=self.debug)
            else: # 'chars' - legacy, but works better on Brown corpus!
                chars = list(text)
                for n in range(max_n):
                    grams_count_with_char_freedoms(model[0],model[1],model[2],chars,n+1,cnt=cnt,debug=self.debug)
        #merge n-specific models into joint ones
        for i in range(3):
            for d in model[i]:
                #self.model[i].update(d)
                dict_update(self.model[i],d)
        return self

    def train_folder(self,folder_path,model_path=None,name=None,debug = False):
        #TODO recursion, if specified
        onlyfiles = [f for f in listdir(folder_path) if isfile(join(folder_path, f))]
        cnt = 0
        for file in onlyfiles:
            with open(join(path, file),errors='ignore') as f:
                lines = f.readlines()
                cnt += 1
                if debug and (cnt % 100) == 0:
                    print(cnt,file)
                self.train(lines)
        if model_path is not None:
            self.store(model_path)
        if debug:
            print(self.count_params())
        
    def tokenize(self,text):
        #TODO pass!!!???
        return text.split()

    def count_params(self):
        return count_subelements(self.model)
    
    def store(self,path):
        pickle.dump(self.model, open(path, 'wb'), pickle.HIGHEST_PROTOCOL)

    
_test_tokenizer = FreedomTokenizer(max_n=2,mode='chars',debug=False).train(["pig"])
assert _test_tokenizer.count_params() == 11
assert str(_test_tokenizer.model) == "[{'p': 1, 'i': 1, 'g': 1, 'pi': 1, 'ig': 1}, {'p': {'i': 1}, 'i': {'g': 1}, 'pi': {'g': 1}}, {'i': {'p': 1}, 'g': {'i': 1}, 'ig': {'p': 1}}]"
_test_tokenizer = FreedomTokenizer(max_n=2,mode='chars').train(["ding","dong"])
#print(_test_tokenizer.count_params())
assert _test_tokenizer.count_params() == 28
#print(str(_test_tokenizer.model[0]))
#print(str(_test_tokenizer.model[1]))
#print(str(_test_tokenizer.model[2]))
#print(str(_test_tokenizer.model))
assert str(_test_tokenizer.model) == "[{'d': 2, 'i': 1, 'n': 2, 'g': 2, 'o': 1, 'di': 1, 'in': 1, 'ng': 2, 'do': 1, 'on': 1}, {'d': {'i': 1, 'o': 1}, 'i': {'n': 1}, 'n': {'g': 2}, 'o': {'n': 1}, 'di': {'n': 1}, 'in': {'g': 1}, 'do': {'n': 1}, 'on': {'g': 1}}, {'i': {'d': 1}, 'n': {'i': 1, 'o': 1}, 'g': {'n': 2}, 'o': {'d': 1}, 'in': {'d': 1}, 'ng': {'i': 1, 'o': 1}, 'on': {'d': 1}}]"
_test_tokenizer = FreedomTokenizer(max_n=2,mode='chars').train({"ding":3,"dong":3})
assert _test_tokenizer.count_params() == 28
assert str(_test_tokenizer.model) == "[{'d': 6, 'i': 3, 'n': 6, 'g': 6, 'o': 3, 'di': 3, 'in': 3, 'ng': 6, 'do': 3, 'on': 3}, {'d': {'i': 3, 'o': 3}, 'i': {'n': 3}, 'n': {'g': 6}, 'o': {'n': 3}, 'di': {'n': 3}, 'in': {'g': 3}, 'do': {'n': 3}, 'on': {'g': 3}}, {'i': {'d': 3}, 'n': {'i': 3, 'o': 3}, 'g': {'n': 6}, 'o': {'d': 3}, 'in': {'d': 3}, 'ng': {'i': 3, 'o': 3}, 'on': {'d': 3}}]"


class FreedomBasedTokenizer(FreedomTokenizer):

    def __init__(self, base, back, forw, nlist=[1], threshold=0.5, debug=False):
        FreedomTokenizer.__init__(self,debug=debug)
        self.model = base.model
        self.mode = base.mode
        self.back = back
        self.forw = forw
        self.nlist = nlist
        self.threshold = threshold
        
    def set_options(self,**kwargs):
        if 'threshold' in kwargs:
            self.threshold = kwargs['threshold']
        if 'nlist' in kwargs:
            self.nlist = kwargs['nlist']

    def tokenize(self,text):
        return tokenize_with_opposite_metrics(self.model,text,self.back,self.forw,self.nlist,self.threshold)


class FreedomBasedTokenizerWithWordBoundaries(FreedomBasedTokenizer):

    def __init__(self, base, back, forw, nlist=[1], threshold=0.5, word_boundary='_', debug=False):
        FreedomBasedTokenizer.__init__(self, base, back, forw, nlist, threshold, debug)
        self.word_boundary = word_boundary

    def tokenize(self,text):
        text = self.word_boundary + text + self.word_boundary
        tokens = tokenize_with_opposite_metrics(self.model,text,self.back,self.forw,self.nlist,self.threshold)
        last = len(tokens)-1
        tokens[0] = tokens[0][1:]
        tokens[last] = tokens[last][:-1]
        if len(tokens[0]) < 1:
            tokens = tokens[1:]
        last = len(tokens)-1
        if len(tokens[last]) < 1:
            tokens = tokens[:-1]
        return tokens


class PrefixSuffixMorphoTokenizer(Tokenizer):
    def __init__(self,prefixes,suffixes,debug=False):
        Tokenizer.__init__(self,debug=debug)
        self.prefixes = [load_word_list_reverse(path) for path in prefixes]
        self.suffixes = [load_word_list_reverse(path) for path in suffixes]
        
    #TODO iteratively detach suffixes like "interesting" -> inter-est-ing-ly
    #TODO when iterating, select the longest prefix or suffix first
    def tokenize(self,text):
        all_prefixes = []
        all_suffixes = []
        keep_trying = True
        while keep_trying:
            keep_trying = False
            for prefixes in self.prefixes:
                for prefix in prefixes:
                    if len(text) > len(prefix) and text.startswith(prefix):
                        if self.debug:
                            print(prefix)
                        all_prefixes.append(prefix)
                        text = text[len(prefix):]
                        keep_trying = True
                        break
            for suffixes in self.suffixes:
                for suffix in suffixes:
                    if len(text) > len(suffix) and text.endswith(suffix):
                        if self.debug:
                            print(suffix)
                        all_suffixes.append(suffix)
                        text = text[:len(text)-len(suffix)]
                        keep_trying = True
                        break
        all_suffixes.reverse()
        return all_prefixes + [text] + all_suffixes

    
class PrefixSuffixMorphoTokenizerCached(PrefixSuffixMorphoTokenizer):
    def __init__(self,prefixes,suffixes,debug=False):
        PrefixSuffixMorphoTokenizer.__init__(self,prefixes,suffixes,debug=debug)
        self.cache = {}

    def tokenize(self,text):
        if text in self.cache:
            if self.debug:
                print(text, '-cache->', self.cache[text])
            return self.cache[text]
        pieces = PrefixSuffixMorphoTokenizer.tokenize(self,text)
        self.cache[text] = pieces
        if self.debug:
            print(text, '-tokenize->', pieces)
        return pieces


def model_compress_with_loss(model,threshold=0.01):
    dict_compress_with_loss(model[0],threshold)
    dict_compress_with_loss(model[1],threshold)
    dict_compress_with_loss(model[2],threshold)


def profile_freedoms_ex_df(model,text,n,denominate=False,debug=False):
    df = pd.DataFrame(profile_freedoms(model,text,n,denominate=denominate,debug=debug),columns=['pos','gram','f+','f-'])
    df['dvf+'] = (df['f+'] - df['f+'].mean()).clip(lower=0)
    df['dvf-'] = (df['f-'] - df['f-'].mean()).clip(lower=0)
    df['dvf+|dvf-'] = df['dvf+'] + df['dvf-'].shift(-1)
    df['dvf+&dvf-'] = df['dvf+'] * df['dvf-'].shift(-1)
    if True: # legacy notebook hack
        df['ddf+'] = df['dvf+']
        df['ddf-'] = df['dvf-']
        df['ddf+|ddf-'] = df['dvf+|dvf-']
        df['ddf+&ddf-'] = df['dvf+&dvf-']
    df['df+'] = df['f+'].diff() 
    df['df-'] = -df['f-'].diff().shift(-1)
    df['df+|df-'] = df['df+'] + df['df-']
    df['df+&df-'] = df['df+'] * df['df-']
    # We assigned a “peak” value to each character transition, 
    # computed by adding the value of the preceding increase in freedom to the following decrease in freedom. 
    # We characterized token boundaries based on the sum of their forward- and backward-reading peak values.
    df['peak+'] = df['df+'] - df['df+'].shift(-1)
    df['peak-'] = df['df-'] - df['df-'].shift(1)
    df['f+|f-'] = df['f+'] + df['f-'].shift(-1)
    df['f+&f-'] = df['f+'] * df['f-'].shift(-1)
    return df


def profile_freedoms_avg_df(model,text,metrics,nlist,denominate=False,debug=False):
    res_df = None
    for n in nlist:
        df = profile_freedoms_ex_df(model,text,n,denominate=denominate)
        if res_df is None:
            res_df = df[['pos','gram']+metrics].copy()
        else:
            for m in metrics:
                res_df[m] = res_df[m] + df[m]
    for m in metrics:
        res_df[m] = res_df[m]/res_df[m].max()
    return res_df


def profile_probabilities_ex_df(model,text,n,debug=False):
    df = pd.DataFrame(profile_probabilities(model[0],text,n,debug=debug),columns=['pos','gram','p+','p-'])
    if n == 1:
        pmax = max(df['p+'].max(),df['p-'].max())
        df['p+'] = df['p+']/pmax
        df['p-'] = df['p-']/pmax
    df['dvp+'] = (df['p+'] - df['p+'].mean()).clip(lower=0)
    df['dvp-'] = (df['p-'] - df['p-'].mean()).clip(lower=0)
    df['dvp+|dvp-'] = df['dvp+'] + df['dvp-'].shift(-1)
    df['dvp+&dvp-'] = df['dvp+'] * df['dvp-'].shift(-1)
    if True: # legacy notebook hack
        df['ddp+'] = df['dvp+']
        df['ddp-'] = df['dvp-']
        df['ddp+|ddp-'] = df['dvp+|dvp-']
        df['ddp+&ddp-'] = df['dvp+&dvp-']
    df['dp+'] = df['p+'].diff() 
    df['dp-'] = -df['p-'].diff().shift(-1)
    df['dp+|dp-'] = df['dp+'] + df['dp-']
    df['dp+&dp-'] = df['dp+'] * df['dp-']
    #TODO!?
    # We assigned a “peak” value to each character transition, 
    # computed by adding the value of the preceding increase in freedom to the following decrease in freedom. 
    # We characterized token boundaries based on the sum of their forward- and backward-reading peak values.
    #df['peak+'] = df['df+'] - df['df+'].shift(-1)
    #df['peak-'] = df['df-'] - df['df-'].shift(1)
    df['p+|p-'] = df['p+'] + df['p-'].shift(-1)
    df['p+&p-'] = df['p+'] * df['p-'].shift(-1)
    return df


def profile_probabilities_avg_df(model,text,metrics,nlist,debug=False):
    res_df = None
    for n in nlist:
        df = profile_probabilities_ex_df(model,text,n)
        if res_df is None:
            res_df = df[['pos','gram']+metrics].copy()
        else:
            for m in metrics:
                res_df[m] = res_df[m] + df[m]
    for m in metrics:
        res_df[m] = res_df[m]/res_df[m].max()
    return res_df


def tokenize_with_opposite_metrics(model,text,back,forw,nlist,threshold=0.5,profiler=profile_freedoms_avg_df,debug=False):
    tokens = []
    token = ''
    df = profiler(model,text,[forw,back],nlist)
    length = len(df)
    for i in range(length):
        iplus1 = i+1
        brk_back = True if iplus1 < length and df.loc[iplus1][back] >= threshold else False
        brk_forw = True if df.loc[i][forw] >= threshold else False
        #token += df.loc[i]['gram']
        token += text[i] # to ensure raw data capitalization
        if debug:
            print("{}\t{}\t{}\t{}\t{}\t{}".format(df.loc[i]['gram'],'-' if brk_back else '', '+' if brk_forw else '',round(df.loc[i][back],2),round(df.loc[i][forw],2),token))
        if len(token) > 0 and (brk_back or brk_forw):
            tokens.append(token)
            token = ''
    if len(token) > 0:
            tokens.append(token)
    return tokens


def tokenize_with_forward_metric(model,text,forw,nlist,threshold=0.5,profiler=profile_freedoms_avg_df,debug=False):
    tokens = []
    token = ''
    df = profiler(model,text,[forw],nlist)
    length = len(df)
    for i in range(length):
        brk_forw = True if df.loc[i][forw] >= threshold else False
        #token += df.loc[i]['gram']
        token += text[i] # to ensure raw data capitalization
        if debug:
            print("{}\t{}\t{}\t{}\t{}".format(df.loc[i]['gram'],'+' if brk_forw else '',round(df.loc[i][back],2),round(df.loc[i][forw],2),token))
        if len(token) > 0 and brk_forw:
            tokens.append(token)
            token = ''
    if len(token) > 0:
            tokens.append(token)
    return tokens


def evaluate_tokenizer_f1(texts,real_tokenizer,test_tokenizer,nospaces=False,expected_collector=None,actual_collector=None,debug=False):
    avg_f1 = 0
    count = 0
    for text in texts:
        expected = real_tokenizer.tokenize(text)
        if nospaces:
            remove_all(expected,' ')
        tokens = test_tokenizer.tokenize(text if not nospaces else text.replace(' ','')) # nospaces=True complicates the problem removing spaces
        if not expected_collector is None:
            dictcount(expected_collector,expected)
        if not actual_collector is None:
            dictcount(actual_collector,tokens)
        f1 = calc_f1(expected,tokens)
        if debug:
            print(text)
            print(expected)
            print(tokens)
            print(round(f1,2))
        avg_f1 += f1
        count += 1
    return round(avg_f1/count,2)


def evaluate_tokenizer(model,texts,forw,back,nlist,threshold,profiler=profile_freedoms_avg_df,spaces=False,output=False,debug=False):
    if output:
        print("N\tthres.\tF1")
    f1_avg = 0
    for text in texts:
        tokens = tokenize_with_opposite_metrics(model,text,forw,back,nlist,threshold=threshold,profiler=profiler) if back is not None else tokenize_with_forward_metric(model,text,forw,nlist,threshold=threshold,profiler=profiler)
        tokens_ref = tokenize_split_with_delimiters_and_quotes(text)
        if not spaces:
            remove_all(tokens,' ')
            remove_all(tokens_ref,' ')
        f1 = calc_f1(tokens_ref,tokens) 
        f1_avg += f1
        if debug:
            print(f1)
            print(text)
            print(calc_diff(tokens,tokens_ref))
            print(str(tokens_ref))
            print(str(tokens))
            print()
    f1 = round(f1_avg/len(texts),2)
    if output:
        print("{}\t{}\t{}".format(nlist,threshold,f1))
    return nlist,threshold,f1


def evaluate_tokenizer_f1_compratio_entropy(texts,real_tokenizer,test_tokenizer,nospaces=False,expected_collector=None,actual_collector=None,text_counts=None,debug=False):
    avg_f1 = 0
    count = 0
    tokenized_texts = []
    for i in range(len(texts)):
        text = texts[i]
        weight = 1 if text_counts is None else text_counts[i]
        expected = real_tokenizer.tokenize(text)
        if nospaces:
            remove_all(expected,' ')
        tokens = test_tokenizer.tokenize(text if not nospaces else text.replace(' ','')) # nospaces=True complicates the problem removing spaces
        tokenized_texts.append(tokens)
        if not expected_collector is None:
            dictcount(expected_collector,expected)
        if not actual_collector is None:
            dictcount(actual_collector,tokens)
        f1 = calc_f1(expected,tokens)
        if debug:
            print(text)
            print(expected)
            print(tokens)
            print(round(f1,2))
        avg_f1 += f1 * weight
        count += weight
    return round(avg_f1/count,2), round(evaluate_compression(texts,tokenized_texts,counts=text_counts),2), round(evaluate_anti_entropy(tokenized_texts,counts=text_counts),2)