CJK_models

drafttopic/about.py

@@ -1,5 +1,5 @@
 __name__ = "drafttopic"
-__version__ = "0.3.0"
+__version__ = "1.4.0"
 __author__ = "Aaron Halfaker, Sumit Asthana"
 __author_email__ = "ahalfaker@wikimedia.org, asthana.sumit23@gmail.com"
 __description__ = "A library for automatic detection of topics of new " +\

drafttopic/feature_lists/jawiki.py

@@ -0,0 +1,26 @@
+from revscoring.datasources.meta import vectorizers, mappers
+from revscoring.features import wikitext
+from revscoring.features.meta import aggregators
+
+
+jawiki_kvs = vectorizers.word2vec.load_gensim_kv(
+    filename="jawiki-20201201-learned_vectors.50_cell.10k.kv", mmap='r')
+
+
+def vectorize_words(words):
+    return vectorizers.word2vec.vectorize_words(jawiki_kvs, words)
+
+
+revision_text_vectors = vectorizers.word2vec(
+    wikitext.revision.datasources.cjk.cjks,
+    vectorize_words,
+    name="revision.text.ja_vectors")
+
+w2v = aggregators.mean(
+    revision_text_vectors,
+    vector=True,
+    name="revision.text.ja_vectors_mean"
+)
+
+drafttopic = [w2v]
+articletopic = drafttopic

drafttopic/feature_lists/kowiki.py

@@ -4,15 +4,15 @@
 
 
 kowiki_kvs = vectorizers.word2vec.load_gensim_kv(
-    filename="kowiki-20200501-learned_vectors.50_cell.10k.kv", mmap='r')
+    filename="kowiki-20201201-learned_vectors.50_cell.10k.kv", mmap='r')
 
 
 def vectorize_words(words):
     return vectorizers.word2vec.vectorize_words(kowiki_kvs, words)
 
 
 revision_text_vectors = vectorizers.word2vec(
-    mappers.lower_case(wikitext.revision.datasources.words),
+    wikitext.revision.datasources.cjk.cjks,
     vectorize_words,
     name="revision.text.ko_vectors")
 

drafttopic/feature_lists/zhwiki.py

@@ -0,0 +1,26 @@
+from revscoring.datasources.meta import vectorizers, mappers
+from revscoring.features import wikitext
+from revscoring.features.meta import aggregators
+
+
+zhwiki_kvs = vectorizers.word2vec.load_gensim_kv(
+    filename="zhwiki-20201201-learned_vectors.50_cell.10k.kv", mmap='r')
+
+
+def vectorize_words(words):
+    return vectorizers.word2vec.vectorize_words(zhwiki_kvs, words)
+
+
+revision_text_vectors = vectorizers.word2vec(
+    wikitext.revision.datasources.cjk.cjks,
+    vectorize_words,
+    name="revision.text.zh_vectors")
+
+w2v = aggregators.mean(
+    revision_text_vectors,
+    vector=True,
+    name="revision.text.zh_vectors_mean"
+)
+
+drafttopic = [w2v]
+articletopic = drafttopic

drafttopic/utilities/extract_from_text.py

@@ -12,6 +12,7 @@
                           [--input=<path>]
                           [--output=<path>]
                           [--extractors=<num>]
+                          [--tok_strategy=<str>]
                           [--verbose]
                           [--debug]
 
@@ -25,6 +26,7 @@
                                 [default: <stdout>]
         --extractors=<num>      The number of parallel extractors to
                                 start [default: <cpu count>]
+        --tok_strategy=<str>    tokenization strategy
         --verbose               Print dots and stuff to stderr
         --debug                 Print debug logs
 """
@@ -39,6 +41,9 @@
 from revscoring.dependencies import solve
 from revscoring.utilities.util import dump_observation, read_observations
 
+from mwtext.content_transformers import Wikitext2Words
+forbidden_link_prefixes = [
+    'category', 'image', 'file']
 
 def main(argv=None):
     args = docopt.docopt(__doc__, argv=argv)
@@ -72,14 +77,17 @@ def main(argv=None):
         extractors = int(args['--extractors'])
 
     verbose = args['--verbose']
+    tok_strategy = str(args['--tok_strategy']) if args['--tok_strategy'] is not None else None
+    wtpp = Wikitext2Words(forbidden_link_prefixes, tok_strategy=tok_strategy)
+    sys.stderr.write("tokenization strategy is: " + tok_strategy)
+    sys.stderr.write("\nnumber of processes: " + str(extractors) + "\n")
+    run(observations, dependents, output, extractors, wtpp, verbose)
 
-    run(observations, dependents, output, extractors, verbose)
 
-
-def run(labelings, dependents, output, extractors, verbose=False):
+def run(labelings, dependents, output, extractors, wtpp, verbose=False):
     extractor_pool = Pool(processes=extractors)
 
-    extractor = LabelingDependentExtractor(dependents)
+    extractor = LabelingDependentExtractor(dependents, wtpp)
 
     for observation in extractor_pool.imap(
             extractor.extract_and_cache, labelings):
@@ -100,15 +108,16 @@ def run(labelings, dependents, output, extractors, verbose=False):
 
 class LabelingDependentExtractor:
 
-    def __init__(self, dependents):
+    def __init__(self, dependents, wtpp):
         self.dependents = dependents
+        self.wtpp = wtpp
 
     def extract_and_cache(self, observation):
         if observation['text'] is None:
             return None
 
         values = extract_from_text(
-            self.dependents, observation['text'],
+            self.dependents, self.wtpp, observation['text'],
             cache=observation.get('cache'))
         dependent_cache = {str(d): val
                            for d, val in zip(self.dependents, values)}
@@ -121,7 +130,7 @@ def extract_and_cache(self, observation):
         return observation
 
 
-def extract_from_text(dependents, text, cache=None, context=None):
+def extract_from_text(dependents, wtpp, text, cache=None, context=None):
     """
     Extracts a set of values from a text an returns a cache containing just
     those values.
@@ -135,6 +144,6 @@ def extract_from_text(dependents, text, cache=None, context=None):
         A list of extracted feature values
     """
     cache = cache if cache is not None else {}
-    cache[revision_oriented.revision.text] = text
+    cache[revision_oriented.revision.text] = ' '.join(wtpp.transform(text))
 
     return list(solve(dependents, cache=cache, context=context))

drafttopic/utilities/fetch_draft_text.py

@@ -96,7 +96,6 @@ def _fetch_text(obs):
                 rev_doc = page_doc['revisions'][0]
                 text = rev_doc['slots']['main']['content']
                 if is_article(text):
-
                     obs['text'] = text
                     obs['title'] = page_doc['title']
                     obs['rev_id'] = rev_doc['revid']