Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
28 lines (23 sloc) 937 Bytes
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import json
def get_char_vocab(input_filenames, output_filename):
vocab = set()
for filename in input_filenames:
with open(filename) as f:
for line in f.readlines():
for sentence in json.loads(line)["sentences"]:
for word in sentence:
vocab.update(word)
vocab = sorted(list(vocab))
with open(output_filename, "w", encoding='utf-8') as f:
for char in vocab:
f.write("{}\n".format(char))
print("Wrote {} characters to {}".format(len(vocab), output_filename))
def get_char_vocab_language(language):
get_char_vocab(["data/{}.{}.jsonlines".format(partition, language) for partition in ("train", "dev", "test")], "char_vocab.{}.txt".format(language))
get_char_vocab_language("english")
#get_char_vocab_language("chinese")
#get_char_vocab_language("arabic")
You can’t perform that action at this time.