added shebang and doc

LiberAI · Jun 9, 2018 · e7887c4 · e7887c4
1 parent c96d5f2
commit e7887c4
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/build_vocab.py b/build_vocab.py
@@ -1,3 +1,7 @@
+#!/usr/bin/env python
+"""
+Usage: python build_vocab.py data.en > vocab.en
+"""
 import numpy as np
 from tensorflow.contrib import learn
 import sys
@@ -44,6 +48,7 @@
     ## word with id i goes at index i of the list.
     vocabulary = set(list(zip(*sorted_vocab))[0])
 
+    # split also by apostrophe
     to_remove = set()
     to_add = set()
     for t0 in vocabulary:
@@ -56,8 +61,7 @@
     for t0 in to_add:
         vocabulary.add(t0)
 
-# print(vocabulary)
-# print(x)
+# print terms
 for v in vocabulary:
     if v != "":
         print v