Code cleanup

dimazest · Dec 21, 2013 · dfd43eb · dfd43eb
1 parent 0e361f7
commit dfd43eb
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 7 deletions.
diff --git a/fowler/corpora/google_ngrams/main.py b/fowler/corpora/google_ngrams/main.py
@@ -70,7 +70,6 @@ def cooccurrence(
     context=('c', 'context.csv.gz', 'The file with context words.'),
     targets=('t', 'targets.csv.gz', 'The file with target words.'),
     input_dir=('i', local('./downloads/google_ngrams/5_cooccurrence'), 'The path to the directory with the Google unigram files.'),
-    with_pos=('', False, 'Include ngrams that are POS tagged.'),
     output=('o', 'matrix.h5', 'The output matrix file.'),
 ):
     """Build the cooccurrence matrix."""
@@ -110,11 +109,6 @@ def cooccurrence(
             quoting=csv.QUOTE_NONE,
         )
 
-        frame['ngram'].fillna('U+F8F0:<INVALIDCHARACTER>', inplace=True)
-
-        if not with_pos:
-            frame = frame[np.invert(frame['ngram'].str.contains('_'))]
-
         piece = (
             frame
             .merge(targets, left_on='ngram', right_index=True, sort=False)

diff --git a/test/google_ngrams/test_main.py b/test/google_ngrams/test_main.py
@@ -42,7 +42,6 @@ def targets_path(tmpdir):
             'aa\n'  # 5
             'ab\n'  # 6
             'ac\n'  # 7
-            'AA_NOUN\n'  # 8
         )
 
     return path