Checks that order of ngram >= length of sentence to avoid Python3.5 w…

…arning (#1532)

* Resolve warning when order of ngram > len(sent)

* Resolve warning when order of ngram > len(sent)

* Remove wrongly committed machete test code

* remove spurious whitespace

* use double newline between imports and functions

nltk/test/unit/translate/test_bleu.py

@@ -181,7 +181,7 @@ def test_corpus_bleu(self):
                 # Note: split() automatically strip().
                 hypothesis = list(map(lambda x: x.split(), hyp_fin))
                 # Note that the corpus_bleu input is list of list of references.
-                references = list(map(lambda x: [x.split()],ref_fin))
+                references = list(map(lambda x: [x.split()], ref_fin))
                 # Without smoothing.
                 for i, mteval_bleu in zip(range(1,10), mteval_bleu_scores):
                     nltk_bleu = corpus_bleu(references, hypothesis, weights=(1.0/i,)*i)

nltk/translate/bleu_score.py

@@ -267,13 +267,12 @@ def modified_precision(references, hypothesis, n):
     """
     # Extracts all ngrams in hypothesis
     # Set an empty Counter if hypothesis is empty.
-    counts = Counter(ngrams(hypothesis, n)) if hypothesis else Counter()
-
+    counts = Counter(ngrams(hypothesis, n)) if len(hypothesis) >= n else Counter()
     # Extract a union of references' counts.
     ## max_counts = reduce(or_, [Counter(ngrams(ref, n)) for ref in references])
     max_counts = {}
     for reference in references:
-        reference_counts = Counter(ngrams(reference, n)) if reference else Counter()
+        reference_counts = Counter(ngrams(reference, n)) if len(reference) >= n else Counter()
         for ngram in counts:
             max_counts[ngram] = max(max_counts.get(ngram, 0),
                                     reference_counts[ngram])