Allowing punctuation

Words with punctuation weren't being read into the dictionary or matched
correctly against dictionary words. No more.

CMUPronouncingDictionary.hs

@@ -11,7 +11,7 @@ module CMUPronouncingDictionary (
   syllables, phonemes
 ) where
 
-import Data.Char(toLower,toUpper)
+import Data.Char(toLower,toUpper,isAlpha)
 import Data.List(isPrefixOf, isSuffixOf)
 import Data.Map(Map)
 import qualified Data.Map as Map
@@ -75,7 +75,7 @@ loadWords strDict strWords = Map.fromList wordList where
   -- | Returns "" if the word is not found
   findWordLine w = if null ws then "" else head ws where
     ws = findWordLines w
-  findWordLines w = filter (isPrefixOf ((map toUpper w)++" ")) dictLines
+  findWordLines w = filter (isPrefixOf ((filter isAlpha (map toUpper w))++" ")) dictLines
   wordList = map wordTuple relevantDictionaryLines
   wordTuple line = (strWord word, word) where
     word = dictLineToWord line

Main.hs

@@ -5,34 +5,32 @@ module Main (main) where
 import System.IO
 import PoemClassifier(classify)
 import Data.List
-import Data.List.Split(splitOneOf)
-import Data.Char(toUpper)
+import Data.Char(toUpper, isAlpha)
 
 main :: IO ()
 main = do
   -- Read the poem in from STDIN
   inh <- openFile "dictionary/cmudict.0.7a" ReadMode
   putStrLn "Opened file"
   poem <- getContents
-  loadDictionary poem inh (wordList poem) "" 
+  loadDictionary poem inh (map (filter isAlpha) (wordList poem)) "" 
 
 wordList :: String -> [String]
-wordList str = map upper $ sort $ filter (/= "") $ nub $ splitOneOf " \n" str
-
-upper :: String -> String
-upper = map toUpper
+wordList str = map (map toUpper) $ sort $ nub $ concatMap words (lines str) 
 
 -- | Given a dictionary filename, and list of words, excerpts of the dictionary
 -- will be returned corresponding to the list of words
 loadDictionary :: String -> Handle -> [String] -> String -> IO ()
 loadDictionary poem inh wrds excerpt = do
   ineof <- hIsEOF inh
-  if ineof || null wrds then runClassifier poem excerpt
-    else do
+  if ineof || null wrds 
+  then runClassifier poem excerpt
+  else do
       line <- hGetLine inh
-      if null (prefixes line) then
-        loadDictionary poem inh wrds excerpt 
-      else loadDictionary poem inh (delete (prefix line) wrds) (excerpt ++ ('\n':line)) where
+      if null (prefixes line) 
+      then loadDictionary poem inh wrds excerpt 
+      else loadDictionary poem inh (delete (prefix line) wrds) 
+                                   (excerpt ++ ('\n':line)) where
         prefixes l = filter (\word -> isPrefixOf word l) wrds
         prefix l = head $ prefixes l
 

PoemAnalyzer.hs

@@ -3,7 +3,7 @@ module PoemAnalyzer(getWords, justWords) where
 import CMUPronouncingDictionary
 import Test.HUnit
 import Data.Map (Map)
-import Data.Char (toLower)
+import Data.Char (toLower, isAlpha)
 import qualified Data.Map as Map
 import Data.Maybe
 
@@ -39,7 +39,8 @@ testJustWords = "Test justWords" ~: TestList [
 
 -- | Returns statistics about a word given the word
 getWord :: Dictionary -> String -> Maybe Word
-getWord dict str = Map.lookup (map toLower str) dict
+getWord dict str = Map.lookup str' dict where
+  str' = filter isAlpha $ map toLower str
 
 testGetWord :: Test
 testGetWord = "Test getWord" ~: TestList [