Fixed bug allowing empty tokens through.

app/models/freq.rb

@@ -18,7 +18,9 @@ def persisted?
   end
 
   def tokens
-    @input_text.split.map { |token| normalize(token) }
+    @input_text.split
+               .map { |token| normalize(token) }
+               .select { |token| ! token.empty? }
   end
 
   def normalize(token)

spec/models/freq_model_spec.rb

@@ -22,6 +22,37 @@
         (/[A-Z]/ =~ token).should == nil
       end
     end
+
+    it 'should throw out empty tokens.' do
+      fm = Freq.new(:input_text => <<EOS
+Four score and seven years ago our fathers brought forth on this continent, a
+new nation, conceived in Liberty, and dedicated to the proposition that all men
+are created equal.
+
+Now we are engaged in a great civil war, testing whether that nation, or any
+nation so conceived and so dedicated, can long endure. We are met on a great
+battle-field of that war. We have come to dedicate a portion of that field, as
+a final resting place for those who here gave their lives that that nation
+might live. It is altogether fitting and proper that we should do this.
+
+But, in a larger sense, we can not dedicate -- we can not consecrate -- we can
+not hallow -- this ground. The brave men, living and dead, who struggled here,
+have consecrated it, far above our poor power to add or detract. The world will
+little note, nor long remember what we say here, but it can never forget what
+they did here. It is for us the living, rather, to be dedicated here to the
+unfinished work which they who fought here have thus far so nobly advanced. It
+is rather for us to be here dedicated to the great task remaining before us --
+that from these honored dead we take increased devotion to that cause for which
+they gave the last full measure of devotion -- that we here highly resolve that
+these dead shall not have died in vain -- that this nation, under God, shall
+have a new birth of freedom -- and that government of the people, by the
+people, for the people, shall not perish from the earth.
+EOS
+)
+      fm.tokens.each do |token|
+        token.should_not be_empty
+      end
+    end
   end
 
   describe "freqs" do