Fully load a stream, fetching up to the last public post.

This also changes mocks in related tests.

lib/tumble_out/contentizer.rb

@@ -4,16 +4,17 @@ class Contentizer
 
     def initialize(url)
       @url = url
-      @total_posts = 0
+      @total_posts = nil
       @chunk_size  = 50
       @posts = []
       @done  = false
     end
 
     def posts
-      unless @done
+      until @done
         chunk = raw_posts(@posts.size)
         @posts += chunk.map { |rp| Post.new rp }
+        @done = @posts.size == @total_posts
       end
 
       @posts
@@ -24,7 +25,7 @@ def each_post(&blk)
     end
 
     def dump(directory)
-      directory = File.join(directory, @url, "posts")
+      directory = File.join(directory, @url, "_posts")
 
       if !File.exist?(directory)
         FileUtils.mkdir_p directory
@@ -34,9 +35,14 @@ def dump(directory)
     end
 
     private
-
     def raw_posts(offset=0)
-      doc = Nokogiri::XML(Net::HTTP.get(URI.parse("http://#{@url}/api/read")))
+      uri = "http://#{@url}/api/read?start=#{offset}"
+      doc = Nokogiri::XML(Net::HTTP.get(URI.parse(uri)))
+
+      if @total_posts.nil?
+        @total_posts = doc.search("posts").first.
+          attr("total").to_i
+      end
 
       doc.search "post"
     end

test/unit/test_contentizer.rb

@@ -9,7 +9,7 @@ def setup
                     )
 
     Net::HTTP.expects(:get).
-      with(URI.parse("http://sample.tumblr.com/api/read")).returns raw_data
+      with(URI.parse("http://sample.tumblr.com/api/read?start=0")).returns raw_data
     @contentizer = TumbleOut::Contentizer.new("sample.tumblr.com")
   end
 
@@ -39,7 +39,7 @@ def test_whether_posts_are_of_specific_types
   # TODO: mock the crap out of this.
   def test_that_dump_creates_a_file_for_each_post
     full_path = File.join("/tmp", @contentizer.url,
-                          "posts", "*")
+                          "_posts", "*")
 
     @contentizer.dump "/tmp"
 

test/unit/test_post.rb

@@ -9,7 +9,7 @@ def setup
                     )
 
     Net::HTTP.expects(:get).
-      with(URI.parse("http://sample.tumblr.com/api/read")).returns raw_data
+      with(URI.parse("http://sample.tumblr.com/api/read?start=0")).returns raw_data
     contentizer = TumbleOut::Contentizer.new("sample.tumblr.com")
     @posts = contentizer.posts
   end