Skip to content
Retrieve is an extensible library for retrieving resources identified by a URI. Retrieval methods may be registered to URI schemes.
Ruby
Fetching latest commit…
Cannot retrieve the latest commit at this time.
Failed to load latest commit information.
lib Changed how URI normalization occurs. Feb 5, 2009
spec
tasks
vendor
website
.gitignore
.gitmodules
CHANGELOG
LICENSE
README
Rakefile

README

== Retrieve

Homepage::  retrieve.rubyforge.org[http://retrieve.rubyforge.org/]
Author::    Bob Aman  (mailto:bob@sporkmonger.com)
Copyright:: Copyright © 2008 Bob Aman
License::   MIT

== Description

Retrieve is an extensible library for retrieving resources identified by a
URI. Retrieval methods are registered to URI schemes.

== Features

* Access any resource with a client defined for its scheme.
* Read and write to anything that can be addressed by a URI.
* Easily implement additional clients for other URI schemes.
* Supports "http" and "file" schemes by default.
* The HTTP client is faster and more efficient than Net::HTTP.

== Example Usage

  require "retrieve/clients/file"

  res = Retrieve.open("file:///home/sporkmonger/todo.txt", :mode => :read)
  #=> #<Retrieve::Resource:0x847c2c URI:file:///home/sporkmonger/todo.txt>
  res.read
  #=> "TODO: Write some code."
  res.close

  require "retrieve/clients/http"

  status, headers, body = nil
  Retrieve.open("http://www.google.com/", :method => :get) do |resource|
    body = resource.read
    status = resource.metadata[:status]
    reason = resource.metadata[:reason]
    headers = resource.metadata[:headers]
  end
  status
  #=> "200" # NOTE: This is a String
  reason
  #=> "OK"
  headers
  #=> {"Content-Type" => "text/html; charset=ISO-8859-1", ...}
  body[0..50] + " ..."
  #=> "<html><head><meta http-equiv=\"content-type\" content ..."

  # Usage for persistent connections

  connections = {}
  status, headers, body = nil
  Retrieve.open(
    "http://google.com/",
    :redirect => true,
    :connections => connections
  ) do |resource|
    body = resource.read
  end
  body[0..50] + " ..."
  #=> "<html><head><meta http-equiv=\"content-type\" content ..."

  # Advanced example using ranges

  connections = {}
  uri = "http://www.example.com/"
  File.open("example.html", "w") do |file|
    file.write(File.open("http.log", "a") do |log|
      accept_ranges = nil
      content_length = nil
      bytes_received = 0
      Retrieve.open(
        uri, :method => :head, :connections => connections, :log => log
      ) do |r|
        accept_ranges = r.metadata[:headers]["Accept-Ranges"]
        content_length = r.metadata[:headers]["Content-Length"]
      end
      buffer = StringIO.new
      if accept_ranges == "bytes"
        log.write("* Server claims to support partial content.\n")
        while content_length.to_i > bytes_received
          if (131072 + bytes_received) > content_length.to_i
            range = bytes_received..-1
          else
            range = bytes_received..131072 + bytes_received
          end
          Retrieve.open(
            uri, :connections => connections, :range => range, :log => log
          ) do |r|
            buffer.write(r.read)
            bytes_received += r.metadata[:headers]["Content-Length"].to_i
            if r.metadata[:status] == "200"
              log.write("* Server did not return partial content.\n")
            end
          end
        end
      else
        log.write("* Server does NOT claim to support partial content.\n")
        Retrieve.open(uri, :connections => connections, :log => log) do |r|
          buffer.write(r.read)
        end
      end
      for pair, connection in connections
        log.write("* Closing connection to #{pair[0]} port #{pair[1]}\n")
        connection.close if connection
        connections.delete(pair)
      end
      buffer.string
    end)
  end

== Requirements

* addressable

== Install

* sudo gem install retrieve
Something went wrong with that request. Please try again.