Permalink
Browse files

Rename Shuck codebase to FakeS3

  • Loading branch information...
1 parent b00db5d commit bb35f3902a33b9c6d017faa255fa2986ba5e8bff @jubos committed Apr 13, 2012
View
@@ -0,0 +1,4 @@
+pkg/*
+*.gem
+.bundle
+tmp
View
@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fakes3.gemspec
+gemspec
View
@@ -0,0 +1,32 @@
+PATH
+ remote: .
+ specs:
+ fakes3 (0.1.0)
+ builder
+ thor
+
+GEM
+ remote: http://rubygems.org/
+ specs:
+ aws-s3 (0.6.2)
+ builder
+ mime-types
+ xml-simple
+ builder (2.1.2)
+ mime-types (1.16)
+ right_aws (2.0.0)
+ right_http_connection (>= 1.2.1)
+ right_http_connection (1.2.4)
+ thor (0.14.4)
+ xml-simple (1.0.12)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ aws-s3
+ builder
+ bundler (>= 1.0.0)
+ fakes3!
+ right_aws
+ thor
View
@@ -0,0 +1,28 @@
+== Introduction ==
+FakeS3 is a lightweight server that responds to the same calls Amazon S3 responds to.
+It is extremely useful for testing of S3 in a sandbox environment without actually
+making calls to Amazon, which not only require network, but also cost you precious dollars.
+
+For now there is a basic file store backend.
+
+FakeS3 doesn't support all of the S3 command set, but the basic ones like put, get,
+list, copy, and make bucket are supported. More coming soon.
+
+== Installation ==
+ gem install fakes3
+
+== Running ==
+To run a fakes3 server, you just specify a root and a port.
+
+ fakes3 -r /mnt/fakes3_root -p 4567
+
+== Connecting to FakeS3 ==
+
+Take a look at the test cases to see client example usage. For now, FakeS3 is
+mainly tested with s3cmd, aws-s3 gem, and right_aws. There are plenty more
+libraries out there, and please do mention other clients.
+
+== Running Tests ==
+In order to run the tests add the following line to your /etc/hosts:
+
+ 127.0.0.1 s3.localhost
View
@@ -0,0 +1,14 @@
+require 'bundler'
+require 'rake/testtask'
+include Rake::DSL
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new(:test) do |t|
+ t.test_files = FileList['test/*_test.rb']
+ t.ruby_opts = ['-rubygems'] if defined? Gem
+ t.ruby_opts << '-I.'
+end
+
+task :test_server do |t|
+ system("bundle exec bin/fakes3 --port 10453 --root test_root")
+end
View
@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'fakes3/cli'
+FakeS3::CLI.start
View
@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "fakes3/version"
+
+Gem::Specification.new do |s|
+ s.name = "fakes3"
+ s.version = FakeS3::VERSION
+ s.platform = Gem::Platform::RUBY
+ s.authors = ["Curtis Spencer"]
+ s.email = ["curtis@sevenforge.com"]
+ s.homepage = ""
+ s.summary = %q{FakeS3 is a phony S3 engine with minimal dependencies}
+ s.description = %q{Use FakeS3 to test basic S3 functionality without actually connecting to S3}
+
+ s.rubyforge_project = "fakes3"
+
+ s.add_development_dependency "bundler", ">= 1.0.0"
+ s.add_development_dependency "aws-s3"
+ s.add_development_dependency "right_aws"
+ #s.add_development_dependency "aws-sdk"
+ #s.add_development_dependency "ruby-debug19"
+ s.add_dependency "thor"
+ s.add_dependency "builder"
+
+ s.files = `git ls-files`.split("\n")
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+ s.require_paths = ["lib"]
+end
View
@@ -0,0 +1,3 @@
+require 'fakes3/version'
+require 'fakes3/file_store'
+require 'fakes3/server'
View
@@ -0,0 +1,18 @@
+require 'builder'
+require 'fakes3/s3_object'
+
+module FakeS3
+ class Bucket
+ attr_accessor :name,:creation_date,:objects
+
+ def initialize(name,creation_date,objects)
+ @name = name
+ @creation_date = creation_date
+ @objects = []
+ objects.each do |obj|
+ @objects << obj
+ end
+ end
+
+ end
+end
View
@@ -0,0 +1,59 @@
+require 'thor'
+require 'fakes3/server'
+require 'fakes3/version'
+
+module FakeS3
+ class CLI < Thor
+ default_task("server")
+
+ desc "server", "Run a server on a particular hostname"
+ method_option :root, :type => :string, :aliases => '-r', :required => true
+ method_option :port, :type => :numeric, :aliases => '-p', :required => true
+ method_option :hostname, :type => :string, :aliases => '-h', :desc => "The root name of the host. Defaults to s3.amazonaws.com."
+ method_option :limit, :aliases => '-l', :type => :string, :desc => 'Rate limit for serving (ie. 50K, 1.0M)'
+ def server
+ store = nil
+ if options[:root]
+ root = File.expand_path(options[:root])
+ store = FileStore.new(root)
+ end
+
+ if store.nil?
+ puts "You must specify a root to use a file store (the current default)"
+ exit(-1)
+ end
+
+ hostname = 's3.amazonaws.com'
+ if options[:hostname]
+ hostname = options[:hostname]
+ # In case the user has put a port on the hostname
+ if hostname =~ /:(\d+)/
+ hostname = hostname.split(":")[0]
+ end
+ end
+
+ if options[:limit]
+ begin
+ store.rate_limit = options[:limit]
+ rescue
+ puts $!.message
+ exit(-1)
+ end
+ end
+
+ puts "Loading FakeS3 with #{root} on port #{options[:port]} with hostname #{hostname}"
+ server = FakeS3::Server.new(options[:port],store,hostname)
+ server.serve
+ end
+
+ desc "version", "Report the current fakes3 version"
+ def version
+ puts <<"EOF"
+======================
+FakeS3 #{FakeS3::VERSION}
+
+Copyright 2012, Curtis Spencer (@jubos)
+EOF
+ end
+ end
+end
View
@@ -0,0 +1,153 @@
+require 'fileutils'
+require 'time'
+require 'fakes3/s3_object'
+require 'fakes3/bucket'
+require 'fakes3/rate_limitable_file'
+require 'digest/md5'
+require 'yaml'
+
+module FakeS3
+ class FileStore
+ SHUCK_METADATA_DIR = ".fakes3_metadataFFF"
+
+ def initialize(root)
+ @root = root
+ @buckets = []
+ @bucket_hash = {}
+ Dir[File.join(root,"*")].each do |bucket|
+ bucket_name = File.basename(bucket)
+ bucket_obj = Bucket.new(bucket_name,Time.now,[])
+ @buckets << bucket_obj
+ @bucket_hash[bucket_name] = bucket_obj
+ end
+ end
+
+ # Pass a rate limit in bytes per second
+ def rate_limit=(rate_limit)
+ if rate_limit.is_a?(String)
+ if rate_limit =~ /^(\d+)$/
+ RateLimitableFile.rate_limit = rate_limit.to_i
+ elsif rate_limit =~ /^(.*)K$/
+ RateLimitableFile.rate_limit = $1.to_f * 1000
+ elsif rate_limit =~ /^(.*)M$/
+ RateLimitableFile.rate_limit = $1.to_f * 1000000
+ elsif rate_limit =~ /^(.*)G$/
+ RateLimitableFile.rate_limit = $1.to_f * 1000000000
+ else
+ raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)"
+ end
+ else
+ RateLimitableFile.rate_limit = nil
+ end
+ end
+
+ def buckets
+ @buckets
+ end
+
+ def get_bucket(bucket)
+ @bucket_hash[bucket]
+ end
+
+ def create_bucket(bucket)
+ FileUtils.mkdir_p(File.join(@root,bucket))
+ bucket_obj = Bucket.new(bucket,Time.now,[])
+ if !@bucket_hash[bucket]
+ @buckets << bucket_obj
+ @bucket_hash[bucket] = bucket_obj
+ end
+ end
+
+ def get_object(bucket,object, request)
+ begin
+ real_obj = S3Object.new
+ obj_root = File.join(@root,bucket,object,SHUCK_METADATA_DIR)
+ metadata = YAML.parse(File.open(File.join(obj_root,"metadata"),'rb').read)
+ real_obj.name = object
+ real_obj.md5 = metadata[:md5].value
+ real_obj.content_type = metadata[:content_type] ? metadata[:content_type].value : "application/octet-stream"
+ #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
+ real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
+ return real_obj
+ rescue
+ puts $!
+ return nil
+ end
+ end
+
+ def object_metadata(bucket,object)
+ end
+
+ def copy_object(src_bucket,src_object,dst_bucket,dst_object)
+ src_root = File.join(@root,src_bucket,src_object,SHUCK_METADATA_DIR)
+ src_obj = S3Object.new
+ src_metadata_filename = File.join(src_root,"metadata")
+ src_metadata = YAML.parse(File.open(src_metadata_filename,'rb').read)
+ src_content_filename = File.join(src_root,"content")
+
+ dst_filename= File.join(@root,dst_bucket,dst_object)
+ FileUtils.mkdir_p(dst_filename)
+
+ metadata_dir = File.join(dst_filename,SHUCK_METADATA_DIR)
+ FileUtils.mkdir_p(metadata_dir)
+
+ content = File.join(metadata_dir,"content")
+ metadata = File.join(metadata_dir,"metadata")
+
+ File.open(content,'wb') do |f|
+ File.open(src_content_filename,'rb') do |input|
+ f << input.read
+ end
+ end
+
+ File.open(metadata,'w') do |f|
+ File.open(src_metadata_filename,'r') do |input|
+ f << input.read
+ end
+ end
+
+ obj = S3Object.new
+ obj.md5 = src_metadata[:md5]
+ obj.content_type = src_metadata[:content_type]
+ return obj
+ end
+
+ def store_object(bucket,object,request)
+ begin
+ filename = File.join(@root,bucket,object)
+ FileUtils.mkdir_p(filename)
+
+ metadata_dir = File.join(filename,SHUCK_METADATA_DIR)
+ FileUtils.mkdir_p(metadata_dir)
+
+ content = File.join(filename,SHUCK_METADATA_DIR,"content")
+ metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")
+
+ md5 = Digest::MD5.new
+
+ File.open(content,'wb') do |f|
+ request.body do |chunk|
@alexmuntean

alexmuntean Jan 27, 2014

this line is taking 3 seconds.. do you have any idea how to fix this?

+ f << chunk
+ md5 << chunk
+ end
+ end
+
+ metadata_struct = {}
+ metadata_struct[:md5] = md5.hexdigest
+ metadata_struct[:content_type] = request.header["content-type"].first
+
+ File.open(metadata,'w') do |f|
+ f << YAML::dump(metadata_struct)
+ end
+ obj = S3Object.new
+ obj.md5 = metadata_struct[:md5]
+ obj.content_type = metadata_struct[:content_type]
+ return obj
+ rescue
+ puts $!
+ $!.backtrace.each { |line| puts line }
+ return nil
+ end
+ end
+ end
+end
@@ -0,0 +1,21 @@
+module FakeS3
+ class RateLimitableFile < File
+ @@rate_limit = nil
+ # Specify a rate limit in bytes per second
+ def self.rate_limit
+ @@rate_limit
+ end
+
+ def self.rate_limit=(rate_limit)
+ @@rate_limit = rate_limit
+ end
+
+ def read(args)
+ if @@rate_limit
+ time_to_sleep = args / @@rate_limit
+ sleep(time_to_sleep)
+ end
+ return super(args)
+ end
+ end
+end
View
@@ -0,0 +1,5 @@
+module FakeS3
+ class S3Object
+ attr_accessor :name,:size,:creation_date,:md5,:io,:content_type
+ end
+end
Oops, something went wrong.

0 comments on commit bb35f39

Please sign in to comment.