Permalink
Browse files

Fix when results are > 1000 due to prefixes.

  • Loading branch information...
1 parent 84ad465 commit 8d0a80110751adb51690a7028ec3f551f77aa5e9 @stephenh committed Jan 25, 2011
Showing with 20 additions and 5 deletions.
  1. +2 −1 bin/s3fsr
  2. +4 −0 lib/aws-matt/s3/bucket.rb
  3. +14 −4 lib/s3fsr.rb
View
@@ -16,7 +16,8 @@ if ENV['AWS_ACCESS_KEY_ID'] == nil or ENV['AWS_SECRET_ACCESS_KEY'] == nil
end
root = BUCKET != nil ? SBucketDir.new(nil, BUCKET) : SBucketsDir.new
-s3fsr = MethodLogger.new(S3fsr.new(root))
+s3fsr = S3fsr.new(root)
+# s3fsr = MethodLogger.new(s3fsr)
FuseFS.set_root s3fsr
FuseFS.mount_under MOUNT #, "allow_other"
begin
@@ -303,6 +303,10 @@ def update(action, object) #:nodoc:
when :deleted then object_cache.delete(object)
end
end
+
+ def truncated?
+ attributes['is_truncated']
+ end
private
def build_contents!
View
@@ -85,7 +85,11 @@ def get_contents
marker = ''
loop do
s3_bucket = AWS::S3::Bucket.find(bucket, :prefix => prefix, :delimiter => '/', :marker => marker, :max_keys => 1000)
- s3_bucket.object_cache.each do |s3_obj|
+
+ oc = s3_bucket.object_cache
+ cpc = s3_bucket.common_prefix_cache
+
+ oc.each do |s3_obj|
# Technically we should use S3SYNC_DIR_LENGTH but aws-s3 decides it
# needs to issue an HEAD request for every dir for that.
if s3_obj.etag == S3SYNC_DIR_ETAG or s3_obj.key.end_with? S3ORGANIZER_DIR_SUFFIX
@@ -98,12 +102,18 @@ def get_contents
@data << SFile.new(self, s3_obj)
end
end
- s3_bucket.common_prefix_cache.reject { |p| p == '/' }.each do |prefix|
+
+ cpc.reject { |p| p == '/' }.each do |prefix|
hidden = SPrefixDir.new(self, prefix)
@data << hidden unless @data.find { |i| i.name == hidden.name }
end
- break unless s3_bucket.object_cache.length > 0 && s3_bucket.object_cache.length % 1000 == 0
- marker = s3_bucket.object_cache.last.key
+
+ break unless s3_bucket.truncated?
+
+ # find which of the object/prefix keys is alphabetically last
+ last_object_key = oc.size == 0 ? '' : oc.last.key
+ last_prefix_key = cpc.size == 0 ? '' : cpc.last
+ marker = last_object_key < last_prefix_key ? last_prefix_key : last_object_key
end
puts "done"
@data

0 comments on commit 8d0a801

Please sign in to comment.