@@ -16,7 +16,7 @@ class OptS3rquery def self.parse(args) options = {} opts = OptionParser.new do |opts| - opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats> <parameters> (parameters can be name=test or simply test)" + opts.banner = "Usage: s3query.rb [options] <search|get|unpack|delete|stats|delete_older> <parameters> (parameters can be name=test or simply test)" opts.on("-s", "search words", String, "Search something") do |name| options[:op] = "search" @@ -78,6 +78,10 @@ class OptS3rquery options[:config_num] = name end + opts.on("-e", "--item-to-keep NUM", Integer, "Number of item to keep after deleting the olders (delete_older)") do |name| + options[:delete_older] = name + end + opts.on("--initialize", "Inizializza bucket and db") do |name| options[:initialize] = true end @@ -85,7 +89,7 @@ class OptS3rquery opts.on("--destroy", "Destroy bucket and db") do |name| options[:destroy] = true end - + opts.on("--test", "Test something") do |name| options[:test] = true end @@ -129,7 +133,7 @@ s3db = S3SyncDb.new(config.current) command = ARGV.shift case command - when /[search|get|unpack|delete|stats]/ + when /[search|get|unpack|delete|stats|delete_older]/ results = s3db.find(ARGV, nil, options) results = get_last(results) if options[:last] results = get_first(results) if options[:first] @@ -166,6 +170,27 @@ case command puts "Deleting of #{ret["aws_name"]}" s3db.delete(ret) end + when 'delete_older' + #cancella i piu' vecchi mantenendo options[:delete_older] item + # per ongi nome trovato + if options[:delete_older] == nil + options[:delete_older] = 2 + end + group = {} + results.each do |ret| + group[ret["name"]] ||= [] + group[ret["name"]] << ret + end + group.each do |key, arr_ret| + arr_ret.sort! {|x,y| x["datetime"] <=> y["datetime"] } + options[:delete_older].times do |num| + arr_ret.pop + end + arr_ret.each do |ret| + puts "Deleting of #{ret["aws_name"]}" + s3db.delete(ret) + end + end when 'stats' #get size bucks_s = {} bin/s3query.rb @@ -40,6 +40,10 @@ class OptS3rbackup options[:log] = true end + opts.on("--md5", "Check md5, upload only if md5 differ") do |name| + options[:md5_check] = true + end + opts.on("--nolog", "Log disabled") do |name| options[:log] = false end @@ -77,5 +81,5 @@ config.current["log"] = options[:log] if options[:log] != nil config.current["bucket_log"] = options[:bucket_log] if options[:bucket_log] config.current["compression"] = options[:compression] if options[:compression] s3db = S3SyncDb.new(config.current) -s3db.bak(ARGV, options[:name], options[:descr]) +s3db.bak(ARGV, options[:name], options[:descr], options[:md5_check]) #s3db.salva_db bin/s3rbackup.rb @@ -60,7 +60,7 @@ class S3SyncDb end #s3 sdb - def bak(dirs, name, descr) + def bak(dirs, name, descr, md5_check = false) name = dirs[0] if !name tf = Tempfile.new("s3rbackup") tf_l = Tempfile.new("s3rbackup-listfile") @@ -95,45 +95,58 @@ class S3SyncDb doc["host"] = `hostname`.gsub("\n","").to_s doc["user"] = `whoami`.gsub("\n","").to_s doc["size"] = File.size(file_path) + doc["md5"] = `md5sum #{file_path} | cut -d " " -f 1`.gsub("\n", "") doc["compression"] = @config["compression"] doc["archive"] = "tar" #@db << doc aws_name = "#{doc["name"]}##{`date +%Y%m%d_%H.%M.%S`}".gsub("\n","") doc["aws_name"] = aws_name - @sdb.put_attributes @domain_db, aws_name, doc - if @config['files_db'] - #tutti in uno - filez_name = [] - filez.each do |fil| - begin - filez_name << File.basename(fil.gsub("\n", "")) - rescue + + carica = true + if md5_check + search = "['md5' = '#{doc["md5"]}']" + @sdb.query(@domain_db, search) do |result| + result[:items].each do |item| + hattr = @sdb.get_attributes(@domain_db, item)[:attributes] + if hattr["name"][0] == doc["name"] + carica = false + end end end - @sdb.put_attributes @config['files_db'], aws_name, { - :files_full => filez.map {|t| t.gsub("\n","")}, - :files_name => filez_name} end - - #TODO aggiungere md5 - # Store it! - options = {} - options["x-amz-meta-host"] = doc["host"] - options["x-amz-meta-user"] = doc["user"] - options["x-amz-meta-descrizione"] = doc["description"] - options["x-amz-meta-current_path"] = doc["current_path"] - options["x-amz-meta-size"] = doc["size"] - options["x-amz-meta-compression"] = doc["compression"] - options["x-amz-meta-archive"] = doc["archive"] - - store = S3Object.store(aws_name, open(file_path), @config["bucket"], options) - #obj = S3Object.find(aws_name, @config["bucket"]) - #obj.store - #obj.about.each do |key,val| - # doc[key] = val - #end - #TODO aggiungere check - send_mail("S3rbackup - Saved #{doc["name"]}", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}") + if carica + @sdb.put_attributes @domain_db, aws_name, doc + if @config['files_db'] + #tutti in uno + filez_name = [] + filez.each do |fil| + begin + filez_name << File.basename(fil.gsub("\n", "")) + rescue + end + end + @sdb.put_attributes @config['files_db'], aws_name, { + :files_full => filez.map {|t| t.gsub("\n","")}, + :files_name => filez_name} + end + + #TODO aggiungere md5 + # Store it! + options = {} + # options["x-amz-meta-host"] = doc["host"] + # options["x-amz-meta-user"] = doc["user"] + # options["x-amz-meta-descrizione"] = doc["description"] + # options["x-amz-meta-current_path"] = doc["current_path"] + # options["x-amz-meta-size"] = doc["size"] + # options["x-amz-meta-compression"] = doc["compression"] + # options["x-amz-meta-archive"] = doc["archive"] + + store = S3Object.store(aws_name, open(file_path), @config["bucket"], options) + #TODO aggiungere check + send_mail("S3rbackup - Saved #{doc["name"]}", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}") + else + send_mail("S3rbackup - Not Saved #{doc["name"]} - md5 already exist", (doc.to_a.map {|val| "#{val[0]}: #{val[1]}"}).join("\n") + "\n\nFiles:\n\t#{filez.join("\t")}") + end end def find(words, bucket = nil, cmd_opt = {}) lib/s3dbsync.rb @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = "s3rbackup" - s.version = "0.3.4" - s.date = "2008-09-28" + s.version = "0.4.0" + s.date = "2008-11-28" s.summary = "Simple backup on Amazon S3" s.email = "sacchi.nicola@gmail.com" s.homepage = "http://github.com/niky81/s3rbackup" s3rbackup.gemspec 7fc2b7805445bf8bb84fa401bb91e27b46179639 nik sacchi.nicola@gmail.com http://github.com/niky81/s3rbackup/commit/343095aa18ccb198b1d46254b37f43be02312851 343095aa18ccb198b1d46254b37f43be02312851 2008-11-28T11:20:51-08:00 2008-11-28T11:20:51-08:00 added md5 check and delete_older 9a54235a3eca641d2172d38208c535f2bf22a3e1 nik sacchi.nicola@gmail.com