Permalink
Browse files

finally accessdb and downloader are working together :)

  • Loading branch information...
1 parent 3021f75 commit 972bd2e360771274967b6ff486d22edda528c402 @pawelkl-zz committed Jul 2, 2012
Showing with 161 additions and 84 deletions.
  1. +78 −76 AccessDb.rb
  2. +52 −0 AccessDbTest.rb
  3. +8 −5 Downloader.rbw
  4. +23 −3 Etl.rb
View
@@ -3,104 +3,106 @@
require 'json'
class AccessDb
- def initialize dbname, collection #, username, password
- @dbname = dbname
- @collection = collection
- @conn = Mongo::Connection.new
- @db = @conn[dbname]
- @coll = @db[collection]
- end
+ def initialize dbname, collection #, username, password
+ @dbname = dbname
+ @collection = collection
+ @conn = Mongo::Connection.new
+ @db = @conn[dbname]
+ @coll = @db[collection]
+ end
- def upsert_by_id id, json
- @coll.update({ :_id => id }, json, :upsert => true)
- end
+ def upsert_by_id id, json
+ @coll.update({ :_id => id }, json, :upsert => true)
+ end
- def upsert_by_meta json
- # print json
- @coll.update({ "hash_md5" => json[:hash_md5] }, json, :upsert => true)
- end
+ def upsert_by_meta json
+ # print json
+ @coll.update({ "hash_md5" => json[:hash_md5] }, json, :upsert => true)
+ end
- def remove json
- if json[:_id].nil?
- then @coll.remove({ :hash_md5 => json[:hash_md5]})
- else
- @coll.remove({:_id => json[:id]})
- end
- end
+ def remove json
+ if json[:_id].nil?
+ then @coll.remove({ :hash_md5 => json[:hash_md5]})
+ else
+ @coll.remove({:_id => json[:id]})
+ end
+ end
- def add_info json, param, value
- if json[:_id].nil?
- then @coll.update({ :hash_md5 => json[:hash_md5] }, '$set' => { param => value })
- else
- @coll.update({ :_id => json[:id] }, '$set' => { param => value })
- end
- end
+ def add_info json, param, value
+ if json[:_id].nil?
+ then @coll.update({ :hash_md5 => json[:hash_md5] }, '$set' => { param => value })
+ else
+ @coll.update({ :_id => json[:id] }, '$set' => { param => value })
+ end
+ end
- def remove_info json, param
- if json[:_id].nil?
- then @coll.update({ :hash_md5 => json[:hash_md5] }, '$set' => { param => nil })
- else
- @coll.update({ :_id => json[:id] }, '$set' => { param => nil})
- end
- end
+ def remove_info json, param
+ if json[:_id].nil?
+ then @coll.update({ :hash_md5 => json[:hash_md5] }, '$set' => { param => nil })
+ else
+ @coll.update({ :_id => json[:id] }, '$set' => { param => nil})
+ end
+ end
- def find json
- @coll.find_one({:hash_md5 => json[:hash_md5]},{:fields => {:_id=>0}})
- end
+ def find json
+ @coll.find_one({:hash_md5 => json[:hash_md5]},{:fields => {:_id=>0}})
+ end
- def update query, update
- @coll.find_and_modify(:query => query, :update => update)
- end
+ def update query, update
+ @coll.find_and_modify(:query => query, :update => update)
+ end
end
if __FILE__ == $0
- require 'test/unit'
- require 'active_support/core_ext/hash'
+ require 'test/unit'
+ require 'active_support/core_ext/hash'
+ require 'mongo'
+ require 'json'
- class AccessDbTest < Test::Unit::TestCase
- def setup
- @coll = AccessDb.new "meta","meta"
- end
+ class AccessDbTest < Test::Unit::TestCase
+ def setup
+ @coll = AccessDb.new "meta","meta"
+ end
- def teardown
- @coll.remove({:hash_md5 => [:sara,:ania]})
- end
+ def teardown
+ @coll.remove({:hash_md5 => [:sara,:ania]})
+ end
- def test_read
- json = {:hash_md5 => :sara}
- id = @coll.upsert_by_meta json
- # puts id
- find = @coll.find json
- # assert_equal(json[:hash_md5], find.except!("_id")["hash_md5"])
- assert_equal json[:hash_md5], find["hash_md5"], "json file didn't match input file"
- #find.except!("_id")["hash_md5"])
- # assert_equal(82,id)
+ def test_read
+ json = {:hash_md5 => :sara}
+ id = @coll.upsert_by_meta json
+ # puts id
+ find = @coll.find json
+ # assert_equal(json[:hash_md5], find.except!("_id")["hash_md5"])
+ assert_equal json[:hash_md5], find["hash_md5"], "json file didn't match input file"
+ #find.except!("_id")["hash_md5"])
+ # assert_equal(82,id)
end
def test_read_with_hash
- json = {:hash_md5 => :lolcats}
- id = @coll.upsert_by_meta json
- find = @coll.find json
- # json[:_id] = id
- assert_equal json, find, "retrieved json file isn't exactly the same as input json"
+ json = {:hash_md5 => :lolcats}
+ id = @coll.upsert_by_meta json
+ find = @coll.find json
+ # json[:_id] = id
+ assert_equal json, find, "retrieved json file isn't exactly the same as input json"
end
def test_added_info
- json = {:hash_md5 => :ania}
- @coll.upsert_by_meta json
- @coll.add_info(json, :stan, :zajeta)
- find = @coll.find(json)
- assert_equal(json[:stan], find[:stan])
- @coll.remove_info(json, :stan)
- assert_equal(json[:stan], nil)
+ json = {:hash_md5 => :ania}
+ @coll.upsert_by_meta json
+ @coll.add_info(json, :stan, :zajeta)
+ find = @coll.find(json)
+ assert_equal(json[:stan], find[:stan])
+ @coll.remove_info(json, :stan)
+ assert_equal(json[:stan], nil)
end
def test_delete
- json = {:hash_md5 => :beata}
- @coll.upsert_by_meta json
- @coll.remove({:hash_md5 => :beata})
- find = @coll.find(json)
- assert_equal(nil, find)
+ json = {:hash_md5 => :beata}
+ @coll.upsert_by_meta json
+ @coll.remove({:hash_md5 => :beata})
+ find = @coll.find(json)
+ assert_equal(nil, find)
end
end
end
View
@@ -0,0 +1,52 @@
+require 'test/unit'
+require 'active_support/core_ext/hash'
+require 'mongo'
+require 'json'
+require 'AccessDb'
+
+class AccessDbTest < Test::Unit::TestCase
+def setup
+ @coll = AccessDb.new "meta","meta"
+end
+
+def teardown
+ @coll.remove({:hash_md5 => [:sara,:ania]})
+end
+
+def test_read
+ json = {:hash_md5 => :sara}
+ id = @coll.upsert_by_meta json
+ # puts id
+ find = @coll.find json
+ # assert_equal(json[:hash_md5], find.except!("_id")["hash_md5"])
+ assert_equal json[:hash_md5], find["hash_md5"], "json file didn't match input file"
+ #find.except!("_id")["hash_md5"])
+ # assert_equal(82,id)
+ end
+
+ def test_read_with_hash
+ json = {:hash_md5 => :lolcats}
+ id = @coll.upsert_by_meta json
+ find = @coll.find json
+ # json[:_id] = id
+ assert_equal json, find, "retrieved json file isn't exactly the same as input json"
+ end
+
+ def test_added_info
+ json = {:hash_md5 => :ania}
+ @coll.upsert_by_meta json
+ @coll.add_info(json, :stan, :zajeta)
+ find = @coll.find(json)
+ assert_equal(json[:stan], find[:stan])
+ @coll.remove_info(json, :stan)
+ assert_equal(json[:stan], nil)
+ end
+
+ def test_delete
+ json = {:hash_md5 => :beata}
+ @coll.upsert_by_meta json
+ @coll.remove({:hash_md5 => :beata})
+ find = @coll.find(json)
+ assert_equal(nil, find)
+ end
+end
View
@@ -139,8 +139,8 @@ class Downloader
then json[:hash_bigfile] = @hash end
json[:hash_md5] = Digest::MD5.hexdigest(File.read(@save_location))
-
- JSON.pretty_generate(json)
+ # puts JSON.pretty_generate(json)
+ json
end
def add_links(url_array,cred=nil,ref=nil,cookie=nil)
@@ -156,13 +156,16 @@ class Downloader
json = parse_link_info single_url
# puts json
id = @mongo.upsert_by_meta json
- puts id
- json["_id"] = id
- File.open(@save_location + :"meta.json","w").write json
+ # puts id
+ # json["_id"] = id
+ puts @save_location
+ File.open(@save_location + :":meta.json".to_s,"w").write json
end
end
end
+if __FILE__ == $0; options[:destination] = 'c:/temp' end
+
manager = Downloader.new options[:destination]
if ARGV.nil?
View
26 Etl.rb
@@ -23,7 +23,7 @@
optparse.parse!
if __FILE__ == $0
- options[:directory] = 'c:/foobar2000'
+ options[:directory] = 'c:/temp'
pp "Options:", options
pp "ARGV:", ARGV
end
@@ -39,7 +39,27 @@
# end
# }
-Find.find(options[:directory]) do |f|
- File.exists? f:":meta.json"
+Find.find(options[:directory]) do |f|
+ target = f + :":meta.json".to_s
+ if File.exists? target; pp target end
+ # puts target
+ puts File.open(target,"r").read
end
+=begin
+if __FILE__ == $0
+ require 'test/unit'
+ require 'active_support/core_ext/hash'
+
+ class EtlTest < Test::Unit::TestCase
+ def setup
+ @coll = AccessDb.new "meta","meta"
+ end
+
+ def teardown
+ @coll.remove({:hash_md5 => [:sara,:ania]})
+ end
+
+ end
+end
+=end

0 comments on commit 972bd2e

Please sign in to comment.