Permalink
Browse files

test in accessdb working, TODO: hash checking & downloader integration

  • Loading branch information...
1 parent a80f5a5 commit 823b55ca3e73553851b3d66e6c79a64ce187d1cb @pawelkl-zz committed Jun 26, 2012
Showing with 93 additions and 73 deletions.
  1. +3 −0 .gitignore
  2. +67 −24 AccessDb.rb
  3. +21 −48 Downloader.rbw
  4. +1 −0 start-mongo-shell.bat
  5. +1 −1 start-mongo-standalone.bat
View
@@ -4,6 +4,9 @@ journal-i.ri
journal1.tex
journal2.tex
journals.bib
+meta.0
+meta.ns
+lsn
#################
## Eclipse
View
@@ -1,35 +1,49 @@
-require 'rubygems'
+# require 'rubygems'
require 'mongo'
require 'json'
-module AccessDb
- def AccessDb.setup
- @db = Connection.new.db('meta-files');
- setup_collections
- end
- def AccessDb.connect
- @db = Mongo::Connection.new.db('meta',
- :pool_size => 5, :timeout => 5)
+class AccessDb
+ def initialize dbname, collection #, username, password
+ # @db = Connection.new.db('meta');
+ # @coll = db.collection('meta')
+
+ # @conn = new Mongo(:pool_size => 5, :timeout => 5)
+ # @db = conn.getDb dbname
+ # db.auth username, password
+ @dbname = dbname
+ @collection = collection
+ @conn = Mongo::Connection.new
+ @db = @conn[dbname]
+ @coll = @db[collection]
end
- def AccessDb.setup_collections
- @meta = db.collection('meta')
+ def upsert_by_id id, json
+ @coll.update({ :_id => id }, json, :upsert => true) # upsert!
end
- def AccessDb.insert meta
- # id = @meta.insert(meta)
- # TODO:
- # odczytanie z mety hasha i probowanie znalezienie w bazie tego elementu i dopiero wtedy dodanie poprzez uzycie upserta (3 arg true)
- @meta.update({ :hash => hash}, meta, true) # upsert!
- # @meta.update({ :_id => id }, '$set' => { db_id => id })
+ def upsert_by_meta json
+ puts json
+ @coll.update({ :hash => json[:hash] }, json, :upsert => true, :return_key => false)
end
- def AccessDb.update id, meta
- @meta.update({ :_id => id }, meta, true) # upsert!
+ # def insert json
+ # # id = @coll.insert(json)
+ # # TODO:
+ # # odczytanie z mety hasha i probowanie znalezienie w bazie tego elementu i dopiero wtedy dodanie poprzez uzycie upserta (3 arg true)
+ # # this = @coll.find({ :hash => json[:hash] })
+ # # update({ :hash => hash}, @collection, true) # upsert!
+ # update_by_id({ :hash => json[:hash]}, @collection, true) # upsert!
+ # # @coll.update({ :_id => id }, '$set' => { db_id => id })
+ # end
+ def delete id
+ @coll.remove({ :_id => id})
end
- def AccessDb.delete id
- @meta.remove({ :_id => id})
+ def add id, param, value
+ @coll.update({ :_id => id }, '$set' => { param => value })
+ # @coll.find_and_modify(:_ibd => id)
end
- def AccessDb.add id, param, value
- @meta.update({ :_id => id }, '$set' => { param => value })
- # @meta.find_and_modify(:_ibd => id)
+ # def fetch
+ # @coll.find_one
+ # end
+ def find json
+ @coll.find_one(:hash => json[:hash])
end
end
@@ -47,3 +61,32 @@ def AccessDb.add id, param, value
puts "There are #{@coll.count} records. Here they are:"
@coll.find.each { |doc| puts doc.inspect }
=end
+
+if __FILE__ == $0
+ require 'test/unit'
+ require 'active_support/core_ext/hash'
+
+ class AccessDbTest < Test::Unit::TestCase
+ def setup
+ @coll = AccessDb.new "meta","meta"
+ end
+ def teardown
+ end
+ def test_read
+ json = {:hash => "sara"}
+ @coll.upsert_by_meta json
+ find = @coll.find(json)
+ # puts find.except!("_id")
+ # puts find.convert_fields_for_query {:hash}
+ assert_equal(json[:hash], find.except!("_id")["hash"])
+ end
+
+ # def test_compute_hash_large_file
+ # assert_equal("61f7751fc2a72bfb", MovieHasher::compute_hash('dummy.bin'))
+ # end
+
+ # def test_jpeg
+ # assert_equal("bed5a0ecf41d0d96", MovieHasher::compute_hash('2638909.jpg')) # => bed5a0ecf41d0d96, zle 8228bb57b5b72510
+ # end
+ end
+end
View
@@ -14,30 +14,6 @@ load 'D:\Dropbox\#code\PracaInz-Program\AccessDb.rb'
require 'optparse'
require 'pp'
-require 'tk'
-require 'tkextlib/tile'
-
-root = TkRoot.new {title "Feet to Meters"}
-content = Tk::Tile::Frame.new(root) {padding "3 3 12 12"}.grid( :sticky => 'nsew')
-TkGrid.columnconfigure root, 0, :weight => 1; TkGrid.rowconfigure root, 0, :weight => 1
-
-$feet = TkVariable.new; $meters = TkVariable.new
-f = Tk::Tile::Entry.new(content) {width 7; textvariable $feet}.grid( :column => 2, :row => 1, :sticky => 'we' )
-Tk::Tile::Label.new(content) {textvariable $meters}.grid( :column => 2, :row => 2, :sticky => 'we');
-Tk::Tile::Button.new(content) {text 'Calculate'; command {calculate}}.grid( :column => 3, :row => 3, :sticky => 'w')
-
-Tk::Tile::Label.new(content) {text 'feet'}.grid( :column => 3, :row => 1, :sticky => 'w')
-Tk::Tile::Label.new(content) {text 'is equivalent to'}.grid( :column => 1, :row => 2, :sticky => 'e')
-Tk::Tile::Label.new(content) {text 'meters'}.grid( :column => 3, :row => 2, :sticky => 'w')
-
-TkWinfo.children(content).each {|w| TkGrid.configure w, :padx => 5, :pady => 5}
-f.focus
-root.bind("Return") {calculate}
-
-
-
-
-
STDOUT.sync = true; exit_requested = false; Kernel.trap( "INT" ) { exit_requested = true }
options = {}
@@ -96,7 +72,7 @@ if __FILE__ == $0; pp "Options:", options; pp "ARGV:", ARGV end
def initialize()
super(nil, -1, 'My Frame Title')
@my_panel = Panel.new(self)
- @my_label = StaticText.new(@my_panel, -1, 'My Label Text', DEFAULT_POSITION, DEFAULT_SIZE, ALIGN_CENTER)
+ @my_label = StaticText.new(@mby_panel, -1, 'My Label Text', DEFAULT_POSITION, DEFAULT_SIZE, ALIGN_CENTER)
@my_textbox = TextCtrl.new(@my_panel, -1, 'Default Textbox Value')
@my_combo = ComboBox.new(@my_panel, -1, 'Default Combo Text', DEFAULT_POSITION, DEFAULT_SIZE, ['Item 1', 'Item 2', 'Item 3'])
@my_button = Button.new(@my_panel, -1, 'My Button Text')
@@ -108,20 +84,17 @@ if __FILE__ == $0; pp "Options:", options; pp "ARGV:", ARGV end
exit!
=end
-
class Downloader
def initialize(directory)
@PASS=nil
@COOKIE=nil
@filename=nil
@full_file_location = nil
- @myjson=nil
@target_dir = directory
File.exists? @target_dir # File.directory? @target_dir
@c = Curl::Easy.new
curl_setup
- AccessDb.connect
- AccessDb.setup_collections
+ @mongo = AccessDb.new "meta","meta"
end
def curl_setup
@@ -148,22 +121,22 @@ class Downloader
def parse_link_info(url)
data, link, content, hash = {}, {}, {}, {}
- link["requested"] = url
+ link[:link_requested] = url
if @c.last_effective_url != url
- then link["final"] = @c.last_effective_url end
+ then link[:final] = @c.last_effective_url end
- link["requested-filename"] = @filename
+ link[:requested_filename] = @filename
@final_filename = @c.last_effective_url.split(/\?/).first.split(/\//).last
if @final_filename != @filename
- then link["final-filename"] = @final_filename end
+ then link[:final_filename] = @final_filename end
- data["Link"] = link
+ data[:Link] = link
- content["lenght"] = @c.downloaded_content_length
- content["type"] = @c.content_type
- data["Content"] = content
+ content[:"content-lenght"] = @c.downloaded_content_length
+ content[:type] = @c.content_type
+ data[:Content] = content
- data["filetime"] = Time.at(@c.file_time).utc.to_s
+ data[:filetime] = Time.at(@c.file_time).utc.to_s
@hash = MovieHasher::compute_hash(@save_location)
@hash = MovieHasher::compute_hash(@save_location)
@@ -177,10 +150,9 @@ class Downloader
data["Hashes"] = hash
@myjson = JSON.pretty_generate(data)
- result = AccessDb.insert @myjson
- print @myjson
- print result
-
+ # result = AccessDb.insert @myjson
+ # print @myjson
+ # print result
end
def add_link(single_url,cred=nil,ref=nil,cookie=nil)
@@ -210,9 +182,12 @@ class Downloader
# if File.file?(@save_location)
# then parse_additional_info @save_location end
# puts "#{@save_location} #{@hash}"
- parse_link_info single_url
- id = update_db
- File.open(@save_location + ":meta.json","w").write @myjson
+
+ json = parse_link_info single_url
+ puts json
+ id = @mongo.upsert_by_meta json
+ json["_id"] = id
+ File.open(@save_location + :"meta.json","w").write json
end
end
end
@@ -252,6 +227,4 @@ if __FILE__ == $0
'http://static3.blip.pl/user_generated/update_pictures/2638909.jpg'
]
manager.add_links(urls_to_download)
-end
-
-Tk.mainloop
+end
View
@@ -0,0 +1 @@
+start %CD%\bin\mongodb\bin\mongo meta
@@ -1 +1 @@
-start %CD%\PracaInz-Program\bin\mongodb\bin\mongod.exe --dbpath %CD%\PracaInz-Program\db
+start %CD%\bin\mongodb\bin\mongod.exe --dbpath %CD%\db

0 comments on commit 823b55c

Please sign in to comment.