added yields to the loader class to pass statuses to the calling class

History.txt

@@ -1,3 +1,8 @@
+== 1.2.2 2011-03-07
+
+* 1 enhancement:
+  * added yields to the loader class to pass statuses to the calling class.
+
 == 1.2.1 2010-12-21
 
 * 1 enhancement:

lib/ofac/models/ofac_sdn_loader.rb

@@ -9,12 +9,14 @@ class OfacSdnLoader
   def self.load_current_sdn_file
     puts "Reloading OFAC sdn data"
     puts "Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn"
+    yield "Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn" if block_given?
     #get the 3 data files
-    sdn = Tempfile.new('sdn')
+    sdn   = Tempfile.new('sdn')
     bytes = sdn.write(Net::HTTP.get(URI.parse('http://www.treasury.gov/ofac/downloads/sdn.pip')))
     sdn.rewind
     if bytes == 0 || convert_line_to_array(sdn.readline).size != 12
       puts "Trouble downloading file.  The url may have changed."
+      yield "Trouble downloading file.  The url may have changed." if block_given?
       return
     else
       sdn.rewind
@@ -25,9 +27,10 @@ def self.load_current_sdn_file
     alt = Tempfile.new('sdn')
     alt.write(Net::HTTP.get(URI.parse('http://www.treasury.gov/ofac/downloads/alt.pip')))
     alt.rewind
-    
+
     if OfacSdn.connection.kind_of?(ActiveRecord::ConnectionAdapters::MysqlAdapter)
       puts "Converting file to csv format for Mysql import.  This could take several minutes."
+      yield "Converting file to csv format for Mysql import.  This could take several minutes." if block_given?
 
       csv_file = convert_to_flattened_csv(sdn, address, alt)
 
@@ -47,9 +50,9 @@ def self.load_current_sdn_file
   #convert the file's null value to an empty string
   #and removes " chars.
   def self.clean_file_string(line)
-    line.gsub!(/-0-(\s)?/,'')
-    line.gsub!(/\n/,'')
-    line.gsub(/\"/,'')
+    line.gsub!(/-0-(\s)?/, '')
+    line.gsub!(/\n/, '')
+    line.gsub(/\"/, '')
   end
 
   #split the line into an array
@@ -61,7 +64,7 @@ def self.convert_line_to_array(line)
   #1 array of address records and one array of alt records
   def self.foreign_key_records(sdn_id)
     address_records = []
-    alt_records = []
+    alt_records     = []
 
     #the first element in each array is the primary and foreign keys
     #we are denormalizing the data
@@ -94,29 +97,29 @@ def self.foreign_key_records(sdn_id)
   def self.sdn_text_to_hash(line)
     unless line.nil?
       value_array = convert_line_to_array(line)
-      {:id => value_array[0],
-        :name => value_array[1],
-        :sdn_type => value_array[2],
-        :program => value_array[3],
-        :title => value_array[4],
-        :vessel_call_sign => value_array[5],
-        :vessel_type => value_array[6],
-        :vessel_tonnage => value_array[7],
-        :gross_registered_tonnage => value_array[8],
-        :vessel_flag => value_array[9],
-        :vessel_owner => value_array[10],
-        :remarks => value_array[11]
+      {:id                       => value_array[0],
+       :name                     => value_array[1],
+       :sdn_type                 => value_array[2],
+       :program                  => value_array[3],
+       :title                    => value_array[4],
+       :vessel_call_sign         => value_array[5],
+       :vessel_type              => value_array[6],
+       :vessel_tonnage           => value_array[7],
+       :gross_registered_tonnage => value_array[8],
+       :vessel_flag              => value_array[9],
+       :vessel_owner             => value_array[10],
+       :remarks                  => value_array[11]
       }
     end
   end
 
   def self.address_text_to_hash(line)
     unless line.nil?
       value_array = convert_line_to_array(line)
-      {:id => value_array[0],
-       :address => value_array[2],
-       :city => value_array[3],
-       :country => value_array[4],
+      {:id              => value_array[0],
+       :address         => value_array[2],
+       :city            => value_array[3],
+       :country         => value_array[4],
        :address_remarks => value_array[5]
       }
     end
@@ -125,9 +128,9 @@ def self.address_text_to_hash(line)
   def self.alt_text_to_hash(line)
     unless line.nil?
       value_array = convert_line_to_array(line)
-      {:id => value_array[0],
-       :alternate_identity_type => value_array[2],
-       :alternate_identity_name => value_array[3],
+      {:id                         => value_array[0],
+       :alternate_identity_type    => value_array[2],
+       :alternate_identity_name    => value_array[3],
        :alternate_identity_remarks => value_array[4]
       }
     end
@@ -136,68 +139,68 @@ def self.alt_text_to_hash(line)
   def self.convert_hash_to_mysql_import_string(record_hash)
     # empty field for id to be generated by mysql.
     new_line = "``|" +
-      #    :name
+        #    :name
     "`#{record_hash[:name]}`|" +
-      #    :sdn_type
+        #    :sdn_type
     "`#{record_hash[:sdn_type]}`|" +
-      #    :program
+        #    :program
     "`#{record_hash[:program]}`|" +
-      #    :title
+        #    :title
     "`#{record_hash[:title]}`|" +
-      #    :vessel_call_sign
+        #    :vessel_call_sign
     "`#{record_hash[:vessel_call_sign]}`|" +
-      #    :vessel_type
+        #    :vessel_type
     "`#{record_hash[:vessel_type]}`|" +
-      #    :vessel_tonnage
+        #    :vessel_tonnage
     "`#{record_hash[:vessel_tonnage]}`|" +
-      #    :gross_registered_tonnage
+        #    :gross_registered_tonnage
     "`#{record_hash[:gross_registered_tonnage]}`|" +
-      #    :vessel_flag
+        #    :vessel_flag
     "`#{record_hash[:vessel_flag]}`|" +
-      #    :vessel_owner
+        #    :vessel_owner
     "`#{record_hash[:vessel_owner]}`|" +
-      #    :remarks
+        #    :remarks
     "`#{record_hash[:remarks]}`|" +
-    #    :address
+        #    :address
     "`#{record_hash[:address]}`|" +
-    #    :city
+        #    :city
     "`#{record_hash[:city]}`|" +
-    #    :country
+        #    :country
     "`#{record_hash[:country]}`|" +
-    #    :address_remarks
+        #    :address_remarks
     "`#{record_hash[:address_remarks]}`|" +
-    #    :alternate_identity_type
+        #    :alternate_identity_type
     "`#{record_hash[:alternate_identity_type]}`|" +
-    #    :alternate_identity_name
+        #    :alternate_identity_name
     "`#{record_hash[:alternate_identity_name]}`|" +
-    #    :alternate_identity_remarks
+        #    :alternate_identity_remarks
     "`#{record_hash[:alternate_identity_remarks]}`|" +
-    #:created_at
+        #:created_at
     "`#{Time.now.to_s(:db)}`|" +
-      # updated_at
+        # updated_at
     "`#{Time.now.to_s(:db)}`" + "\n"
 
     new_line
   end
 
   def self.convert_to_flattened_csv(sdn_file, address_file, alt_file)
-    @address = address_file
-    @alt = alt_file
+    @address              = address_file
+    @alt                  = alt_file
 
-    csv_file = Tempfile.new("ofac")    # create temp file for converted csv format.
+    csv_file              = Tempfile.new("ofac") # create temp file for converted csv format.
     #get the first line from the address and alt files
     @current_address_hash = address_text_to_hash(@address.gets)
-    @current_alt_hash = alt_text_to_hash(@alt.gets)
+    @current_alt_hash     = alt_text_to_hash(@alt.gets)
+
+    start                 = Time.now
 
-    start = Time.now
-
     sdn_file.each_with_index do |line, i|
-      
+
       #initialize the address and alt atributes to empty strings
       address_attributes = address_text_to_hash("|||||")
-      alt_attributes = alt_text_to_hash("||||")
+      alt_attributes     = alt_text_to_hash("||||")
 
-      sdn_attributes = sdn_text_to_hash(line)
+      sdn_attributes     = sdn_text_to_hash(line)
 
       #get the foreign key records for this sdn
       address_records, alt_records = foreign_key_records(sdn_attributes[:id])
@@ -224,31 +227,35 @@ def self.convert_to_flattened_csv(sdn_file, address_file, alt_file)
           end
         end
       end
-      puts "#{i} records processed." if (i % 1000 == 0) && (i > 0)
+      if (i % 1000 == 0) && (i > 0)
+        puts "#{i} records processed."
+        yield "#{i} records processed." if block_given?
+      end
     end
     puts "File conversion ran for #{(Time.now - start) / 60} minutes."
+    yield "File conversion ran for #{(Time.now - start) / 60} minutes." if block_given?
     return csv_file
   end
 
   def self.active_record_file_load(sdn_file, address_file, alt_file)
     @address = address_file
-    @alt = alt_file
+    @alt     = alt_file
 
     #OFAC data is a complete list, so we have to dump and load
     OfacSdn.delete_all
 
     #get the first line from the address and alt files
     @current_address_hash = address_text_to_hash(@address.gets)
-    @current_alt_hash = alt_text_to_hash(@alt.gets)
-    attributes = {}
+    @current_alt_hash     = alt_text_to_hash(@alt.gets)
+    attributes            = {}
     sdn_file.each_with_index do |line, i|
 
       #initialize the address and alt atributes to empty strings
       address_attributes = address_text_to_hash("|||||")
-      alt_attributes = alt_text_to_hash("||||")
+      alt_attributes     = alt_text_to_hash("||||")
+
+      sdn_attributes     = sdn_text_to_hash(line)
 
-      sdn_attributes = sdn_text_to_hash(line)
-
       #get the foreign key records for this sdn
       address_records, alt_records = foreign_key_records(sdn_attributes[:id])
 
@@ -282,8 +289,10 @@ def self.active_record_file_load(sdn_file, address_file, alt_file)
           end
         end
       end
-
-      puts "#{i} records processed." if (i % 5000 == 0) && (i > 0)
+      if (i % 5000 == 0) && (i > 0)
+        puts "#{i} records processed."
+        yield "#{i} records processed." if block_given?
+      end
     end
   end
 
@@ -293,18 +302,21 @@ def self.active_record_file_load(sdn_file, address_file, alt_file)
   # see http://dev.mysql.com/doc/refman/5.1/en/load-data.html
   def self.bulk_mysql_update(csv_file)
     puts "Deleting all records in ofac_sdn..."
+    yield "Deleting all records in ofac_sdn..." if block_given?
 
     #OFAC data is a complete list, so we have to dump and load
     OfacSdn.delete_all
 
     puts "Importing into Mysql..."
-
+    yield "Importing into Mysql..." if block_given?
+
     mysql_command = <<-TEXT
     LOAD DATA LOCAL INFILE '#{csv_file.path}' REPLACE INTO TABLE ofac_sdns FIELDS TERMINATED BY '|' ENCLOSED BY "`" LINES TERMINATED BY '\n';
     TEXT
 
     OfacSdn.connection.execute(mysql_command)
     puts "Mysql import complete."
+    yield "Mysql import complete." if block_given?
 
   end
 

rdoc/classes/OfacSdnLoader.html

@@ -113,7 +113,7 @@ <h3 class="section-bar">Public Class methods</h3>
 
         <div class="method-heading">
           <a href="#M000009" class="method-signature">
-          <span class="method-name">load_current_sdn_file</span><span class="method-args">()</span>
+          <span class="method-name">load_current_sdn_file</span><span class="method-args">() {|&quot;Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn&quot; if block_given?| ...}</span>
           </a>
         </div>
 
@@ -130,12 +130,14 @@ <h3 class="section-bar">Public Class methods</h3>
   <span class="ruby-keyword kw">def</span> <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">load_current_sdn_file</span>
     <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;Reloading OFAC sdn data&quot;</span>
     <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn&quot;</span>
+    <span class="ruby-keyword kw">yield</span> <span class="ruby-value str">&quot;Downloading OFAC data from http://www.treas.gov/offices/enforcement/ofac/sdn&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block_given?</span>
     <span class="ruby-comment cmt">#get the 3 data files</span>
-    <span class="ruby-identifier">sdn</span> = <span class="ruby-constant">Tempfile</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'sdn'</span>)
+    <span class="ruby-identifier">sdn</span>   = <span class="ruby-constant">Tempfile</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'sdn'</span>)
     <span class="ruby-identifier">bytes</span> = <span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">write</span>(<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">get</span>(<span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-value str">'http://www.treasury.gov/ofac/downloads/sdn.pip'</span>)))
     <span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">rewind</span>
     <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">bytes</span> <span class="ruby-operator">==</span> <span class="ruby-value">0</span> <span class="ruby-operator">||</span> <span class="ruby-identifier">convert_line_to_array</span>(<span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">readline</span>).<span class="ruby-identifier">size</span> <span class="ruby-operator">!=</span> <span class="ruby-value">12</span>
       <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;Trouble downloading file.  The url may have changed.&quot;</span>
+      <span class="ruby-keyword kw">yield</span> <span class="ruby-value str">&quot;Trouble downloading file.  The url may have changed.&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block_given?</span>
       <span class="ruby-keyword kw">return</span>
     <span class="ruby-keyword kw">else</span>
       <span class="ruby-identifier">sdn</span>.<span class="ruby-identifier">rewind</span>
@@ -146,9 +148,10 @@ <h3 class="section-bar">Public Class methods</h3>
     <span class="ruby-identifier">alt</span> = <span class="ruby-constant">Tempfile</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'sdn'</span>)
     <span class="ruby-identifier">alt</span>.<span class="ruby-identifier">write</span>(<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">get</span>(<span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-value str">'http://www.treasury.gov/ofac/downloads/alt.pip'</span>)))
     <span class="ruby-identifier">alt</span>.<span class="ruby-identifier">rewind</span>
-    
+
     <span class="ruby-keyword kw">if</span> <span class="ruby-constant">OfacSdn</span>.<span class="ruby-identifier">connection</span>.<span class="ruby-identifier">kind_of?</span>(<span class="ruby-constant">ActiveRecord</span><span class="ruby-operator">::</span><span class="ruby-constant">ConnectionAdapters</span><span class="ruby-operator">::</span><span class="ruby-constant">MysqlAdapter</span>)
       <span class="ruby-identifier">puts</span> <span class="ruby-value str">&quot;Converting file to csv format for Mysql import.  This could take several minutes.&quot;</span>
+      <span class="ruby-keyword kw">yield</span> <span class="ruby-value str">&quot;Converting file to csv format for Mysql import.  This could take several minutes.&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">block_given?</span>
 
       <span class="ruby-identifier">csv_file</span> = <span class="ruby-identifier">convert_to_flattened_csv</span>(<span class="ruby-identifier">sdn</span>, <span class="ruby-identifier">address</span>, <span class="ruby-identifier">alt</span>)
 

rdoc/created.rid

@@ -1 +1 @@
-Tue, 21 Dec 2010 12:36:28 -0500
+Mon, 07 Mar 2011 14:20:44 -0500

rdoc/files/lib/ofac/models/ofac_sdn_loader_rb.html

@@ -56,7 +56,7 @@ <h1>ofac_sdn_loader.rb</h1>
     </tr>
     <tr class="top-aligned-row">
       <td><strong>Last Update:</strong></td>
-      <td>Tue Dec 21 12:34:18 -0500 2010</td>
+      <td>Mon Mar 07 14:17:12 -0500 2011</td>
     </tr>
     </table>
   </div>

rdoc/index.html

@@ -5,12 +5,12 @@
 
 <!--
 
-    ofac 1.2.1
+    ofac 1.2.2
 
   -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 <head>
-  <title>ofac 1.2.1</title>
+  <title>ofac 1.2.2</title>
   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
 </head>
 <frameset rows="20%, 80%">