Fixed importer to work with custom fields as the unique column.

Added lots more import samples.
Added reference to the documentation in progress.
Fixed the interface to correspond to the new features.
Did some preparatory work towards supporting importing
issue relationships.

README.rdoc

@@ -12,3 +12,5 @@ To install:
 
 en, zh, and ja localizations included.
 
+User documentation at https://github.com/leovitch/redmine_importer/wiki.
+

app/controllers/importer_controller.rb

@@ -1,6 +1,12 @@
 require 'fastercsv'
 require 'tempfile'
 
+class MultipleIssuesForUniqueValue < Exception
+end
+
+class NoIssueForUniqueValue < Exception
+end
+
 class ImporterController < ApplicationController
   unloadable
 
@@ -62,6 +68,35 @@ def match
     end
     @attrs.sort!
   end
+
+  def issue_for_unique_attr(unique_attr, attr_value)
+    if unique_attr == "id"
+      issues = [Issue.find_by_id(attr_value)]
+    else
+      query = Query.new(:name => "_importer", :project => @project)
+      query.add_filter("status_id", "*", [1])
+      query.add_filter(unique_attr, "=", [attr_value])
+      logger.info("Querying for issue with #{unique_attr} = '#{attr_value}'")
+
+      issues = Issue.find :all, :conditions => query.statement, :limit => 2, :include => [ :assigned_to, :status, :tracker, :project, :priority, :category, :fixed_version ]
+      logger.info("condition for parent query is #{query.statement}")
+    end
+
+    if issues.size > 1
+      issues.each do |pi|
+        logger.info("Found parent #{pi}")
+      end
+      flash[:warning] = "Unique field #{unique_attr}  with value '#{attr_value}' has duplicate record"
+      @failed_count += 1
+      @failed_issues[@handle_count + 1] = row
+      raise MultipleIssuesForUniqueValue, "Unique field #{unique_attr}  with value '#{attr_value}' has duplicate record"
+    else
+      if issues.size == 0
+        raise NoIssueForUniqueValue, "No issue with #{unique_attr} of '#{attr_value}' found"
+      end
+      issues.first
+    end
+  end
 
   def result
     @handle_count = 0
@@ -94,6 +129,7 @@ def result
     ignore_non_exist = params[:ignore_non_exist]
     fields_map = params[:fields_map]
     unique_attr = fields_map[unique_field]
+    unique_attr_checked = false  # Used to optimize some work that has to happen inside the loop   
 
     # attrs_map is fields_map's invert
     attrs_map = fields_map.invert
@@ -122,64 +158,52 @@ def result
       issue.tracker_id = tracker != nil ? tracker.id : default_tracker
       issue.author_id = author != nil ? author.id : User.current.id
 
-      if update_issue
-        # custom field
+      # reprocess unique_attr if it's a custom field -- only on the first issue
+      if !unique_attr_checked
         if !ISSUE_ATTRS.include?(unique_attr.to_sym)
           issue.available_custom_fields.each do |cf|
             if cf.name == unique_attr
               unique_attr = "cf_#{cf.id}"
               break
             end
-          end 
+          end
         end
-
-        if unique_attr == "id"
-          issues = [Issue.find_by_id(row[unique_field])]
-        else
-          query = Query.new(:name => "_importer", :project => @project)
-          query.add_filter("status_id", "*", [1])
-          query.add_filter(unique_attr, "=", [row[unique_field]])
+        unique_attr_checked = true
+      end
 
-          issues = Issue.find :all, :conditions => query.statement, :limit => 2, :include => [ :assigned_to, :status, :tracker, :project, :priority, :category, :fixed_version ]
-        end
-
-        if issues.size > 1
-          flash[:warning] = "Unique field #{unique_field} has duplicate record"
-          @failed_count += 1
-          @failed_issues[@handle_count + 1] = row
-          break
-        else
-          if issues.size > 0
-            # found issue
-            issue = issues.first
-
-            # ignore other project's issue or not
-            if issue.project_id != @project.id && !update_other_project
-              @skip_count += 1
-              next              
-            end
-
-            # ignore closed issue except reopen
-            if issue.status.is_closed?
-              if status == nil || status.is_closed?
-                @skip_count += 1
-                next
-              end
-            end
-
-            # init journal
-            note = row[journal_field] || ''
-            journal = issue.init_journal(author || User.current, 
-              note || '')
-
-            @update_count += 1
-          else
-            # ignore none exist issues
-            if ignore_non_exist
+      if update_issue
+        begin
+          issue = issue_for_unique_attr(unique_attr,row[unique_field])
+
+          # ignore other project's issue or not
+          if issue.project_id != @project.id && !update_other_project
+            @skip_count += 1
+            next
+          end
+
+          # ignore closed issue except reopen
+          if issue.status.is_closed?
+            if status == nil || status.is_closed?
               @skip_count += 1
               next
             end
           end
+
+          # init journal
+          note = row[journal_field] || ''
+          journal = issue.init_journal(author || User.current, 
+            note || '')
+
+          @update_count += 1
+
+        rescue NoIssueForUniqueValue
+          if ignore_non_exist
+            @skip_count += 1
+            next
+          end
+
+        rescue MultipleIssuesForUniqueValue
+          break
         end
       end
 
@@ -205,37 +229,18 @@ def result
       issue.done_ratio = row[attrs_map["done_ratio"]] || issue.done_ratio
       issue.estimated_hours = row[attrs_map["estimated_hours"]] || issue.estimated_hours
 
-      # parent issue
-      if row[attrs_map["parent_issue"]] != nil
-        if unique_attr == "id"
-          parent_issues = [Issue.find_by_id(row[unique_field])]
-        else
-          query = Query.new(:name => "_importer", :project => @project)
-          query.add_filter("status_id", "*", [1])
-          query.add_filter(unique_attr, "=", [row[attrs_map["parent_issue"]]])
-          logger.info("Querying for parent issue with #{unique_attr} = '#{row[unique_field]}")
-
-          parent_issues = Issue.find :all, :conditions => query.statement, :limit => 2, :include => [ :assigned_to, :status, :tracker, :project, :priority, :category, :fixed_version ]
+      # issue relations & parent issues
+      begin
+        if row[attrs_map["parent_issue"]] != nil
+          issue.parent_issue_id = issue_for_unique_attr(unique_attr,row[attrs_map["parent_issue"]]).id
         end
-
-        if parent_issues.size > 1
-          flash[:warning] = "Unique field #{unique_field} has duplicate record"
-          @failed_count += 1
-          @failed_issues[@handle_count + 1] = row
-          break
-        else
-          if parent_issues.size > 0
-            # found issue
-            issue.parent_issue_id = parent_issues.first.id
-
-          else
-            # ignore none exist issues
-            if ignore_non_exist
-              @skip_count += 1
-              next
-            end
-          end
+      rescue NoIssueForUniqueValue
+        if ignore_non_exist
+          @skip_count += 1
+          next
         end
+      rescue MultipleIssuesForUniqueValue
+        break
       end
 
       # custom fields

app/views/importer/match.html.erb

@@ -28,20 +28,19 @@
 		<label><%= l(:label_default_tracker) %>
 		<%= select_tag "default_tracker", options_from_collection_for_select(@project.trackers, 'id', 'name') %></label><br/>
 
+		<label><%= l(:label_unique_field) %>
+		<%= select_tag "unique_field", options_for_select(@headers) %></label><br/>
+
 		<label><%= check_box_tag "update_issue", true, false %> <%= l(:label_update_issue) %>
 		</label><br/>
 		<%= observe_field("update_issue", :function => <<END_OF_STRING
-document.getElementById("unique_field").disabled = !element.checked;
 document.getElementById("journal_field").disabled = !element.checked;
 document.getElementById("update_other_project").disabled = !element.checked;
 document.getElementById("ignore_non_exist").disabled = !element.checked;
 END_OF_STRING
 )
 		%>
 
-		&nbsp;&nbsp;&nbsp;&nbsp;<label><%= l(:label_unique_field) %>
-		<%= select_tag "unique_field", options_for_select(@headers), {:disabled => true} %></label><br/>
-
 		&nbsp;&nbsp;&nbsp;&nbsp;<label><%= l(:label_journal_field) %>
 		<%= select_tag "journal_field", "<option value=\"\">#{l(:option_ignore)}</option>" + options_for_select(@headers), {:disabled => true} %></label><br/>
 

config/locales/en.yml

@@ -10,14 +10,14 @@ en:
   label_load_rules: "Load saved rules"
   label_toplines: "Refer to top lines of {{value}}："
   label_match_columns: "Matching Columns"
-  label_match_select: "Select match field"
+  label_match_select: "Select field to set from each column:"
   label_import_rule: "Import rules"
   label_default_tracker: "Default tracker:"
-  label_update_issue: "Update exists issue"
-  label_journal_field: "Select field as journal:"
-  label_unique_field: "Select unique field for identify issue:"
-  label_update_other_project: "Allow update issues of other projects"
-  label_ignore_non_exist: "Ignore none exist issues"
+  label_update_issue: "Update existing issues"
+  label_journal_field: "Select column to use as note:"
+  label_unique_field: "Select unique-valued column for updating or importing relations:"
+  label_update_other_project: "Allow updating issues from other projects"
+  label_ignore_non_exist: "Ignore non-existant issues"
   label_rule_name: "Input rule name"
 
   label_import_result: "Import Result"

test/samples/AllStandardFields.csv

@@ -0,0 +1,2 @@
+"Subject","Description","Assignee","Target version","Author","Category","Priority","Tracker","Status","Start date","Due date","% done","Estimated time"
+"A full task","A lengthily described set of activities.","admin","The Target Version","admin","Default","High","Bug","In Progress",2011-05-01,2011-08-28,25,200

test/samples/CustomField.csv

@@ -0,0 +1,3 @@
+"Subject","Description","External id"
+"Important Task","A truly critical deed.","5643-4"
+"Less Important Task","Something that would be useful.","5644-6"

test/samples/CustomFieldUpdate.csv

@@ -0,0 +1,3 @@
+"Subject","Description","External id"
+"Important Task","A truly critical deed.","5643-4"
+"Less Important Task","Altering this task to make it even more useful.","5644-6"

test/samples/ParentTaskByCustomField.csv

@@ -0,0 +1,4 @@
+"Subject","Description","External id","Parent task"
+"The overall task","A truly critical deed.","7643-4",
+"The first sub-task","Something that would be useful.","7644-6","7643-4"
+"The second sub-task","A really important component","7644-7","7643-4"