Merge branch 'develop'

HISTORY.ja.rdoc

@@ -1,3 +1,12 @@
+=== 1.0.2 2013/06/08
+
+* gmilk
+  * ファイル内検索を外部ツールと連携して高速化する機能を追加
+  * -m(--match-files) オプションを追加
+  * -e (-external-tool) オプションを追加
+  * 検索するレコード数が多い時は"-e grep"で検索するように
+  * --no-external オプションを追加
+
 === 1.0.0 2013/05/21
 
 * milk web

HISTORY.rdoc

@@ -1,3 +1,12 @@
+=== 1.0.2 2013/06/08
+
+* gmilk
+  * Add feature to accelerate search speed with an external tool file search
+  * Add -m(--match-files) option
+  * Add -e (-external-tool) option
+  * Search using "-e grep" when the number of records is large
+  * Add --no-external option
+
 === 1.0.0 2013/05/21
 
 * milk web

VERSION

@@ -1 +1 @@
-1.0.0
+1.0.2

bin/gmilk

@@ -6,5 +6,5 @@
 require 'rubygems'
 require 'milkode/grep/cli_grep'
 
-Version = "1.0.0"
+Version = "1.0.2"
 Milkode::CLI_Grep.execute(STDOUT, ARGV)

bin/milk

@@ -6,5 +6,5 @@
 require 'rubygems'
 require 'milkode/cli'
 
-Version = "1.0.0"
+Version = "1.0.2"
 Milkode::CLI.start(ARGV)

lib/milkode/cdweb/app.rb

@@ -29,7 +29,7 @@
 get '/' do
   if Database.validate?
     @setting = WebSetting.new
-    @version = "1.0.0"
+    @version = "1.0.2"
 
     @package_num = Database.instance.yaml_package_num
     @file_num = Database.instance.totalRecords

lib/milkode/common/util.rb

@@ -4,6 +4,7 @@
 require 'fileutils'
 require 'pathname'
 require 'kconv'
+require 'open3'
 
 module Milkode
   module Util
@@ -230,6 +231,15 @@ def groonga_table_sort(table, keys, options = {})
         table.sort(keys, options)
       end
     end
+
+    # 指定したコマンドが存在するか？
+    def exist_command?(command)
+      begin
+        Open3.capture3("type #{command}")[2].exited?
+      rescue Errno::ENOENT
+        false
+      end
+    end
   end
 end
 

lib/milkode/grep/cli_grep.rb

@@ -7,9 +7,12 @@
 require 'milkode/common/util'
 require 'milkode/grep/findgrep_option'
 require 'optparse'
+require 'tempfile'
 
 module Milkode
   class CLI_Grep
+    AUTO_EXTERNAL_RECORD_NUM = 500
+
     def self.execute(stdout, arguments=[])
       # 引数の文字コードをUTF-8に変換
       if (Util::platform_win?)
@@ -63,15 +66,18 @@ def self.execute(stdout, arguments=[])
 Normal:
 EOF
       opt.on('-a', '--all', 'Search all package.') {|v| my_option[:all] = true }
-      opt.on('-c', '--count', 'Disp count num.') {|v| my_option[:count] = true }
+      opt.on('-c', '--count', 'Display count num.') {|v| my_option[:count] = true }
       opt.on('--cache', 'Search only db.') {|v| option.groongaOnly = true }
       opt.on('--color', 'Color highlight.') {|v| option.colorHighlight = true}
       opt.on('--cs', '--case-sensitive', 'Case sensitivity.') {|v| option.caseSensitive = true }
       opt.on('-d DIR', '--directory DIR', 'Start directory. (deafult:".")') {|v| current_dir = File.expand_path(v); my_option[:find_mode] = true} 
       opt.on('--db DB_DIR', "Specify dbdir. (Use often with '-a')") {|v| option.dbFile = Dbdir.groonga_path(v) }
+      opt.on('-e GREP', '--external-tool GREP', "Use external tool for file search. (e.g. grep, ag)") {|v| my_option[:external_tool] = v}
       opt.on('-f FILE_PATH', '--file-path FILE_PATH', 'File path. (Enable multiple call)') {|v| option.filePatterns << v; my_option[:find_mode] = true }
       opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
+      opt.on('-m', '--match-files', 'Display match files.') {|v| my_option[:match_files] = true}
       opt.on('-n NUM', 'Limits the number of match to show.') {|v| option.matchCountLimit = v.to_i }
+      opt.on('--no-external', 'Disable auto external.') {|v| my_option[:no_external] = true }
       opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
       opt.on('-p PACKAGE', '--package PACKAGE', 'Specify search package.') {|v| setup_package(option, my_option, v) }
       opt.on('-r', '--root', 'Search from package root.') {|v| current_dir = package_root_dir(File.expand_path(".")); my_option[:find_mode] = true }
@@ -150,12 +156,31 @@ def self.execute(stdout, arguments=[])
           require 'milkode/grep/findgrep'
 
           if (my_option[:count])
-            # count mode
             option.isSilent = true
-            findGrep = FindGrep.new(arguments, option)
-            records = findGrep.pickupRecords
-            # stdout.puts "#{records.size} records (#{findGrep.time_s})"
-            stdout.puts "#{records.size} records"
+            stdout.puts "#{pickup_records(arguments, option).size} records"
+
+          elsif (my_option[:external_tool])
+            option.isSilent = true
+            records = pickup_records(arguments, option)
+
+            case my_option[:external_tool]
+            when 'grep'
+              search_external_tool(arguments, option, records, 'grep -n', 'grep')              
+            when 'ag'
+              search_external_tool(arguments, option, records, 'ag', 'ag')
+            else
+              search_external_tool(arguments, option, records, my_option[:external_tool], my_option[:external_tool])
+            end
+
+          elsif (my_option[:match_files])
+            option.isSilent = true
+            records = pickup_records(arguments, option)
+            files   = pickup_files(records, '\ ', option.matchCountLimit)
+
+            files.each do |filename|
+              stdout.puts filename
+            end
+
           elsif my_option[:gotoline_data]
             # gotoline mode
             basePatterns = option.filePatterns 
@@ -176,9 +201,22 @@ def self.execute(stdout, arguments=[])
               findGrep.searchAndPrint(stdout)
             end
           else
-            # search mode
+            # normal search
             findGrep = FindGrep.new(arguments, option)
-            findGrep.searchAndPrint(stdout)
+            records  = findGrep.pickupRecords
+
+            if (my_option[:no_external] || records.length < AUTO_EXTERNAL_RECORD_NUM)
+              findGrep.searchAndPrint2(stdout, records)
+            else
+              # レコード数が多い時は"-e grep"で検索
+              if Util::exist_command?('grep') && Util::exist_command?('xargs')
+                $stderr.puts "Number of records is large. Use auto external tool (gmilk -e grep)"
+                search_external_tool(arguments, option, records, 'grep -n', 'grep')
+              else
+                findGrep.searchAndPrint2(stdout, records)
+              end
+            end
+
           end
         end
       else
@@ -188,6 +226,43 @@ def self.execute(stdout, arguments=[])
 
     private
 
+    def self.search_external_tool(arguments, option, records, first_command, second_command)
+      files   = pickup_files(records, '\\\\ ', option.matchCountLimit)
+
+      unless files.empty?
+        cmd = []
+
+        tmpfile = Tempfile.open("gmilk_external_tool")
+        tmpfile.binmode
+        tmpfile.write(files.join("\n"))
+        tmpfile.close
+        tmpfile.open
+        cmd << "cat #{tmpfile.path}"
+
+        cmd << "xargs #{first_command} #{arguments[0]}"
+
+        (1...arguments.size).each do |index|
+          cmd << "#{second_command} #{arguments[index]}"
+        end
+
+        system(cmd.join(" | "))
+
+        tmpfile.close(true)
+      end
+    end
+
+    def self.pickup_records(arguments, option)
+      FindGrep.new(arguments, option).pickupRecords
+    end
+
+    def self.pickup_files(records, conv_space, length = -1)
+      files = []
+      records.each do |r|
+        files << r.path.gsub(' ', conv_space) if File.exist?(r.path)
+      end
+      (length > 0) ? files[0, length] : files
+    end
+
     def self.setup_package(option, my_option, keyword)
       # @memo package指定が簡単になった
       # dirs = yaml_load.contents.find_all {|p| p.name.include? keyword }.map{|p| p.directory}

lib/milkode/grep/findgrep.rb

@@ -109,10 +109,15 @@ def strs2regs_simple(strs)
     end
 
     def searchAndPrint(stdout)
+      records = searchDatabase
+      searchAndPrint2(stdout, records)
+    end
+
+    def searchAndPrint2(stdout, records)
       unless (@option.dbFile)
         searchFromDir(stdout, @option.directory, 0)
       else
-        searchFromDB(stdout, @option.directory)
+        searchFromDB(stdout, records, @option.directory)
       end
 
       @result.time_stop
@@ -153,10 +158,7 @@ def time_s
       Gren::Util::time_s(@result.time)
     end
 
-    def searchFromDB(stdout, dir)
-      # データベースを検索
-      records = searchDatabase
-
+    def searchFromDB(stdout, records, dir)
       # ヒットしたレコード数
       stdout.puts "Found   : #{records.size} records." if (!@option.dispHtml && !@option.isSilent)
 

milkode.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{milkode}
-  s.version = "1.0.0"
+  s.version = "1.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["ongaeshi"]
-  s.date = %q{2013-05-21}
+  s.date = %q{2013-06-08}
   s.description = %q{Line based local source code search engine & grep-command & web-app.}
   s.email = %q{ongaeshi0621@gmail.com}
   s.executables = ["gmilk", "milk"]

test/test_cli_grep.rb

@@ -26,6 +26,7 @@ def test_main
     t_case_sensitive
     t_keyword
     t_db
+    t_match_files
   end
 
   def teardown
@@ -98,6 +99,12 @@ def t_db
     io = StringIO.new
     CLI_Grep.execute(io, "--db #{@work.path("db1")} -a db_dir_expand".split)
   end
+
+  def t_match_files
+    io = StringIO.new
+    CLI_Grep.execute(io, "-m -p a_project default".split)
+    assert_equal 1, io.string.split("\n").count
+  end
 end