Support creating all pages

groonga · Jul 17, 2014 · 0508bb6 · 0508bb6
1 parent 564e3d4
commit 0508bb6
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 6 deletions.
diff --git a/lib/wikipedia-search/path.rb b/lib/wikipedia-search/path.rb
@@ -93,6 +93,10 @@ def pages
       data_dir + "#{@language}-pages.grn"
     end
 
+    def all_pages
+      data_dir + "#{@language}-all-pages.grn"
+    end
+
     def database_dir
       data_dir + "db"
     end
@@ -128,6 +132,10 @@ def pages
       data_dir + "#{@language}-pages.jsons"
     end
 
+    def all_pages
+      data_dir + "#{@language}-all-pages.jsons"
+    end
+
     def schema
       data_dir + "schema.json"
     end

diff --git a/lib/wikipedia-search/task.rb b/lib/wikipedia-search/task.rb
@@ -69,13 +69,15 @@ def define_data_convert_tasks
 
     def define_data_convert_groonga_tasks
       namespace :groonga do
+        base_command_line = [
+          "bzcat",
+          Shellwords.escape(@path.wikipedia.pages.to_s),
+          "|",
+          RbConfig.ruby,
+          "bin/wikipedia-to-groonga.rb",
+        ]
         file @path.groonga.pages.to_s => @path.wikipedia.pages.to_s do
-          command_line = []
-          command_line << "bzcat"
-          command_line << Shellwords.escape(@path.wikipedia.pages.to_s)
-          command_line << "|"
-          command_line << RbConfig.ruby
-          command_line << "bin/wikipedia-to-groonga.rb"
+          command_line = base_command_line.dup
           command_line << "--max-n-records"
           command_line << "5000"
           command_line << "--max-n-characters"
@@ -85,6 +87,13 @@ def define_data_convert_groonga_tasks
           sh(command_line.join(" "))
         end
 
+        file @path.groonga.all_pages.to_s => @path.wikipedia.pages.to_s do
+          command_line = base_command_line.dup
+          command_line << "--output"
+          command_line << @path.groonga.all_pages.to_s
+          sh(command_line.join(" "))
+        end
+
         desc "Convert Japanese Wikipedia page data to Groonga page data."
         task :ja => @path.groonga.pages.to_s
       end
@@ -113,6 +122,12 @@ def define_data_convert_droonga_tasks
                @path.groonga.pages.to_s)
           end
 
+          file @path.droonga.all_pages.to_s => @path.groonga.all_pages.to_s do
+            sh("grn2drn",
+               "--output", @path.droonga.all_pages.to_s,
+               @path.groonga.all_pages.to_s)
+          end
+
           desc "Convert Japanese Wikipedia page data to Droonga page data."
           task :ja => @path.droonga.pages.to_s
         end