logical_table_remove: start implementing "--force yes"

It's not completed yet.

plugins/sharding/logical_table_remove.rb

@@ -10,10 +10,12 @@ class LogicalTableRemoveCommand < Command
                  "max",
                  "max_border",
                  "dependent",
+                 "force",
                ])
 
       def run_body(input)
         @dependent = (input[:dependent] == "yes")
+        @force = (input[:force] == "yes")
 
         enumerator = LogicalEnumerator.new("logical_table_remove", input)
 
@@ -37,38 +39,52 @@ def remove_shard(shard, shard_range, target_range)
         end
         table = shard.table
 
+        if cover_type == :all or (table.nil? and @force)
+          remove_table(shard, table)
+          return
+        end
+
         expression_builder = RangeExpressionBuilder.new(shard_key,
                                                         target_range,
                                                         nil)
-
         case cover_type
-        when :all
-          remove_table(shard)
         when :partial_min
           remove_records(table) do |expression|
             expression_builder.build_partial_min(expression)
           end
-          remove_table(shard) if table.empty?
+          remove_table(shard, table) if table.empty?
         when :partial_max
           remove_records(table) do |expression|
             expression_builder.build_partial_max(expression)
           end
-          remove_table(shard) if table.empty?
+          remove_table(shard, table) if table.empty?
         when :partial_min_and_max
           remove_records(table) do |expression|
             expression_builder.build_partial_min_and_max(expression)
           end
-          remove_table(shard) if table.empty?
+          remove_table(shard, table) if table.empty?
         end
       end
 
-      def remove_table(shard)
-        table = shard.table
+      def remove_table(shard, table)
+        if table.nil? and @force
+          Context.instance.clear_error
+        end
+
         shard_suffix = shard.range_data.to_suffix
 
         referenced_table_ids = []
         if @dependent
-          table.columns.each do |column|
+          if table
+            columns = table.columns
+          else
+            prefix = "#{shard.table_name}."
+            column_names = context.database.each_name(:prefix => prefix)
+            columns = column_names.collect do |column_name|
+              context[column_name]
+            end
+          end
+          columns.each do |column|
             range = column.range
             if range.is_a?(Table)
               referenced_table_ids << range.id
@@ -82,9 +98,27 @@ def remove_table(shard)
           end
         end
 
-        table.remove(:dependent => @dependent)
+        if table.nil?
+          if @force
+            remove_table_force(shard)
+          else
+            message = "[logical_table_remove] table is broken: " +
+                      "<#{shard.table_name}>"
+            raise InvalidArgument, message
+          end
+        else
+          options = {:dependent => @dependent}
+          if @force
+            begin
+              table.remove(options)
+            rescue
+              remove_table_force(shard)
+            end
+          else
+            table.remove(options)
+          end
+        end
 
-        context = Groonga::Context.instance
         referenced_table_ids.each do |referenced_table_id|
           referenced_table = context[referenced_table_id]
           next if referenced_table.nil?
@@ -94,6 +128,25 @@ def remove_table(shard)
         end
       end
 
+      def remove_table_force(shard)
+        database = context.database
+
+        prefix = "#{shard.table_name}."
+        database.each_raw(:prefix => prefix) do |id, cursor|
+          column = context[id]
+          if column.nil?
+            column_name = cursor.key
+            Object.remove_force(column_name)
+          else
+            column.remove(:dependent => @dependent)
+          end
+        end
+
+        # TODO: Remove objects that refers the table
+
+        Object.remove_force(shard.table_name)
+      end
+
       def remove_records(table)
         expression = nil
 

test/command/suite/sharding/logical_table_remove/broken/table/default.expected

@@ -0,0 +1,107 @@
+plugin_register sharding
+[[0,0.0,0.0],true]
+table_create Paths_20150203 TABLE_HASH_KEY|KEY_LARGE ShortText
+[[0,0.0,0.0],true]
+table_create Logs_20150203 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150203 path COLUMN_SCALAR Paths_20150203
+[[0,0.0,0.0],true]
+table_create Times_20150203 TABLE_PAT_KEY Time
+[[0,0.0,0.0],true]
+column_create Times_20150203 timestamp_index COLUMN_INDEX Logs_20150203 timestamp
+[[0,0.0,0.0],true]
+table_create PathTerms_20150203 TABLE_PAT_KEY ShortText   --default_tokenizer TokenBigram   --normalizer NormalizerAuto
+[[0,0.0,0.0],true]
+column_create PathTerms_20150203 index COLUMN_INDEX|WITH_POSITION   Paths_20150203 _key
+[[0,0.0,0.0],true]
+table_create NotRelated_20150204 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+table_create Paths_20150204 TABLE_HASH_KEY|KEY_LARGE ShortText
+[[0,0.0,0.0],true]
+table_create Logs_20150204 TABLE_NO_KEY
+[[0,0.0,0.0],true]
+column_create Logs_20150204 timestamp COLUMN_SCALAR Time
+[[0,0.0,0.0],true]
+column_create Logs_20150204 path COLUMN_SCALAR Paths_20150204
+[[0,0.0,0.0],true]
+table_create Times_20150204 TABLE_PAT_KEY Time
+[[0,0.0,0.0],true]
+column_create Times_20150204 timestamp_index COLUMN_INDEX Logs_20150204 timestamp
+[[0,0.0,0.0],true]
+table_create PathTerms_20150204 TABLE_PAT_KEY ShortText   --default_tokenizer TokenBigram   --normalizer NormalizerAuto
+[[0,0.0,0.0],true]
+column_create PathTerms_20150204 index COLUMN_INDEX|WITH_POSITION   Paths_20150204 _key
+[[0,0.0,0.0],true]
+load --table Logs_20150203
+[
+{"timestamp": "2015-02-03 12:49:00", "path": "/tmp/log"},
+{"timestamp": "2015-02-03 23:59:59", "path": "/var/"}
+]
+[[0,0.0,0.0],2]
+load --table Logs_20150204
+[
+{"timestamp": "2015-02-04 00:00:00", "path": "/tmp/log"},
+{"timestamp": "2015-02-04 13:49:00", "path": "/root/"},
+{"timestamp": "2015-02-04 13:50:00", "path": "/etc/aliases"}
+]
+[[0,0.0,0.0],3]
+logical_table_remove Logs timestamp   --min "2015-02-04 00:00:00"   --min_border "include"
+[[[-22,0.0,0.0],"[logical_table_remove] table is broken: <Logs_20150204>"]]
+#|e| [io][open] file size is too small: <8>(required: >= 64): <db/db.0000110>
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+dump
+plugin_register sharding
+
+table_create Logs_20150203 TABLE_NO_KEY
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+
+table_create NotRelated_20150204 TABLE_NO_KEY
+
+table_create PathTerms_20150203 TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto
+
+table_create PathTerms_20150204 TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto
+
+table_create Paths_20150203 TABLE_HASH_KEY|KEY_LARGE ShortText
+
+table_create Paths_20150204 TABLE_HASH_KEY|KEY_LARGE ShortText
+
+table_create Times_20150203 TABLE_PAT_KEY Time
+
+table_create Times_20150204 TABLE_PAT_KEY Time
+
+column_create Logs_20150203 path COLUMN_SCALAR Paths_20150203
+
+load --table Paths_20150203
+[
+["_key"],
+["/tmp/log"],
+["/var/"]
+]
+
+load --table Logs_20150203
+[
+["_id","path","timestamp"],
+[1,"/tmp/log",1422935340.0],
+[2,"/var/",1422975599.0]
+]
+
+load --table Paths_20150204
+[
+["_key"],
+["/tmp/log"],
+["/root/"],
+["/etc/aliases"]
+]
+
+column_create PathTerms_20150203 index COLUMN_INDEX|WITH_POSITION Paths_20150203 _key
+column_create PathTerms_20150204 index COLUMN_INDEX|WITH_POSITION Paths_20150204 _key
+column_create Times_20150203 timestamp_index COLUMN_INDEX Logs_20150203 timestamp
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)
+#|e| grn_ctx_at: failed to open object: <272>(<Logs_20150204>):<51>(<table:no_key>)

test/command/suite/sharding/logical_table_remove/broken/table/default.test

@@ -0,0 +1,59 @@
+#@on-error omit
+plugin_register sharding
+#@on-error default
+
+table_create Paths_20150203 TABLE_HASH_KEY|KEY_LARGE ShortText
+table_create Logs_20150203 TABLE_NO_KEY
+column_create Logs_20150203 timestamp COLUMN_SCALAR Time
+column_create Logs_20150203 path COLUMN_SCALAR Paths_20150203
+table_create Times_20150203 TABLE_PAT_KEY Time
+column_create Times_20150203 timestamp_index COLUMN_INDEX Logs_20150203 timestamp
+
+table_create PathTerms_20150203 TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram \
+  --normalizer NormalizerAuto
+column_create PathTerms_20150203 index COLUMN_INDEX|WITH_POSITION \
+  Paths_20150203 _key
+
+
+table_create NotRelated_20150204 TABLE_NO_KEY
+
+table_create Paths_20150204 TABLE_HASH_KEY|KEY_LARGE ShortText
+table_create Logs_20150204 TABLE_NO_KEY
+column_create Logs_20150204 timestamp COLUMN_SCALAR Time
+column_create Logs_20150204 path COLUMN_SCALAR Paths_20150204
+table_create Times_20150204 TABLE_PAT_KEY Time
+column_create Times_20150204 timestamp_index COLUMN_INDEX Logs_20150204 timestamp
+
+table_create PathTerms_20150204 TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram \
+  --normalizer NormalizerAuto
+column_create PathTerms_20150204 index COLUMN_INDEX|WITH_POSITION \
+  Paths_20150204 _key
+
+
+load --table Logs_20150203
+[
+{"timestamp": "2015-02-03 12:49:00", "path": "/tmp/log"},
+{"timestamp": "2015-02-03 23:59:59", "path": "/var/"}
+]
+
+load --table Logs_20150204
+[
+{"timestamp": "2015-02-04 00:00:00", "path": "/tmp/log"},
+{"timestamp": "2015-02-04 13:49:00", "path": "/root/"},
+{"timestamp": "2015-02-04 13:50:00", "path": "/etc/aliases"}
+]
+
+#@disable-logging
+thread_limit 1
+database_unmap
+#@enable-logging
+
+#@copy-path fixture/sharding/logical_table_remove/broken #{db_path}.0000110
+
+logical_table_remove Logs timestamp \
+  --min "2015-02-04 00:00:00" \
+  --min_border "include"
+
+dump