Can now skip to arbitrary byte patterns

ChangeLog.rdoc

@@ -3,6 +3,7 @@
 == Version 2.3.x (2016-xx-xx)
 
 * IO#num_bytes_remaining now works inside Buffers.
+* Added ability to skip to arbitrary byte patterns.  Requested by Stefan Kolb.
 
 == Version 2.3.1 (2016-06-17)
 

lib/bindata/dsl.rb

@@ -122,7 +122,8 @@ def fields
       end
 
       def dsl_params
-        send(parser_abilities[@parser_type].at(0))
+        abilities = parser_abilities[@parser_type]
+        send(abilities.at(0), abilities.at(1))
       end
 
       def method_missing(*args, &block)
@@ -135,17 +136,18 @@ def method_missing(*args, &block)
 
       def parser_abilities
         @abilities ||= {
-          :struct     => [:to_struct_params, [:multiple_fields, :optional_fieldnames, :hidden_fields]],
-          :array      => [:to_array_params,  [:multiple_fields, :optional_fieldnames]],
-          :buffer     => [:to_array_params,  [:multiple_fields, :optional_fieldnames, :hidden_fields]],
-          :choice     => [:to_choice_params, [:multiple_fields, :all_or_none_fieldnames, :fieldnames_are_values]],
-          :delayed_io => [:to_array_params,  [:multiple_fields, :optional_fieldnames, :hidden_fields]],
-          :primitive  => [:to_struct_params, [:multiple_fields, :optional_fieldnames]]
+          :struct     => [:to_struct_params, :struct,      [:multiple_fields, :optional_fieldnames, :hidden_fields]],
+          :array      => [:to_object_params, :type,        [:multiple_fields, :optional_fieldnames]],
+          :buffer     => [:to_object_params, :type,        [:multiple_fields, :optional_fieldnames, :hidden_fields]],
+          :choice     => [:to_choice_params, :choices,     [:multiple_fields, :all_or_none_fieldnames, :fieldnames_are_values]],
+          :delayed_io => [:to_object_params, :type,        [:multiple_fields, :optional_fieldnames, :hidden_fields]],
+          :primitive  => [:to_struct_params, :struct,      [:multiple_fields, :optional_fieldnames]],
+          :skip       => [:to_object_params, :until_valid, [:multiple_fields, :optional_fieldnames]],
         }
       end
 
       def option?(opt)
-        parser_abilities[@parser_type].at(1).include?(opt)
+        parser_abilities[@parser_type].at(2).include?(opt)
       end
 
       def ensure_hints
@@ -219,30 +221,30 @@ def dsl_raise(exception, message)
         raise exception, message + " in #{@the_class}", backtrace
       end
 
-      def to_array_params
+      def to_object_params(key)
         case fields.length
         when 0
           {}
         when 1
-          {:type => fields[0].prototype}
+          {key => fields[0].prototype}
         else
-          {:type => [:struct, to_struct_params]}
+          {key=> [:struct, to_struct_params]}
         end
       end
 
-      def to_choice_params
+      def to_choice_params(key)
         if fields.length == 0
           {}
         elsif fields.all_field_names_blank?
-          {:choices => fields.collect { |f| f.prototype }}
+          {key => fields.collect { |f| f.prototype }}
         else
           choices = {}
           fields.each { |f| choices[f.name] = f.prototype }
-          {:choices => choices}
+          {key => choices}
         end
       end
 
-      def to_struct_params
+      def to_struct_params(*unused)
         result = {:fields => fields}
         if not endian.nil?
           result[:endian] = endian
@@ -386,7 +388,8 @@ def params_from_block(&block)
           :buffer     => BinData::Buffer,
           :choice     => BinData::Choice,
           :delayed_io => BinData::DelayedIO,
-          :struct     => BinData::Struct
+          :skip       => BinData::Skip,
+          :struct     => BinData::Struct,
         }
 
         if bindata_classes.include?(@type)

lib/bindata/io.rb

@@ -85,6 +85,17 @@ def num_bytes_remaining
           bytes_remaining
         end
 
+        # All io calls in +block+ are rolled back after this
+        # method completes.
+        def with_readahead(&block)
+          mark = @raw_io.pos
+          begin
+            block.call
+          ensure
+            @raw_io.seek(mark, ::IO::SEEK_SET)
+          end
+        end
+
         #-----------
         private
 
@@ -120,6 +131,26 @@ def num_bytes_remaining
           raise IOError, "stream is unseekable"
         end
 
+        # All io calls in +block+ are rolled back after this
+        # method completes.
+        def with_readahead(&block)
+          mark = @offset
+          @read_data = ""
+          @in_readahead = true
+
+          class << self
+            alias_method :read_raw_without_readahead, :read_raw
+            alias_method :read_raw, :read_raw_with_readahead
+          end
+
+          begin
+            block.call
+          ensure
+            @offset = mark
+            @in_readahead = false
+          end
+        end
+
         #-----------
         private
 
@@ -133,6 +164,33 @@ def read_raw(n)
           data
         end
 
+        def read_raw_with_readahead(n)
+          data = ""
+
+          if @read_data.length > 0 and not @in_readahead
+            bytes_to_consume = [n, @read_data.length].min
+            data << @read_data.slice!(0, bytes_to_consume)
+            n -= bytes_to_consume
+
+            if @read_data.length == 0
+              class << self
+                alias_method :read_raw, :read_raw_without_readahead
+              end
+            end
+          end
+
+          raw_data = @raw_io.read(n)
+          data << raw_data if raw_data
+
+          if @in_readahead
+            @read_data << data
+          end
+
+          @offset += data.size
+
+          data
+        end
+
         def write_raw(data)
           @offset += data.size
           @raw_io.write(data)

lib/bindata/skip.rb

@@ -16,24 +16,42 @@ module BinData
   #   obj = A.read("abcdefghij")
   #   obj.a #=> "fghij"
   #
+  #
+  #   class B < BinData::Record
+  #     skip :until_valid => [:string, {:read_length => 2, :assert => "ef"} ]
+  #     string :b, :read_length => 5
+  #   end
+  #
+  #   obj = B.read("abcdefghij")
+  #   obj.b #=> "efghi"
+  #
+  #
   # == Parameters
   #
   # Skip objects accept all the params that BinData::BasePrimitive
   # does, as well as the following:
   #
   # <tt>:length</tt>::        The number of bytes to skip.
   # <tt>:to_abs_offset</tt>:: Skips to the given absolute offset.
+  # <tt>:until_valid</tt>::   Skips untils a given byte pattern is matched.
+  #                           This parameter contains a type that will raise
+  #                           a BinData::ValidityError unless an acceptable byte
+  #                           sequence is found.  The type is represented by a
+  #                           Symbol, or if the type is to have params #
+  #                           passed to it, then it should be provided as #
+  #                           <tt>[type_symbol, hash_params]</tt>.
   #
   class Skip < BinData::BasePrimitive
 
     arg_processor :skip
 
-    optional_parameters :length, :to_abs_offset
-    mutually_exclusive_parameters :length, :to_abs_offset
+    optional_parameters :length, :to_abs_offset, :until_valid
+    mutually_exclusive_parameters :length, :to_abs_offset, :until_valid
 
     def initialize_shared_instance
       extend SkipLengthPlugin      if has_parameter?(:length)
       extend SkipToAbsOffsetPlugin if has_parameter?(:to_abs_offset)
+      extend SkipUntilValidPlugin if has_parameter?(:until_valid)
       super
     end
 
@@ -66,10 +84,17 @@ def sensible_default
 
   class SkipArgProcessor < BaseArgProcessor
     def sanitize_parameters!(obj_class, params)
-      unless (params.has_parameter?(:length) or params.has_parameter?(:to_abs_offset))
-        raise ArgumentError, "#{obj_class} requires either :length or :to_abs_offset"
+      unless params.has_parameter?(:length) or
+             params.has_parameter?(:to_abs_offset) or
+             params.has_parameter?(:until_valid)
+        raise ArgumentError, "#{obj_class} requires either :length, :to_abs_offset or :until_valid"
       end
       params.must_be_integer(:to_abs_offset, :length)
+
+      if params.needs_sanitizing?(:until_valid)
+        el_type, el_params = params[:until_valid]
+        params[:until_valid] = params.create_sanitized_object_prototype(el_type, el_params)
+      end
     end
   end
 
@@ -86,4 +111,29 @@ def skip_length
       eval_parameter(:to_abs_offset) - abs_offset
     end
   end
+
+  # Logic for the :until_valid parameter
+  module SkipUntilValidPlugin
+    def skip_length
+      # no skipping when writing
+      0
+    end
+
+    def read_and_return_value(io)
+      prototype = get_parameter(:until_valid)
+      validator = prototype.instantiate(nil, self)
+
+      valid = false
+      until valid
+        begin
+          io.with_readahead do
+            validator.read(io)
+            valid = true
+          end
+        rescue ValidityError
+          io.readbytes(1)
+        end
+      end
+    end
+  end
 end

test/io_test.rb

@@ -165,6 +165,64 @@
   end
 end
 
+module IOReadWithReadahead
+  def test_rolls_back_short_reads
+    io.readbytes(2).must_equal "ab"
+    io.with_readahead do
+      io.readbytes(4).must_equal "cdef"
+    end
+    io.offset.must_equal 2
+  end
+
+  def test_rolls_back_read_all_bytes
+    io.readbytes(3).must_equal "abc"
+    io.with_readahead do
+      io.read_all_bytes.must_equal "defghijklmnopqrst"
+    end
+    io.offset.must_equal 3
+  end
+
+  def test_inside_buffer_rolls_back_reads
+    io.with_buffer(10) do
+      io.with_readahead do
+        io.readbytes(4).must_equal "abcd"
+      end
+      io.offset.must_equal 0
+    end
+    io.offset.must_equal 10
+  end
+
+  def test_outside_buffer_rolls_back_reads
+    io.with_readahead do
+      io.with_buffer(10) do
+        io.readbytes(4).must_equal "abcd"
+      end
+      io.offset.must_equal 10
+    end
+    io.offset.must_equal 0
+  end
+end
+
+describe BinData::IO::Read, "#with_readahead" do
+  let(:stream) { StringIO.new "abcdefghijklmnopqrst" }
+  let(:io) { BinData::IO::Read.new(stream) }
+
+  include IOReadWithReadahead
+end
+
+describe BinData::IO::Read, "unseekable stream #with_readahead" do
+  let(:stream) {
+    io = StringIO.new "abcdefghijklmnopqrst"
+    def io.pos
+      raise Errno::EPIPE
+    end
+    io
+  }
+  let(:io) { BinData::IO::Read.new(stream) }
+
+  include IOReadWithReadahead
+end
+
 describe BinData::IO::Write, "writing to non seekable stream" do
   before do
     @rd, @wr = IO::pipe

test/skip_test.rb

@@ -85,3 +85,53 @@
     }.must_raise BinData::ValidityError
   end
 end
+
+describe BinData::Skip, "with :until_valid" do
+  let(:io) { StringIO.new("abcdefghij") }
+
+  it "skips to valid match" do
+    skip_obj = [:string, { :read_length => 1, :assert => "f" }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    obj.read(io)
+    io.pos.must_equal 5
+  end
+
+  it "doesn't skip when validator doesn't assert" do
+    skip_obj = [:string, { :read_length => 1 }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    obj.read(io)
+    io.pos.must_equal 0
+  end
+
+  it "raises EOFError when no match" do
+    skip_obj = [:string, { :read_length => 1, :assert => "X" }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    lambda {
+      obj.read(io)
+    }.must_raise EOFError
+  end
+
+  it "raises IOError when validator reads beyond stream" do
+    skip_obj = [:string, { :read_length => 30 }]
+    fields = [ [:skip, :s, { :until_valid => skip_obj }] ]
+    obj = BinData::Struct.new(:fields => fields)
+    lambda {
+      obj.read(io)
+    }.must_raise IOError
+  end
+
+  class DSLSkip < BinData::Record
+    skip :s do
+      string :read_length => 1, :assert => "f"
+    end
+    string :a, :read_length => 1
+  end
+
+  it "uses block form" do
+    obj = DSLSkip.read(io)
+    obj.a.must_equal "f"
+  end
+end