Adjust all data buffer pointers when we slice the array.

This is the result of discussion in Lighthouse bug
#72.

No unit tests were broken in the implementation of
this fix.  The example code in the ticket now
functions correctly over several iterations.

One thing that may still need to be adjusted is
the size of the buffer.  The larger the buffer,
the slower AMI performance is.  This is due to the
many places string offsets are used in the parsing.

See: https://adhearsion.lighthouseapp.com/projects/5871-adhearsion/tickets/72-ami-lexer-buffer-offset#ticket-72-26

lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb

@@ -10,8 +10,28 @@ class AbstractAsteriskManagerInterfaceStreamLexer
 
           BUFFER_SIZE = 128.kilobytes unless defined? BUFFER_SIZE
 
-
-# line 52 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+          ##
+          # IMPORTANT! See method documentation for adjust_pointers!
+          #
+          # @see  adjust_pointers
+          #
+          POINTERS = [
+            :@current_pointer,
+            :@token_start,
+            :@token_end,
+            :@version_start,
+            :@event_name_start,
+            :@current_key_position,
+            :@current_value_position,
+            :@last_seen_value_end,
+            :@error_reason_start,
+            :@follows_text_start,
+            :@current_syntax_error_start,
+            :@immediate_response_start
+            ]
+
+
+# line 72 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 ##
 
           attr_accessor(:ami_version)
@@ -22,7 +42,7 @@ def initialize
             @ragel_stack = []
 
 
-# line 26 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
+# line 46 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
 class << self
 	attr_accessor :_ami_protocol_parser_actions
 	private :_ami_protocol_parser_actions, :_ami_protocol_parser_actions=
@@ -839,7 +859,7 @@ class << self
 self.ami_protocol_parser_en_response_follows = 268;
 
 
-# line 843 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
+# line 863 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
 begin
 	     @current_pointer ||= 0
 	    @data_ending_pointer ||=   @data.length
@@ -850,7 +870,7 @@ class << self
 	   @ragel_act = 0
 end
 
-# line 78 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 98 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 ##
 
           end
@@ -862,7 +882,7 @@ def <<(new_data)
 
           def resume!
 
-# line 866 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
+# line 886 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
 begin
 	_klen, _trans, _keys, _acts, _nacts = nil
 	_goto_level = 0
@@ -896,7 +916,7 @@ def resume!
 		begin
     @token_start =      @current_pointer
 		end
-# line 900 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
+# line 920 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
 		end # from state action switch
 	end
 	if _trigger_goto
@@ -964,87 +984,87 @@ def resume!
 			_acts += 1
 			case _ami_protocol_parser_actions[_acts - 1]
 when 0 then
-# line 17 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 37 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  init_success 		end
 when 1 then
-# line 19 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 39 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  init_response_follows 		end
 when 2 then
-# line 21 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 41 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  init_error 		end
 when 3 then
-# line 23 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 43 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  message_received @current_message 		end
 when 4 then
-# line 24 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 44 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
    error_received @current_message 		end
 when 5 then
-# line 26 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 46 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  version_starts 		end
 when 6 then
-# line 27 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 47 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  version_stops  		end
 when 7 then
-# line 29 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 49 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  key_starts 		end
 when 8 then
-# line 30 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 50 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  key_stops  		end
 when 9 then
-# line 32 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 52 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  value_starts 		end
 when 10 then
-# line 33 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 53 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  value_stops  		end
 when 11 then
-# line 35 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 55 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  error_reason_starts 		end
 when 12 then
-# line 36 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 56 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  error_reason_stops  		end
 when 13 then
-# line 38 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 58 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  syntax_error_starts 		end
 when 14 then
-# line 39 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 59 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  syntax_error_stops  		end
 when 15 then
-# line 41 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 61 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  immediate_response_starts 		end
 when 16 then
-# line 42 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 62 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  immediate_response_stops  		end
 when 17 then
-# line 44 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 64 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  follows_text_starts 		end
 when 18 then
-# line 45 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 65 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  follows_text_stops  		end
 when 19 then
-# line 47 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 67 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  event_name_starts 		end
 when 20 then
-# line 48 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 68 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 		begin
  event_name_stops  		end
 when 21 then
@@ -1368,7 +1388,7 @@ def resume!
 end
 end
 			end
-# line 1372 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
+# line 1392 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
 			end # action switch
 		end
 	end
@@ -1393,7 +1413,7 @@ def resume!
 		begin
    @ragel_act = 0
 		end
-# line 1397 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
+# line 1417 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rb"
 		end # to state action switch
 	end
 	if _trigger_goto
@@ -1424,36 +1444,56 @@ def resume!
 	end
 	end
 
-# line 88 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
+# line 108 "lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb"
 ##
           end
 
           def extend_buffer_with(new_data)
-            if new_data.size > BUFFER_SIZE
+            length = new_data.size
+
+            if length > BUFFER_SIZE
               raise Exception, "ERROR: Buffer overrun! Input size (#{new_data.size}) larger than buffer (#{BUFFER_SIZE})"
             end
 
-            if new_data.size + @data.size > BUFFER_SIZE
-              offset = @data.size
+            if length + @data.size > BUFFER_SIZE
               if @data.size != @current_pointer
-                if @current_pointer < new_data.size
+                if @current_pointer < length
                   # We are about to shift more bytes off the array than we have
                   # parsed.  This will cause the parser to lose state so
                   # integrity cannot be guaranteed.
                   raise Exception, "ERROR: Buffer overrun! AMI parser cannot guarantee sanity. New data size: #{new_data.size}; Current pointer at #{@current_pointer}; Data size: #{@data.size}"
-                else
-                  offset = @current_pointer - new_data.size
                 end
               end
-              @data.slice! 0...new_data.size
-              @current_pointer = offset
+              @data.slice! 0...length
+              adjust_pointers -length
             end
             @data << new_data
             @data_ending_pointer = @data.size
           end
 
           protected
 
+          ##
+          # This method will adjust all pointers into the buffer according
+          # to the supplied offset.  This is necessary any time the buffer
+          # changes, for example when the sliding window is incremented forward
+          # after new data is received.
+          #
+          # It is VERY IMPORTANT that when any additional pointers are defined
+          # that they are added to this method.  Unpredictable results may
+          # otherwise occur!
+          #
+          # @see https://adhearsion.lighthouseapp.com/projects/5871-adhearsion/tickets/72-ami-lexer-buffer-offset#ticket-72-26
+          #
+          # @param offset Adjust pointers by offset.  May be negative.
+          #
+          def adjust_pointers(offset)
+            POINTERS.each do |ptr|
+              value = instance_variable_get(ptr)
+              instance_variable_set(ptr, value + offset) if !value.nil?
+            end
+          end
+
           ##
           # Called after a response or event has been successfully parsed.
           #
@@ -1497,12 +1537,12 @@ def init_error
           end
 
           def version_starts
-            @start_of_version = @current_pointer
+            @version_start = @current_pointer
           end
 
           def version_stops
-            self.ami_version = @data[@start_of_version...@current_pointer].to_f
-            @start_of_version = nil
+            self.ami_version = @data[@version_start...@current_pointer].to_f
+            @version_start = nil
           end
 
           def event_name_starts

lib/adhearsion/voip/asterisk/manager_interface/ami_lexer.rl.rb

@@ -8,12 +8,32 @@ class AbstractAsteriskManagerInterfaceStreamLexer
 
           BUFFER_SIZE = 128.kilobytes unless defined? BUFFER_SIZE
 
+          ##
+          # IMPORTANT! See method documentation for adjust_pointers!
+          #
+          # @see  adjust_pointers
+          #
+          POINTERS = [
+            :@current_pointer,
+            :@token_start,
+            :@token_end,
+            :@version_start,
+            :@event_name_start,
+            :@current_key_position,
+            :@current_value_position,
+            :@last_seen_value_end,
+            :@error_reason_start,
+            :@follows_text_start,
+            :@current_syntax_error_start,
+            :@immediate_response_start
+            ]
+
           %%{
           	machine ami_protocol_parser;
 
             # All required Ragel actions are implemented as Ruby methods.
 
-            # Executed after a "Respone: Success" or "Response: Pong"
+            # Executed after a "Response: Success" or "Response: Pong"
             action init_success { init_success }
 
             action init_response_follows { init_response_follows }
@@ -89,31 +109,51 @@ def resume!
           end
 
           def extend_buffer_with(new_data)
-            if new_data.size > BUFFER_SIZE
+            length = new_data.size
+
+            if length > BUFFER_SIZE
               raise Exception, "ERROR: Buffer overrun! Input size (#{new_data.size}) larger than buffer (#{BUFFER_SIZE})"
             end
 
-            if new_data.size + @data.size > BUFFER_SIZE
-              offset = @data.size
+            if length + @data.size > BUFFER_SIZE
               if @data.size != @current_pointer
-                if @current_pointer < new_data.size
+                if @current_pointer < length
                   # We are about to shift more bytes off the array than we have
                   # parsed.  This will cause the parser to lose state so
                   # integrity cannot be guaranteed.
                   raise Exception, "ERROR: Buffer overrun! AMI parser cannot guarantee sanity. New data size: #{new_data.size}; Current pointer at #{@current_pointer}; Data size: #{@data.size}"
-                else
-                  offset = @current_pointer - new_data.size
                 end
               end
-              @data.slice! 0...new_data.size
-              @current_pointer = offset
+              @data.slice! 0...length
+              adjust_pointers -length
             end
             @data << new_data
             @data_ending_pointer = @data.size
           end
 
           protected
 
+          ##
+          # This method will adjust all pointers into the buffer according
+          # to the supplied offset.  This is necessary any time the buffer
+          # changes, for example when the sliding window is incremented forward
+          # after new data is received.
+          #
+          # It is VERY IMPORTANT that when any additional pointers are defined
+          # that they are added to this method.  Unpredictable results may
+          # otherwise occur!
+          #
+          # @see https://adhearsion.lighthouseapp.com/projects/5871-adhearsion/tickets/72-ami-lexer-buffer-offset#ticket-72-26
+          #
+          # @param offset Adjust pointers by offset.  May be negative.
+          #
+          def adjust_pointers(offset)
+            POINTERS.each do |ptr|
+              value = instance_variable_get(ptr)
+              instance_variable_set(ptr, value + offset) if !value.nil?
+            end
+          end
+
           ##
           # Called after a response or event has been successfully parsed.
           #
@@ -157,12 +197,12 @@ def init_error
           end
 
           def version_starts
-            @start_of_version = @current_pointer
+            @version_start = @current_pointer
           end
 
           def version_stops
-            self.ami_version = @data[@start_of_version...@current_pointer].to_f
-            @start_of_version = nil
+            self.ami_version = @data[@version_start...@current_pointer].to_f
+            @version_start = nil
           end
 
           def event_name_starts