Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Add UniversalDetector#contains_high_bit?

  • Loading branch information...
commit 3c697a3dc11676177def0a2d37afd5f00d6ff7bc 1 parent 1438034
@CraigCottingham authored
Showing with 27 additions and 3 deletions.
  1. +7 −3 lib/UniversalDetector.rb
  2. +20 −0 spec/universal_detector_spec.rb
View
10 lib/UniversalDetector.rb
@@ -74,7 +74,7 @@ class Detector
attr_reader :result
def initialize
- @_highBitDetector = Regexp.new('[\x80-\xFF]', nil, 'n')
+ # @_highBitDetector = Regexp.new('[\x80-\xFF]', nil, 'n')
@_escDetector = /\033|~\{/
@_mEscCharSetProber = nil
@_mCharSetProbers = []
@@ -87,7 +87,7 @@ def reset
@_mStart = true
@_mGotData = false
@_mInputState = :PureAscii
- @_mLastChar = ""
+ @_mLastChar = []
if @_mEscCharSetProber
@_mEscCharSetProber.reset
end
@@ -96,6 +96,10 @@ def reset
end
end
+ def contains_high_bit?(data)
+ data.any? { | b | (b & 0x80) != 0 }
+ end
+
def feed(data)
if @done || data.empty?
return
@@ -132,7 +136,7 @@ def feed(data)
end
if @_mInputState == :PureAscii
- if data =~ @_highBitDetector
+ if contains_high_bit?(data)
@_mInputState = :Highbyte
elsif (@_mLastChar + data) =~ @_escDetector
@_mInputState = :EscAscii
View
20 spec/universal_detector_spec.rb
@@ -0,0 +1,20 @@
+require 'UniversalDetector'
+
+describe UniversalDetector, '#contains_high_bit?' do
+
+ it 'returns true if data contains one byte with its high bit set' do
+ u = UniversalDetector::Detector.instance
+ u.contains_high_bit?([ 0x00, 0x80, 0x00 ]).should eq(true)
+ end
+
+ it 'returns true if data contains more than one byte with its high bit set' do
+ u = UniversalDetector::Detector.instance
+ u.contains_high_bit?([ 0x00, 0xDE, 0xAD, 0xBE, 0xEF, 0x00 ]).should eq(true)
+ end
+
+ it 'returns false if data contains only bytes with their high bits cleared' do
+ u = UniversalDetector::Detector.instance
+ u.contains_high_bit?([ 0x00, 0x7F ]).should eq(false)
+ end
+
+end
Please sign in to comment.
Something went wrong with that request. Please try again.