Merge pull request #142 from sebastianludwig/unified_reading

Unified reading

lib/twine/cli.rb

@@ -83,10 +83,7 @@ def self.parse(args)
         opts.on('-c', '--consume-comments', 'Normally, when consuming a string file, Twine will ignore all comments in the file. With this flag set, any comments encountered will be read and parsed into the strings data file. This is especially useful when creating your first strings data file from an existing project.') do |c|
           options[:consume_comments] = true
         end
-        opts.on('-e', '--encoding ENCODING', 'Twine defaults to encoding all output files in UTF-8. This flag will tell Twine to use an alternate encoding for these files. For example, you could use this to write Apple .strings files in UTF-16. This flag is currently only supported in Ruby 1.9.3 or greater.') do |e|
-          unless "".respond_to? :encode
-            raise Twine::Error.new "The --encoding flag is only supported on Ruby 1.9.3 or greater."
-          end
+        opts.on('-e', '--encoding ENCODING', 'Twine defaults to encoding all output files in UTF-8. This flag will tell Twine to use an alternate encoding for these files. For example, you could use this to write Apple .strings files in UTF-16. When reading files, Twine does its best to determine the encoding automatically. However, if the files are UTF-16 without BOM, you need to specify if it\'s UTF-16LE or UTF16-BE.') do |e|
           options[:output_encoding] = e
         end
         opts.on('--validate', 'Validate the strings file before formatting it') do

lib/twine/encoding.rb

@@ -1,20 +1,22 @@
 module Twine
   module Encoding
-    def self.encoding_for_path path
-      File.open(path, 'rb') do |f|
-        begin
-          a = f.readbyte
-          b = f.readbyte
-          if (a == 0xfe && b == 0xff)
-            return 'UTF-16BE'
-          elsif (a == 0xff && b == 0xfe)
-            return 'UTF-16LE'
-          end
-        rescue EOFError
-        end
-      end
 
-      'UTF-8'
+    def self.bom(path)
+      first_bytes = IO.binread(path, 2)
+      return nil unless first_bytes
+      first_bytes = first_bytes.codepoints.map.to_a
+      return 'UTF-16BE' if first_bytes == [0xFE, 0xFF]
+      return 'UTF-16LE' if first_bytes == [0xFF, 0xFE]
+    rescue EOFError
+      return nil
+    end
+
+    def self.has_bom?(path)
+      !bom(path).nil?
+    end
+
+    def self.encoding_for_path(path)
+      bom(path) || 'UTF-8'      
     end
   end
 end

lib/twine/formatters/abstract.rb

@@ -83,8 +83,8 @@ def output_path_for_language(lang)
         lang
       end
 
-      def read_file(path, lang)
-        raise NotImplementedError.new("You must implement read_file in your formatter class.")
+      def read(io, lang)
+        raise NotImplementedError.new("You must implement read in your formatter class.")
       end
 
       def format_file(lang)

lib/twine/formatters/android.rb

@@ -68,7 +68,7 @@ def set_translation_for_key(key, lang, value)
         super(key, lang, value)
       end
 
-      def read_file(path, lang)
+      def read(io, lang)
         resources_regex = /<resources(?:[^>]*)>(.*)<\/resources>/m
         key_regex = /<string name="(\w+)">/
         comment_regex = /<!-- (.*) -->/
@@ -77,27 +77,25 @@ def read_file(path, lang)
         value = nil
         comment = nil
 
-        File.open(path, 'r:UTF-8') do |f|
-          content_match = resources_regex.match(f.read)
-          if content_match
-            for line in content_match[1].split(/\r?\n/)
-              key_match = key_regex.match(line)
-              if key_match
-                key = key_match[1]
-                value_match = value_regex.match(line)
-                value = value_match ? value_match[1] : ""
-
-                set_translation_for_key(key, lang, value)
-                if comment and comment.length > 0 and !comment.start_with?("SECTION:")
-                  set_comment_for_key(key, comment)
-                end
-                comment = nil
-              end
-
-              comment_match = comment_regex.match(line)
-              if comment_match
-                comment = comment_match[1]
+        content_match = resources_regex.match(io.read)
+        if content_match
+          for line in content_match[1].split(/\r?\n/)
+            key_match = key_regex.match(line)
+            if key_match
+              key = key_match[1]
+              value_match = value_regex.match(line)
+              value = value_match ? value_match[1] : ""
+
+              set_translation_for_key(key, lang, value)
+              if comment and comment.length > 0 and !comment.start_with?("SECTION:")
+                set_comment_for_key(key, comment)
               end
+              comment = nil
+            end
+
+            comment_match = comment_regex.match(line)
+            if comment_match
+              comment = comment_match[1]
             end
           end
         end

lib/twine/formatters/apple.rb

@@ -35,56 +35,28 @@ def output_path_for_language(lang)
         "#{lang}.lproj"
       end
 
-      def read_file(path, lang)
-        encoding = Twine::Encoding.encoding_for_path(path)
-        sep = nil
-        if !encoding.respond_to?(:encode)
-          # This code is not necessary in 1.9.3 and does not work as it did in 1.8.7.
-          if encoding.end_with? 'LE'
-            sep = "\x0a\x00"
-          elsif encoding.end_with? 'BE'
-            sep = "\x00\x0a"
-          else
-            sep = "\n"
-          end
-        end
-
-        if encoding.index('UTF-16')
-          mode = "rb:#{encoding}"
-        else
-          mode = "r:#{encoding}"
-        end
-
-        File.open(path, mode) do |f|
-          last_comment = nil
-          while line = (sep) ? f.gets(sep) : f.gets
-            if encoding.index('UTF-16')
-              if line.respond_to? :encode!
-                line.encode!('UTF-8')
-              else
-                require 'iconv'
-                line = Iconv.iconv('UTF-8', encoding, line).join
-              end
-            end
-            match = /"((?:[^"\\]|\\.)+)"\s*=\s*"((?:[^"\\]|\\.)*)"/.match(line)
-            if match
-              key = match[1]
-              key.gsub!('\\"', '"')
-              value = match[2]
-              value.gsub!('\\"', '"')
-              set_translation_for_key(key, lang, value)
-              if last_comment
-                set_comment_for_key(key, last_comment)
-              end
-            end
-
-            match = /\/\* (.*) \*\//.match(line)
-            if match
-              last_comment = match[1]
-            else
-              last_comment = nil
+      def read(io, lang)
+        last_comment = nil
+        while line = io.gets
+          # matches a `key = "value"` line, where key may be quoted or unquoted. The former may also contain escaped characters
+          match = /^\s*((?:"(?:[^"\\]|\\.)+")|(?:[^"\s=]+))\s*=\s*"((?:[^"\\]|\\.)*)"/.match(line)
+          if match
+            key = match[1]
+            key = key[1..-2] if key[0] == '"' and key[-1] == '"'
+            key.gsub!('\\"', '"')
+            value = match[2]
+            value.gsub!('\\"', '"')
+            set_translation_for_key(key, lang, value)
+            if last_comment
+              set_comment_for_key(key, last_comment)
             end
+          end
 
+          match = /\/\* (.*) \*\//.match(line)
+          if match
+            last_comment = match[1]
+          else
+            last_comment = nil
           end
         end
       end

lib/twine/formatters/django.rb

@@ -29,67 +29,35 @@ def determine_language_given_path(path)
         return
       end
 
-      def read_file(path, lang)
+      def read(io, lang)
         comment_regex = /#\. *"?(.*)"?$/
         key_regex = /msgid *"(.*)"$/
         value_regex = /msgstr *"(.*)"$/m
 
-        encoding = Twine::Encoding.encoding_for_path(path)
-        sep = nil
-        if !encoding.respond_to?(:encode)
-          # This code is not necessary in 1.9.3 and does not work as it did in 1.8.7.
-          if encoding.end_with? 'LE'
-            sep = "\x0a\x00"
-          elsif encoding.end_with? 'BE'
-            sep = "\x00\x0a"
-          else
-            sep = "\n"
+        last_comment = nil
+        while line = io.gets          
+          comment_match = comment_regex.match(line)
+          if comment_match
+            comment = comment_match[1]
           end
-        end
-
-        if encoding.index('UTF-16')
-          mode = "rb:#{encoding}"
-        else
-          mode = "r:#{encoding}"
-        end
-
-        File.open(path, mode) do |f|
-          last_comment = nil
-          while line = (sep) ? f.gets(sep) : f.gets
-            if encoding.index('UTF-16')
-              if line.respond_to? :encode!
-                line.encode!('UTF-8')
-              else
-                require 'iconv'
-                line = Iconv.iconv('UTF-8', encoding, line).join
-              end
-            end
-
-            comment_match = comment_regex.match(line)
-            if comment_match
-              comment = comment_match[1]
-            end
-
-            key_match = key_regex.match(line)
-            if key_match
-                key = key_match[1].gsub('\\"', '"')
-            end
-            value_match = value_regex.match(line)
-            if value_match
-                value = value_match[1].gsub(/"\n"/, '').gsub('\\"', '"')
-            end
 
+          key_match = key_regex.match(line)
+          if key_match
+            key = key_match[1].gsub('\\"', '"')
+          end
+          value_match = value_regex.match(line)
+          if value_match
+            value = value_match[1].gsub(/"\n"/, '').gsub('\\"', '"')
+          end
 
-            if key and key.length > 0 and value and value.length > 0
-                set_translation_for_key(key, lang, value)
-                if comment and comment.length > 0 and !comment.start_with?("--------- ")
-                    set_comment_for_key(key, comment)
-                end
-                key = nil
-                value = nil
-                comment = nil
+          if key and key.length > 0 and value and value.length > 0
+            set_translation_for_key(key, lang, value)
+            if comment and comment.length > 0 and !comment.start_with?("--------- ")
+              set_comment_for_key(key, comment)
             end
-
+            key = nil
+            value = nil
+            comment = nil
           end
         end
       end

lib/twine/formatters/flash.rb

@@ -21,55 +21,25 @@ def determine_language_given_path(path)
         return
       end
 
-      def read_file(path, lang)
-        encoding = Twine::Encoding.encoding_for_path(path)
-        sep = nil
-        if !encoding.respond_to?(:encode)
-          # This code is not necessary in 1.9.3 and does not work as it did in 1.8.7.
-          if encoding.end_with? 'LE'
-            sep = "\x0a\x00"
-          elsif encoding.end_with? 'BE'
-            sep = "\x00\x0a"
-          else
-            sep = "\n"
-          end
-        end
-
-        if encoding.index('UTF-16')
-          mode = "rb:#{encoding}"
-        else
-          mode = "r:#{encoding}"
-        end
-
-        File.open(path, mode) do |f|
-          last_comment = nil
-          while line = (sep) ? f.gets(sep) : f.gets
-            if encoding.index('UTF-16')
-              if line.respond_to? :encode!
-                line.encode!('UTF-8')
-              else
-                require 'iconv'
-                line = Iconv.iconv('UTF-8', encoding, line).join
-              end
+      def read(io, lang)
+        last_comment = nil
+        while line = io.gets
+          match = /((?:[^"\\]|\\.)+)\s*=\s*((?:[^"\\]|\\.)*)/.match(line)
+          if match
+            key = match[1]
+            value = match[2].strip
+            value.gsub!(/\{[0-9]\}/, '%@')
+            set_translation_for_key(key, lang, value)
+            if last_comment
+              set_comment_for_key(key, last_comment)
             end
-            match = /((?:[^"\\]|\\.)+)\s*=\s*((?:[^"\\]|\\.)*)/.match(line)
-            if match
-              key = match[1]
-              value = match[2].strip
-              value.gsub!(/\{[0-9]\}/, '%@')
-              set_translation_for_key(key, lang, value)
-              if last_comment
-                set_comment_for_key(key, last_comment)
-              end
-            end
-
-            match = /# *(.*)/.match(line)
-            if match
-              last_comment = match[1]
-            else
-              last_comment = nil
-            end
-
+          end
+
+          match = /# *(.*)/.match(line)
+          if match
+            last_comment = match[1]
+          else
+            last_comment = nil
           end
         end
       end