Accept other IETF/IANA-registered Content-Types and Content-Transfer-…

…Encodings

Fixes #470

CHANGELOG.rdoc

@@ -1,6 +1,7 @@
 == HEAD
 
 Features:
+* Accept other IETF/IANA-registered Content-Types and Content-Transfer-Encodings (jeremy)
 * Alias shift-jis charset to Shift_JIS Ruby encoding (jeremy)
 * Add support for ks_c_5601-1987 charset, aliased to CP949 Ruby encoding (jeremy)
 * Don't allow colons in header names (jeremy)

lib/mail/fields/content_transfer_encoding_field.rb

@@ -10,8 +10,8 @@ class ContentTransferEncodingField < StructuredField
 
     def initialize(value = nil, charset = 'utf-8')
       self.charset = charset
-      value = '7bit' if value.to_s =~ /7-bit/i
-      value = '8bit' if value.to_s =~ /8-bit/i
+      value = '7bit' if value.to_s =~ /7-?bits?/i
+      value = '8bit' if value.to_s =~ /8-?bits?/i
       super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, value), charset)
       self.parse
       self

lib/mail/parsers/content_transfer_encoding.rb

@@ -85,19 +85,6 @@ def _nt_primary
       r0
     end
 
-    module Encoding0
-      def ietf_token
-        elements[0]
-      end
-
-    end
-
-    module Encoding1
-      def text_value
-        ietf_token.text_value
-      end
-    end
-
     def _nt_encoding
       start_index = index
       if node_cache[:encoding].has_key?(index)
@@ -110,41 +97,93 @@ def _nt_encoding
       end
 
       i0 = index
-      i1, s1 = index, []
-      r2 = _nt_ietf_token
-      s1 << r2
-      if r2
-        if has_terminal?("s", false, index)
-          r4 = instantiate_node(SyntaxNode,input, index...(index + 1))
-          @index += 1
-        else
-          terminal_parse_failure("s")
-          r4 = nil
-        end
-        if r4
-          r3 = r4
-        else
-          r3 = instantiate_node(SyntaxNode,input, index...index)
-        end
-        s1 << r3
-      end
-      if s1.last
-        r1 = instantiate_node(SyntaxNode,input, i1...index, s1)
-        r1.extend(Encoding0)
-        r1.extend(Encoding1)
+      if has_terminal?("7bits", false, index)
+        r1 = instantiate_node(SyntaxNode,input, index...(index + 5))
+        @index += 5
       else
-        @index = i1
+        terminal_parse_failure("7bits")
         r1 = nil
       end
       if r1
         r0 = r1
       else
-        r5 = _nt_custom_x_token
-        if r5
-          r0 = r5
+        if has_terminal?("8bits", false, index)
+          r2 = instantiate_node(SyntaxNode,input, index...(index + 5))
+          @index += 5
+        else
+          terminal_parse_failure("8bits")
+          r2 = nil
+        end
+        if r2
+          r0 = r2
         else
-          @index = i0
-          r0 = nil
+          if has_terminal?("7bit", false, index)
+            r3 = instantiate_node(SyntaxNode,input, index...(index + 4))
+            @index += 4
+          else
+            terminal_parse_failure("7bit")
+            r3 = nil
+          end
+          if r3
+            r0 = r3
+          else
+            if has_terminal?("8bit", false, index)
+              r4 = instantiate_node(SyntaxNode,input, index...(index + 4))
+              @index += 4
+            else
+              terminal_parse_failure("8bit")
+              r4 = nil
+            end
+            if r4
+              r0 = r4
+            else
+              if has_terminal?("binary", false, index)
+                r5 = instantiate_node(SyntaxNode,input, index...(index + 6))
+                @index += 6
+              else
+                terminal_parse_failure("binary")
+                r5 = nil
+              end
+              if r5
+                r0 = r5
+              else
+                if has_terminal?("quoted-printable", false, index)
+                  r6 = instantiate_node(SyntaxNode,input, index...(index + 16))
+                  @index += 16
+                else
+                  terminal_parse_failure("quoted-printable")
+                  r6 = nil
+                end
+                if r6
+                  r0 = r6
+                else
+                  if has_terminal?("base64", false, index)
+                    r7 = instantiate_node(SyntaxNode,input, index...(index + 6))
+                    @index += 6
+                  else
+                    terminal_parse_failure("base64")
+                    r7 = nil
+                  end
+                  if r7
+                    r0 = r7
+                  else
+                    r8 = _nt_ietf_token
+                    if r8
+                      r0 = r8
+                    else
+                      r9 = _nt_custom_x_token
+                      if r9
+                        r0 = r9
+                      else
+                        @index = i0
+                        r0 = nil
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
         end
       end
 
@@ -159,4 +198,4 @@ class ContentTransferEncodingParser < Treetop::Runtime::CompiledParser
     include ContentTransferEncoding
   end
 
-end
+end

lib/mail/parsers/content_transfer_encoding.treetop

@@ -9,12 +9,10 @@ module Mail
     end
 
     rule encoding
-      ietf_token "s"? {
-        def text_value
-          ietf_token.text_value
-        end
-      } / custom_x_token
+      "7bits" / "8bits" /
+      "7bit" / "8bit" / "binary" / "quoted-printable" / "base64" /
+      ietf_token / custom_x_token
     end
 
   end
-end
+end

lib/mail/parsers/rfc2045.rb

@@ -216,64 +216,21 @@ def _nt_ietf_token
         return cached
       end
 
-      i0 = index
-      if has_terminal?("7bit", false, index)
-        r1 = instantiate_node(SyntaxNode,input, index...(index + 4))
-        @index += 4
-      else
-        terminal_parse_failure("7bit")
-        r1 = nil
-      end
-      if r1
-        r0 = r1
-      else
-        if has_terminal?("8bit", false, index)
-          r2 = instantiate_node(SyntaxNode,input, index...(index + 4))
-          @index += 4
-        else
-          terminal_parse_failure("8bit")
-          r2 = nil
-        end
-        if r2
-          r0 = r2
+      s0, i0 = [], index
+      loop do
+        r1 = _nt_token
+        if r1
+          s0 << r1
         else
-          if has_terminal?("binary", false, index)
-            r3 = instantiate_node(SyntaxNode,input, index...(index + 6))
-            @index += 6
-          else
-            terminal_parse_failure("binary")
-            r3 = nil
-          end
-          if r3
-            r0 = r3
-          else
-            if has_terminal?("quoted-printable", false, index)
-              r4 = instantiate_node(SyntaxNode,input, index...(index + 16))
-              @index += 16
-            else
-              terminal_parse_failure("quoted-printable")
-              r4 = nil
-            end
-            if r4
-              r0 = r4
-            else
-              if has_terminal?("base64", false, index)
-                r5 = instantiate_node(SyntaxNode,input, index...(index + 6))
-                @index += 6
-              else
-                terminal_parse_failure("base64")
-                r5 = nil
-              end
-              if r5
-                r0 = r5
-              else
-                @index = i0
-                r0 = nil
-              end
-            end
-          end
+          break
         end
       end
+      if s0.empty?
+        @index = i0
+        r0 = nil
+      else
+        r0 = instantiate_node(SyntaxNode,input, i0...index, s0)
+      end
 
       node_cache[:ietf_token][start_index] = r0
 

lib/mail/parsers/rfc2045.treetop

@@ -8,8 +8,7 @@ module Mail
     end
 
     rule ietf_token
-      "7bit" / "8bit" / "binary" /
-      "quoted-printable" / "base64"
+      token+
     end
 
     rule custom_x_token

spec/mail/field_spec.rb

@@ -120,9 +120,10 @@
 
   describe "error handling" do
     it "should populate the errors array if it finds a field it can't deal with" do
-      field = Mail::Field.new('Content-Transfer-Encoding: bit')
+      field = Mail::Field.new('Content-Transfer-Encoding: 8@bit')
+      field.field.errors.size.should eq 1
       field.field.errors[0][0].should eq 'Content-Transfer-Encoding'
-      field.field.errors[0][1].should eq 'bit'
+      field.field.errors[0][1].should eq '8@bit'
       field.field.errors[0][2].to_s.should =~ /ContentTransferEncodingElement can not parse |17-bit|/
     end
   end

spec/mail/fields/content_transfer_encoding_field_spec.rb

@@ -75,6 +75,16 @@
         t.encoding.should eq encoding
       end
     end
+
+    it "should treat 7bits/7-bit and 8bits/8-bit as 7bit and 8bit" do
+      %w(7bits 7-bit).each do |mechanism|
+        Mail::ContentTransferEncodingField.new(mechanism).encoding.should eq '7bit'
+      end
+
+      %w(8bits 8-bit).each do |mechanism|
+        Mail::ContentTransferEncodingField.new(mechanism).encoding.should eq '8bit'
+      end
+    end
 
     it "should handle any valid 'x-token' value" do
       t = Mail::ContentTransferEncodingField.new('X-This-is_MY-encoding')
@@ -86,6 +96,11 @@
       t.encoding.should eq "x-uuencode"
     end
 
+    it "should handle an ietf encoding (practically, any token)" do
+      t = Mail::ContentTransferEncodingField.new("ietf-token")
+      t.encoding.should eq "ietf-token"
+    end
+
     it "should replace the existing value" do
       t = Mail::ContentTransferEncodingField.new("7bit")
       t.parse("quoted-printable")

spec/mail/fields/content_type_field_spec.rb

@@ -601,6 +601,15 @@
       c.parameters.should eql({"name" => "IM 006.jpg"})
     end
 
+    it "should handle 'unknown/unknown'" do
+      string = %(unknown/unknown; charset=iso-8859-1; name=IMSTP19.gif)
+      c = Mail::ContentTypeField.new(string)
+      c.content_type.should eq 'unknown/unknown'
+      c.main_type.should eq 'unknown'
+      c.sub_type.should eq 'unknown'
+      c.parameters.should eql('charset' => 'iso-8859-1', 'name' => 'IMSTP19.gif')
+    end
+
   end
 
   describe "finding a filename" do

spec/mail/header_spec.rb

@@ -458,11 +458,11 @@
 
   describe "error handling" do
     it "should collect up any of its fields' errors" do
-      header = Mail::Header.new("Content-Transfer-Encoding: vlad\r\nReply-To: a b b")
+      header = Mail::Header.new("Content-Transfer-Encoding: vl@d\r\nReply-To: a b b")
       header.errors.should_not be_blank
       header.errors.size.should eq 2
       header.errors[0][0].should eq 'Content-Transfer-Encoding'
-      header.errors[0][1].should eq 'vlad'
+      header.errors[0][1].should eq 'vl@d'
       header.errors[1][0].should eq 'Reply-To'
       header.errors[1][1].should eq 'a b b'
     end

spec/mail/message_spec.rb

@@ -1534,11 +1534,11 @@ def self.delivering_email(mail)
 
   describe "error handling" do
     it "should collect up any of its fields' errors" do
-      mail = Mail.new("Content-Transfer-Encoding: vlad\r\nReply-To: a b b\r\n")
+      mail = Mail.new("Content-Transfer-Encoding: vl@d\r\nReply-To: a b b\r\n")
       mail.errors.should_not be_blank
       mail.errors.size.should eq 2
       mail.errors[0][0].should eq 'Content-Transfer-Encoding'
-      mail.errors[0][1].should eq 'vlad'
+      mail.errors[0][1].should eq 'vl@d'
       mail.errors[1][0].should eq 'Reply-To'
       mail.errors[1][1].should eq 'a b b'
     end

spec/mail/parsers/content_transfer_encoding_parser_spec.rb

@@ -51,18 +51,12 @@
   end
 
   describe "wild content-transfer-encoding" do
-    it "should convert 8bits to 8bit" do
-      text = '8bits'
-      a = Mail::ContentTransferEncodingParser.new
-      a.parse(text).should_not be_nil
-      a.parse(text).encoding.text_value.should eq '8bit'
-    end
-
-    it "should convert 7bits to 7bit" do
-      text = '7bits'
-      a = Mail::ContentTransferEncodingParser.new
-      a.parse(text).should_not be_nil
-      a.parse(text).encoding.text_value.should eq '7bit'
+    %w(7bits 8bits 7-bit 8-bit).each do |mechanism|
+      it "should parse #{mechanism} variant" do
+        a = Mail::ContentTransferEncodingParser.new
+        a.parse(mechanism).should_not be_nil
+        a.parse(mechanism).encoding.text_value.should eq mechanism
+      end
     end
   end
 end