Handle unknown file types more gracefully

Author: alexrudall

CHANGELOG.md

@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - [BREAKING] Try to JSON parse everything. If it fails, always fall back to returning the raw response. Thank you to [@gregszero](https://github.com/gregszero) and the many others who raised this issue.
+- [BREAKING] An unknown file type will no longer prevent file upload, but instead raise a warning.
+- Add `user_data` and `evals` as options for known File types - thank you to [jontec](https://github.com/jontec) for this fix!
 
 ### Removed
 

lib/openai.rb

@@ -1,6 +1,5 @@
 require "faraday"
 require "faraday/multipart" if Gem::Version.new(Faraday::VERSION) >= Gem::Version.new("2.0")
-
 require_relative "openai/http"
 require_relative "openai/client"
 require_relative "openai/files"
@@ -31,13 +30,7 @@ def call(env)
       @app.call(env)
     rescue Faraday::Error => e
       raise e unless e.response.is_a?(Hash)
-
-      logger = Logger.new($stdout)
-      logger.formatter = proc do |_severity, _datetime, _progname, msg|
-        "\033[31mOpenAI HTTP Error (spotted in ruby-openai #{VERSION}): #{msg}\n\033[0m"
-      end
-      logger.error(e.response[:body])
-
+      OpenAI.log_message("OpenAI HTTP Error", e.response[:body], :error)
       raise e
     end
   end
@@ -73,25 +66,37 @@ def initialize
 
   class << self
     attr_writer :configuration
-  end
 
-  def self.configuration
-    @configuration ||= OpenAI::Configuration.new
-  end
+    def configuration
+      @configuration ||= OpenAI::Configuration.new
+    end
 
-  def self.configure
-    yield(configuration)
-  end
+    def configure
+      yield(configuration)
+    end
 
-  # Estimate the number of tokens in a string, using the rules of thumb from OpenAI:
-  # https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
-  def self.rough_token_count(content = "")
-    raise ArgumentError, "rough_token_count requires a string" unless content.is_a? String
-    return 0 if content.empty?
+    # Estimate the number of tokens in a string, using the rules of thumb from OpenAI:
+    # https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
+    def rough_token_count(content = "")
+      raise ArgumentError, "rough_token_count requires a string" unless content.is_a? String
+      return 0 if content.empty?
+      count_by_chars = content.size / 4.0
+      count_by_words = content.split.size * 4.0 / 3
+      estimate = ((count_by_chars + count_by_words) / 2.0).round
+      [1, estimate].max
+    end
 
-    count_by_chars = content.size / 4.0
-    count_by_words = content.split.size * 4.0 / 3
-    estimate = ((count_by_chars + count_by_words) / 2.0).round
-    [1, estimate].max
+    # Log a message with appropriate formatting
+    # @param prefix [String] Prefix to add to the message
+    # @param message [String] The message to log
+    # @param level [Symbol] The log level (:error, :warn, etc.)
+    def log_message(prefix, message, level = :warn)
+      color = level == :error ? "\033[31m" : "\033[33m"
+      logger = Logger.new($stdout)
+      logger.formatter = proc do |_severity, _datetime, _progname, msg|
+        "#{color}#{prefix} (spotted in ruby-openai #{VERSION}): #{msg}\n\033[0m"
+      end
+      logger.send(level, message)
+    end
   end
 end

lib/openai/files.rb

@@ -20,9 +20,7 @@ def list(parameters: {})
     def upload(parameters: {})
       file_input = parameters[:file]
       file = prepare_file_input(file_input: file_input)
-
       validate(file: file, purpose: parameters[:purpose], file_input: file_input)
-
       @client.multipart_post(
         path: "/files",
         parameters: parameters.merge(file: file)
@@ -57,8 +55,12 @@ def prepare_file_input(file_input:)
 
     def validate(file:, purpose:, file_input:)
       raise ArgumentError, "`file` is required" if file.nil?
+
       unless PURPOSES.include?(purpose)
-        raise ArgumentError, "`purpose` must be one of `#{PURPOSES.join(',')}`"
+        filename = file_input.is_a?(String) ? File.basename(file_input) : "uploaded file"
+        message = "The purpose '#{purpose}' for file '#{filename}' is not in the known purpose "
+        message += "list: #{PURPOSES.join(', ')}."
+        OpenAI.log_message("Warning", message, :warn)
       end
 
       validate_jsonl(file: file) if file_input.is_a?(String) && file_input.end_with?(".jsonl")

spec/openai/client/files_spec.rb

@@ -21,10 +21,21 @@
       end
 
       context "with an invalid purpose" do
-        let(:cassette_label) { "unused" }
+        let(:cassette_label) { "invalid purpose" }
         let(:upload_purpose) { "invalid" }
 
-        it { expect { upload }.to raise_error(ArgumentError) }
+        it "logs a warning" do
+          expected_message = "The purpose 'invalid' for file 'sentiment.jsonl' is not in the known "
+          expected_message += "purpose list: #{OpenAI::Files::PURPOSES.join(', ')}."
+
+          expect(OpenAI).to receive(:log_message)
+            .with("Warning", expected_message, :warn)
+            .and_call_original
+
+          allow_any_instance_of(OpenAI::Client).to receive(:multipart_post).and_return({})
+
+          upload
+        end
       end
 
       context "with a `File` instance content" do