Add more tracer kwargs for logging

src/llm_interface.jl

@@ -356,6 +356,10 @@ It can be composed with any other schema, eg, `TracerSchema` to save additional
 Set environment variable `LOG_DIR` to the directory where you want to save the conversation (see `?PREFERENCES`).
 Conversations are named by the hash of the first message in the conversation to naturally group subsequent conversations together.
 
+If you need to provide logging directory of the file name dynamically, you can provide the following arguments to `tracer_kwargs`:
+- `log_dir` - used as the directory to save the log into when provided. Defaults to `LOG_DIR` if not provided.
+- `log_file_path` - used as the file name to save the log into when provided. This value overrules the `log_dir` and `LOG_DIR` if provided.
+
 To use it automatically, re-register the models you use with the schema wrapped in `SaverSchema`
 
 See also: `meta`, `unwrap`, `TracerSchema`, `initialize_tracer`, `finalize_tracer`
@@ -367,7 +371,7 @@ using PromptingTools: TracerSchema, OpenAISchema, SaverSchema
 
 wrap_schema = OpenAISchema() |> TracerSchema |> SaverSchema
 conv = aigenerate(wrap_schema,:BlankSystemUser; system="You're a French-speaking assistant!",
-    user="Say hi!"; model="gpt-4", api_kwargs=(;temperature=0.1), return_all=true)
+    user="Say hi!", model="gpt-4", api_kwargs=(;temperature=0.1), return_all=true)
 
 # conv is a vector of messages that will be saved to a JSON together with metadata about the template and api_kwargs
 ```
@@ -377,6 +381,13 @@ If you wanted to enable this automatically for models you use, you can do it lik
 PT.register_model!(; name= "gpt-3.5-turbo", schema=OpenAISchema() |> TracerSchema |> SaverSchema)
 ```
 Any subsequent calls `model="gpt-3.5-turbo"` will automatically capture metadata and save the conversation to the disk.
+
+To provide logging file path explicitly, use the `tracer_kwargs`:
+```julia
+conv = aigenerate(wrap_schema,:BlankSystemUser; system="You're a French-speaking assistant!",
+    user="Say hi!", model="gpt-4", api_kwargs=(;temperature=0.1), return_all=true,
+    tracer_kwargs=(; log_file_path="my_logs/my_log.json"))
+```
 """
 struct SaverSchema <: AbstractTracerSchema
     schema::AbstractPromptSchema

src/llm_tracer.jl

@@ -52,6 +52,10 @@ function initialize_tracer(
         prompt::ALLOWED_PROMPT_TYPE = "", api_kwargs::NamedTuple = NamedTuple(),
         kwargs...)
     meta = Dict{Symbol, Any}(k => v for (k, v) in pairs(api_kwargs))
+    if haskey(tracer_kwargs, :meta)
+        ## merge with the provided metadata
+        meta = merge(meta, tracer_kwargs.meta)
+    end
     if haskey(kwargs, :_tracer_template)
         tpl = get(kwargs, :_tracer_template, nothing)
         meta[:template_name] = tpl.name
@@ -60,8 +64,9 @@ function initialize_tracer(
             meta[:template_version] = metadata[1].version
         end
     end
-    return (; time_sent = now(), model, meta,
-        tracer_kwargs...)
+    ## provide meta as last to make sure it's not overwriten by kwargs
+    return (; time_sent = now(), model,
+        tracer_kwargs..., meta)
 end
 
 function finalize_tracer(
@@ -117,9 +122,13 @@ end
 
 Finalizes the calltracer by saving the provided conversation `msg_or_conv` to the disk.
 
-Path is `LOG_DIR/conversation__<first_msg_hash>__<time_received_str>.json`, 
+Default path is `LOG_DIR/conversation__<first_msg_hash>__<time_received_str>.json`, 
  where `LOG_DIR` is set by user preferences or ENV variable (defaults to `log/` in current working directory).
 
+If you want to change the logging directory or the exact file name to log with, you can provide the following arguments to `tracer_kwargs`:
+- `log_dir` - used as the directory to save the log into when provided. Defaults to `LOG_DIR` if not provided.
+- `log_file_path` - used as the file name to save the log into when provided. This value overrules the `log_dir` and `LOG_DIR` if provided.
+
 It can be composed with `TracerSchema` to also attach necessary metadata (see below).
 
 # Example
@@ -145,13 +154,20 @@ function finalize_tracer(
            convert(Vector{AbstractMessage}, msg_or_conv) :
            AbstractMessage[msg_or_conv]
 
-    # Log the conversation to disk, save by hash of the first convo message + timestamp
-    first_msg_hash = hash(first(conv).content)
-    time_received_str = Dates.format(
-        time_received, dateformat"YYYYmmdd_HHMMSS")
-    path = joinpath(
-        LOG_DIR,
-        "conversation__$(first_msg_hash)__$(time_received_str).json")
+    # Log the conversation to disk, 
+    log_dir = get(tracer, :log_dir, LOG_DIR)
+    path = if haskey(tracer, :log_file_path)
+        ## take the provided log file path
+        tracer.log_file_path
+    else
+        ## save by hash of the first convo message + timestamp
+        first_msg_hash = hash(first(conv).content)
+        time_received_str = Dates.format(
+            time_received, dateformat"YYYYmmdd_HHMMSS")
+        path = joinpath(
+            log_dir,
+            "conversation__$(first_msg_hash)__$(time_received_str).json")
+    end
     mkpath(dirname(path))
     save_conversation(path, conv)
     return is_vector ? conv : first(conv)

test/llm_tracer.jl

@@ -148,12 +148,14 @@ end
     # Real generation API
     schema1 = TestEchoOpenAISchema(; response, status = 200) |> TracerSchema
     msg = aigenerate(
-        schema1, "Hello World"; model = "xyz", tracer_kwargs = (; thread_id = :ABC1))
+        schema1, "Hello World"; model = "xyz",
+        tracer_kwargs = (; thread_id = :ABC1, meta = Dict(:meta_key => "meta_value")))
     @test istracermessage(msg)
     @test unwrap(msg) |> isaimessage
     @test msg.content == "Hello!"
     @test msg.model == "xyz"
     @test msg.thread_id == :ABC1
+    @test msg.meta[:meta_key] == "meta_value"
 
     msg = aigenerate(schema1, :BlankSystemUser; system = "abc", user = "xyz")
     @test istracermessage(msg)
@@ -180,6 +182,21 @@ end
     @test loaded_msg.thread_id == :ABC1
     ## clean up
     isfile(fn) && rm(fn)
+
+    ## Use kwargs to define save path
+    file, _ = mktemp()
+    msgs = [SystemMessage("Test message 1"), UserMessage("Hello World")]
+    msg = aigenerate(
+        schema2, msgs; model = "xyz", tracer_kwargs = (;
+            thread_id = :ABC1, log_file_path = file))
+    @test istracermessage(msg)
+    @test isfile(file)
+    load_conv = PT.load_conversation(file)
+    @test length(load_conv) == 3
+    loaded_msg = load_conv[end]
+    @test unwrap(loaded_msg) |> isaimessage
+    @test loaded_msg.content == "Hello!"
+    isfile(fn) && rm(fn)
 end
 
 @testset "aiembed-Tracer" begin