Make token limit configuration for Ollama optional

This way Ollama can decide what the default should be (if any)
quarkiverse · May 15, 2024 · f64c43b · f64c43b
1 parent 62c20d5
commit f64c43b
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 6 deletions.
diff --git a/ollama/runtime/src/main/java/io/quarkiverse/langchain4j/ollama/runtime/OllamaRecorder.java b/ollama/runtime/src/main/java/io/quarkiverse/langchain4j/ollama/runtime/OllamaRecorder.java
@@ -36,8 +36,11 @@ public Supplier<ChatLanguageModel> chatModel(LangChain4jOllamaConfig runtimeConf
             Options.Builder optionsBuilder = Options.builder()
                     .temperature(chatModelConfig.temperature())
                     .topK(chatModelConfig.topK())
-                    .topP(chatModelConfig.topP())
-                    .numPredict(chatModelConfig.numPredict());
+                    .topP(chatModelConfig.topP());
+
+            if (chatModelConfig.numPredict().isPresent()) {
+                optionsBuilder.numPredict(chatModelConfig.numPredict().getAsInt());
+            }
 
             if (chatModelConfig.stop().isPresent()) {
                 optionsBuilder.stop(chatModelConfig.stop().get());
@@ -123,8 +126,11 @@ public Supplier<StreamingChatLanguageModel> streamingChatModel(LangChain4jOllama
             Options.Builder optionsBuilder = Options.builder()
                     .temperature(chatModelConfig.temperature())
                     .topK(chatModelConfig.topK())
-                    .topP(chatModelConfig.topP())
-                    .numPredict(chatModelConfig.numPredict());
+                    .topP(chatModelConfig.topP());
+
+            if (chatModelConfig.numPredict().isPresent()) {
+                optionsBuilder.numPredict(chatModelConfig.numPredict().getAsInt());
+            }
 
             if (chatModelConfig.stop().isPresent()) {
                 optionsBuilder.stop(chatModelConfig.stop().get());

diff --git a/...ntime/src/main/java/io/quarkiverse/langchain4j/ollama/runtime/config/ChatModelConfig.java b/...ntime/src/main/java/io/quarkiverse/langchain4j/ollama/runtime/config/ChatModelConfig.java
@@ -2,6 +2,7 @@
 
 import java.util.List;
 import java.util.Optional;
+import java.util.OptionalInt;
 
 import io.quarkus.runtime.annotations.ConfigDocDefault;
 import io.quarkus.runtime.annotations.ConfigGroup;
@@ -20,8 +21,7 @@ public interface ChatModelConfig {
     /**
      * Maximum number of tokens to predict when generating text
      */
-    @WithDefault("128")
-    Integer numPredict();
+    OptionalInt numPredict();
 
     /**
      * Sets the stop sequences to use. When this pattern is encountered the LLM will stop generating text and return