Merge branch 'main' into main

patterns-ai-core · Jun 1, 2024 · 9208f09 · 9208f09
2 parents 62fd701 + 5b563e4
commit 9208f09
Show file tree

Hide file tree

Showing 4 changed files with 67 additions and 72 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -7,31 +7,30 @@ PATH
       json-schema (~> 4)
       matrix
       pragmatic_segmenter (~> 0.3.0)
-      tiktoken_ruby (~> 0.0.9)
       to_bool (~> 2.0.0)
       zeitwerk (~> 2.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    Ascii85 (1.1.0)
-    actionpack (7.1.3.2)
-      actionview (= 7.1.3.2)
-      activesupport (= 7.1.3.2)
+    Ascii85 (1.1.1)
+    actionpack (7.1.3.3)
+      actionview (= 7.1.3.3)
+      activesupport (= 7.1.3.3)
       nokogiri (>= 1.8.5)
       racc
       rack (>= 2.2.4)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    actionview (7.1.3.2)
-      activesupport (= 7.1.3.2)
+    actionview (7.1.3.3)
+      activesupport (= 7.1.3.3)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    activesupport (7.1.3.2)
+    activesupport (7.1.3.3)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -50,20 +49,20 @@ GEM
       faraday-multipart (>= 1)
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.890.0)
-    aws-sdk-bedrockruntime (1.5.0)
-      aws-sdk-core (~> 3, >= 3.191.0)
+    aws-partitions (1.937.0)
+    aws-sdk-bedrockruntime (1.9.0)
+      aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.191.1)
+    aws-sdk-core (3.196.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
       aws-sigv4 (~> 1.8)
       jmespath (~> 1, >= 1.6.1)
     aws-sigv4 (1.8.0)
       aws-eventstream (~> 1, >= 1.0.2)
-    baran (0.1.10)
+    baran (0.1.12)
     base64 (0.2.0)
-    bigdecimal (3.1.6)
+    bigdecimal (3.1.8)
     builder (3.2.4)
     byebug (11.1.3)
     childprocess (5.0.0)
@@ -74,9 +73,9 @@ GEM
     cohere-ruby (0.9.10)
       faraday (>= 2.0.1, < 3.0)
     colorize (1.1.0)
-    concurrent-ruby (1.2.3)
+    concurrent-ruby (1.3.1)
     connection_pool (2.4.1)
-    crack (0.4.6)
+    crack (1.0.0)
       bigdecimal
       rexml
     crass (1.0.6)
@@ -106,7 +105,7 @@ GEM
       concurrent-ruby (~> 1.0)
       dry-core (~> 1.0, < 2)
       zeitwerk (~> 2.6)
-    dry-schema (1.13.3)
+    dry-schema (1.13.4)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 1.0, >= 1.0.1)
       dry-core (~> 1.0, < 2)
@@ -154,7 +153,7 @@ GEM
       multipart-post (~> 2)
     faraday-net_http (3.1.0)
       net-http
-    faraday-retry (2.2.0)
+    faraday-retry (2.2.1)
       faraday (~> 2.0)
     faraday-typhoeus (1.1.0)
       faraday (~> 2.0)
@@ -175,8 +174,9 @@ GEM
     graphlient (0.7.0)
       faraday (~> 2.0)
       graphql-client
-    graphql (2.2.8)
-    graphql-client (0.20.0)
+    graphql (2.3.4)
+      base64
+    graphql-client (0.22.0)
       activesupport (>= 3.0)
       graphql (>= 1.13.0)
     hashdiff (1.1.0)
@@ -188,16 +188,16 @@ GEM
       multi_xml (>= 0.5.2)
     hugging-face (0.3.5)
       faraday (>= 1.0)
-    i18n (1.14.1)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     io-console (0.7.2)
-    irb (1.11.2)
-      rdoc
+    irb (1.13.1)
+      rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     jmespath (1.6.2)
-    json (2.7.1)
-    json-schema (4.0.0)
+    json (2.7.2)
+    json-schema (4.3.0)
       addressable (>= 2.8)
     jwt (2.8.1)
       base64
@@ -213,30 +213,31 @@ GEM
       net-pop
       net-smtp
     matrix (0.4.2)
-    method_source (1.0.0)
+    method_source (1.1.0)
     milvus (0.9.2)
       faraday (>= 2.0.1, < 3)
     mini_mime (1.1.5)
     mini_portile2 (2.8.6)
-    minitest (5.22.2)
+    minitest (5.23.1)
     mistral-ai (1.2.0)
       event_stream_parser (~> 1.0)
       faraday (~> 2.9)
       faraday-typhoeus (~> 1.1)
     multi_json (1.15.0)
-    multi_xml (0.6.0)
-    multipart-post (2.4.0)
+    multi_xml (0.7.1)
+      bigdecimal (~> 3.1)
+    multipart-post (2.4.1)
     mutex_m (0.2.0)
     net-http (0.4.1)
       uri
-    net-imap (0.4.10)
+    net-imap (0.4.11)
       date
       net-protocol
     net-pop (0.1.2)
       net-protocol
     net-protocol (0.2.2)
       timeout
-    net-smtp (0.4.0.1)
+    net-smtp (0.5.0)
       net-protocol
     nokogiri (1.16.5)
       mini_portile2 (~> 2.8.2)
@@ -259,7 +260,7 @@ GEM
     os (1.1.4)
     paco (0.2.3)
     parallel (1.24.0)
-    parser (3.3.0.5)
+    parser (3.3.2.0)
       ast (~> 2.4.1)
       racc
     pdf-reader (2.12.0)
@@ -268,7 +269,7 @@ GEM
       hashery (~> 2.0)
       ruby-rc4
       ttfunk
-    pg (1.5.4)
+    pg (1.5.6)
     pgvector (0.2.2)
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
@@ -288,11 +289,11 @@ GEM
       pry (>= 0.13, < 0.15)
     psych (5.1.2)
       stringio
-    public_suffix (5.0.4)
+    public_suffix (5.0.5)
     qdrant-ruby (0.9.7)
       faraday (>= 2.0.1, < 3)
-    racc (1.7.3)
-    rack (3.0.9.1)
+    racc (1.8.0)
+    rack (3.0.11)
     rack-session (2.0.0)
       rack (>= 3.0.0)
     rack-test (2.1.0)
@@ -307,22 +308,22 @@ GEM
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
       nokogiri (~> 1.14)
-    railties (7.1.3.2)
-      actionpack (= 7.1.3.2)
-      activesupport (= 7.1.3.2)
+    railties (7.1.3.3)
+      actionpack (= 7.1.3.3)
+      activesupport (= 7.1.3.3)
       irb
       rackup (>= 1.0.0)
       rake (>= 12.2)
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
-    rake (13.1.0)
+    rake (13.2.1)
     rb_sys (0.9.87)
     rdiscount (2.2.7.3)
-    rdoc (6.6.3.1)
+    rdoc (6.7.0)
       psych (>= 4.0.0)
-    regexp_parser (2.9.0)
-    reline (0.4.2)
+    regexp_parser (2.9.2)
+    reline (0.5.8)
       io-console (~> 0.5)
     replicate-ruby (0.2.3)
       addressable
@@ -348,33 +349,33 @@ GEM
     rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.0)
-    rubocop (1.60.2)
+    rspec-support (3.13.1)
+    rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.30.0)
-      parser (>= 3.2.1.0)
-    rubocop-performance (1.20.2)
+    rubocop-ast (1.31.3)
+      parser (>= 3.3.1.0)
+    rubocop-performance (1.21.0)
       rubocop (>= 1.48.1, < 2.0)
-      rubocop-ast (>= 1.30.0, < 2.0)
-    ruby-next (1.0.1)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    ruby-next (1.0.3)
       paco (~> 0.2)
       require-hooks (~> 0.2)
-      ruby-next-core (= 1.0.1)
+      ruby-next-core (= 1.0.3)
       ruby-next-parser (>= 3.2.2.0)
       unparser (~> 0.6.0)
-    ruby-next-core (1.0.1)
+    ruby-next-core (1.0.3)
     ruby-next-parser (3.2.2.0)
       parser (>= 3.0.3.1)
     ruby-ole (1.2.13.1)
@@ -396,23 +397,23 @@ GEM
     spreadsheet (1.3.1)
       bigdecimal
       ruby-ole
-    standard (1.34.0)
+    standard (1.35.0.1)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.60)
+      rubocop (~> 1.62)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.3)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.3.1)
+    standard-performance (1.4.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.20.2)
+      rubocop-performance (~> 1.21.0)
     standardrb (1.0.1)
       standard
     stringio (3.1.0)
     strscan (3.1.0)
-    thor (1.3.0)
+    thor (1.3.1)
     tiktoken_ruby (0.0.9)
       rb_sys (= 0.9.87)
     tiktoken_ruby (0.0.9-aarch64-linux)
@@ -441,15 +442,15 @@ GEM
     weaviate-ruby (0.8.10)
       faraday (>= 2.0.1, < 3.0)
       graphlient (~> 0.7.0)
-    webmock (3.19.1)
+    webmock (3.23.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.8.1)
     wikipedia-client (1.17.0)
       addressable (~> 2.7)
     yard (0.9.36)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.15)
 
 PLATFORMS
   aarch64-linux
@@ -502,11 +503,12 @@ DEPENDENCIES
   safe_ruby (~> 1.0.4)
   sequel (~> 5.68.0)
   standardrb
+  tiktoken_ruby (~> 0.0.9)
   vcr
   weaviate-ruby (~> 0.8.10)
   webmock
   wikipedia-client (~> 1.17.0)
   yard (~> 0.9.34)
 
 BUNDLED WITH
-   2.5.3
+   2.5.11
diff --git a/README.md b/README.md
@@ -420,6 +420,7 @@ Assistants are Agent-like objects that leverage helpful instructions, LLMs, tool
 ### Demos
 1. [Building an AI Assistant that operates a simulated E-commerce Store](https://www.loom.com/share/83aa4fd8dccb492aad4ca95da40ed0b2)
 2. [New Langchain.rb Assistants interface](https://www.loom.com/share/e883a4a49b8746c1b0acf9d58cf6da36)
+3. [Langchain.rb Assistant demo with NewsRetriever and function calling on Gemini](https://youtu.be/-ieyahrpDpM&t=1477s) - [code](https://github.com/palladius/gemini-news-crawler)
 
 ### Creating an Assistant
 1. Instantiate an LLM of your choice

diff --git a/langchain.gemspec b/langchain.gemspec
@@ -29,7 +29,6 @@ Gem::Specification.new do |spec|
   # Not sure if we should require this as it only applies to OpenAI usecase.
   spec.add_dependency "baran", "~> 0.1.9"
   spec.add_dependency "colorize", "~> 1.1.0"
-  spec.add_dependency "tiktoken_ruby", "~> 0.0.9"
   spec.add_dependency "json-schema", "~> 4"
   spec.add_dependency "zeitwerk", "~> 2.5"
   spec.add_dependency "pragmatic_segmenter", "~> 0.3.0"
@@ -54,6 +53,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "elasticsearch", "~> 8.2.0"
   spec.add_development_dependency "epsilla-ruby", "~> 0.0.4"
   spec.add_development_dependency "eqn", "~> 1.6.5"
+  spec.add_development_dependency "faraday"
   spec.add_development_dependency "googleauth"
   spec.add_development_dependency "google_palm_api", "~> 0.1.3"
   spec.add_development_dependency "google_search_results", "~> 2.0.0"
@@ -78,6 +78,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "sequel", "~> 5.68.0"
   spec.add_development_dependency "weaviate-ruby", "~> 0.8.10"
   spec.add_development_dependency "wikipedia-client", "~> 1.17.0"
-  spec.add_development_dependency "faraday"
   spec.add_development_dependency "power_point_pptx", "~> 0.1.0"
+  spec.add_development_dependency "tiktoken_ruby", "~> 0.0.9"
 end
diff --git a/lib/langchain/llm/openai.rb b/lib/langchain/llm/openai.rb
@@ -26,8 +26,6 @@ class OpenAI < Base
       "text-embedding-3-small" => 1536
     }.freeze
 
-    LENGTH_VALIDATOR = Langchain::Utils::TokenLength::OpenAIValidator
-
     attr_reader :defaults
 
     # Initialize an OpenAI LLM instance
@@ -82,8 +80,6 @@ def embed(
         parameters[:dimensions] = EMBEDDING_SIZES[model]
       end
 
-      validate_max_tokens(text, parameters[:model])
-
       response = with_api_error_handling do
         client.embeddings(parameters: parameters)
       end
@@ -177,10 +173,6 @@ def with_api_error_handling
       response
     end
 
-    def validate_max_tokens(messages, model, max_tokens = nil)
-      LENGTH_VALIDATOR.validate_max_tokens!(messages, model, max_tokens: max_tokens, llm: self)
-    end
-
     def response_from_chunks
       grouped_chunks = @response_chunks.group_by { |chunk| chunk.dig("choices", 0, "index") }
       final_choices = grouped_chunks.map do |index, chunks|