Added SharpToken to examples

microsoft · Sep 14, 2023 · 7b1cc19 · anthonypuppo · Sep 14, 2023 · dmytrostruk
1 parent 75e1b3b
commit 7b1cc19
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 1 deletion.
diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props
@@ -16,6 +16,7 @@
     <PackageVersion Include="NRedisStack" Version="0.9.0" />
     <PackageVersion Include="Pgvector" Version="0.1.3" />
     <PackageVersion Include="Polly" Version="7.2.4" />
+    <PackageVersion Include="SharpToken" Version="1.2.12" />
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="6.0.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Text.Json" Version="6.0.8" />

diff --git a/dotnet/samples/KernelSyntaxExamples/Example55_TextChunker.cs b/dotnet/samples/KernelSyntaxExamples/Example55_TextChunker.cs
@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.SemanticKernel.Text;
+using SharpToken;
 
 // ReSharper disable once InconsistentNaming
 public static class Example55_TextChunker
@@ -69,6 +70,9 @@ private static void WriteParagraphsToConsole(List<string> paragraphs)
 
     private static int CustomTokenCounter(string input)
     {
-        return input.Length / 4;
+        var encoding = GptEncoding.GetEncoding("p50k_base");
+        var tokens = encoding.Encode(input);
+
+        return tokens.Count;
     }
 }
diff --git a/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj b/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj
@@ -26,6 +26,7 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Console" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="Polly" />
+    <PackageReference Include="SharpToken" />
     <PackageReference Include="System.Linq.Async" />
   </ItemGroup>
   <ItemGroup>