Merge pull request #10 from tghamm/feature/token-counter

Claude token counter, version bump, readme update
tghamm · Aug 19, 2023 · b65bb7f · b65bb7f
2 parents 1d6a847 + 427a90b
commit b65bb7f
Show file tree

Hide file tree

Showing 6 changed files with 64,826 additions and 6 deletions.
diff --git a/Anthropic.SDK.Tests/Completions.cs b/Anthropic.SDK.Tests/Completions.cs
@@ -2,6 +2,7 @@
 using Anthropic.SDK.Completions;
 using System.Reflection;
 using Anthropic.SDK.Constants;
+using Anthropic.SDK.Tokens;
 
 namespace Anthropic.SDK.Tests
 {
@@ -27,6 +28,8 @@ public async Task TestClaudeCompletion()
             var response = await client.Completions.GetClaudeCompletionAsync(parameters);
             Assert.IsNotNull(response.Completion);
             Debug.WriteLine(response.Completion);
+            Debug.WriteLine(
+                $@"Tokens Used: Input - {prompt.GetClaudeTokenCount()}. Output - {response.Completion.GetClaudeTokenCount()}.");
         }
 
         [TestMethod]
@@ -43,13 +46,16 @@ public async Task TestClaudeStreamingCompletion()
                 Temperature = 0.0m,
                 StopSequences = new[] { AnthropicSignals.HumanSignal },
                 Stream = true,
-                Model = AnthropicModels.ClaudeInstant_v1_2
+                Model = AnthropicModels.Claude_v2
             };
-
+            var totalOutput = string.Empty;
             await foreach (var res in client.Completions.StreamClaudeCompletionAsync(parameters))
             {
                 Debug.Write(res.Completion);
+                totalOutput += res.Completion;
             }
+            Debug.WriteLine(
+                $@"Tokens Used: Input - {prompt.GetClaudeTokenCount()}. Output - {totalOutput.GetClaudeTokenCount()}.");
         }
     }
 }
diff --git a/Anthropic.SDK/Anthropic.SDK.csproj b/Anthropic.SDK/Anthropic.SDK.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
       <TargetFrameworks>netstandard2.0;net6.0</TargetFrameworks>
@@ -17,9 +17,9 @@
 		  Adds new model constants.
 	  </PackageReleaseNotes>
 	  <PackageId>Anthropic.SDK</PackageId>
-	  <Version>1.1.2</Version>
-	  <AssemblyVersion>1.1.2.0</AssemblyVersion>
-	  <FileVersion>1.1.2.0</FileVersion>
+	  <Version>1.2.0</Version>
+	  <AssemblyVersion>1.2.0.0</AssemblyVersion>
+	  <FileVersion>1.2.0.0</FileVersion>
 	  <GenerateDocumentationFile>True</GenerateDocumentationFile>
 	  <PackageReadmeFile>README.md</PackageReadmeFile>
 	  <ProduceReferenceAssembly>True</ProduceReferenceAssembly>
@@ -34,6 +34,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
+    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.20.1" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
 	<PackageReference Include="System.Text.Json" Version="6.0.7" />
@@ -44,4 +45,12 @@
 			<PackagePath>\</PackagePath>
 		</None>
 	</ItemGroup>
+	<ItemGroup>
+	  <None Update="Tokens\anthropic_merges.txt">
+	    <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+	  </None>
+	  <None Update="Tokens\anthropic_vocab.json">
+	    <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+	  </None>
+	</ItemGroup>
 </Project>
diff --git a/Anthropic.SDK/Tokens/TokenHelper.cs b/Anthropic.SDK/Tokens/TokenHelper.cs
@@ -0,0 +1,35 @@
+using Microsoft.ML.Tokenizers;
+using System;
+using System.Collections.Generic;
+using System.Dynamic;
+using System.IO;
+using System.Reflection;
+using System.Text;
+
+namespace Anthropic.SDK.Tokens
+{
+    /// <summary>
+    /// Helper Class to Get Token Counts
+    /// </summary>
+    public static class TokenHelper
+    {
+        private static readonly Tokenizer Tokenizer;
+        static TokenHelper()
+        {
+            var vocabFilePath =  Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tokens", "anthropic_vocab.json");
+            var mergesFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tokens", "anthropic_merges.txt");
+            Tokenizer = new Tokenizer(new Bpe(vocabFilePath, mergesFilePath, null, null), RobertaPreTokenizer.Instance);
+        }
+
+
+        /// <summary>
+        /// Gets Token Count of Input String
+        /// </summary>
+        /// <param name="input"></param>
+        /// <returns></returns>
+        public static int GetClaudeTokenCount(this string input)
+        {
+            return Tokenizer.Encode(input).Tokens.Count;
+        }
+    }
+}