Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Claude token counter, version bump, readme update #10

Merged
merged 1 commit into from
Aug 19, 2023
Merged

Claude token counter, version bump, readme update #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions Anthropic.SDK.Tests/Completions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using Anthropic.SDK.Completions;
using System.Reflection;
using Anthropic.SDK.Constants;
using Anthropic.SDK.Tokens;

namespace Anthropic.SDK.Tests
{
Expand All @@ -27,6 +28,8 @@ public async Task TestClaudeCompletion()
var response = await client.Completions.GetClaudeCompletionAsync(parameters);
Assert.IsNotNull(response.Completion);
Debug.WriteLine(response.Completion);
Debug.WriteLine(
$@"Tokens Used: Input - {prompt.GetClaudeTokenCount()}. Output - {response.Completion.GetClaudeTokenCount()}.");
}

[TestMethod]
Expand All @@ -43,13 +46,16 @@ public async Task TestClaudeStreamingCompletion()
Temperature = 0.0m,
StopSequences = new[] { AnthropicSignals.HumanSignal },
Stream = true,
Model = AnthropicModels.ClaudeInstant_v1_2
Model = AnthropicModels.Claude_v2
};

var totalOutput = string.Empty;
await foreach (var res in client.Completions.StreamClaudeCompletionAsync(parameters))
{
Debug.Write(res.Completion);
totalOutput += res.Completion;
}
Debug.WriteLine(
$@"Tokens Used: Input - {prompt.GetClaudeTokenCount()}. Output - {totalOutput.GetClaudeTokenCount()}.");
}
}
}
17 changes: 13 additions & 4 deletions Anthropic.SDK/Anthropic.SDK.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFrameworks>netstandard2.0;net6.0</TargetFrameworks>
Expand All @@ -17,9 +17,9 @@
Adds new model constants.
</PackageReleaseNotes>
<PackageId>Anthropic.SDK</PackageId>
<Version>1.1.2</Version>
<AssemblyVersion>1.1.2.0</AssemblyVersion>
<FileVersion>1.1.2.0</FileVersion>
<Version>1.2.0</Version>
<AssemblyVersion>1.2.0.0</AssemblyVersion>
<FileVersion>1.2.0.0</FileVersion>
<GenerateDocumentationFile>True</GenerateDocumentationFile>
<PackageReadmeFile>README.md</PackageReadmeFile>
<ProduceReferenceAssembly>True</ProduceReferenceAssembly>
Expand All @@ -34,6 +34,7 @@

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
<PackageReference Include="Microsoft.ML.Tokenizers" Version="0.20.1" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
<PackageReference Include="System.Text.Json" Version="6.0.7" />
Expand All @@ -44,4 +45,12 @@
<PackagePath>\</PackagePath>
</None>
</ItemGroup>
<ItemGroup>
<None Update="Tokens\anthropic_merges.txt">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="Tokens\anthropic_vocab.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
35 changes: 35 additions & 0 deletions Anthropic.SDK/Tokens/TokenHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
using Microsoft.ML.Tokenizers;
using System;
using System.Collections.Generic;
using System.Dynamic;
using System.IO;
using System.Reflection;
using System.Text;

namespace Anthropic.SDK.Tokens
{
/// <summary>
/// Helper Class to Get Token Counts
/// </summary>
public static class TokenHelper
{
private static readonly Tokenizer Tokenizer;
static TokenHelper()
{
var vocabFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tokens", "anthropic_vocab.json");
var mergesFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tokens", "anthropic_merges.txt");
Tokenizer = new Tokenizer(new Bpe(vocabFilePath, mergesFilePath, null, null), RobertaPreTokenizer.Instance);
}


/// <summary>
/// Gets Token Count of Input String
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public static int GetClaudeTokenCount(this string input)
{
return Tokenizer.Encode(input).Tokens.Count;
}
}
}
Loading
Loading