Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding together.ai support #280

Merged
merged 4 commits into from
May 18, 2024
Merged

Adding together.ai support #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a84eb4d
first commit
canbal May 15, 2024
4456a53
some cleanup
canbal May 15, 2024
ccf2b61
linter changes
canbal May 15, 2024
629ee43
Merge branch 'moremodels' into cbal/together_support
ianarawjo May 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions chainforge/react-server/src/GlobalSettingsModal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,12 @@ const GlobalSettingsModal = forwardRef<GlobalSettingsModalRef, object>(
{...form.getInputProps("AlephAlpha")}
/>
<br />
<TextInput
label="Together API Key"
placeholder="Paste your Together API key here"
{...form.getInputProps("Together")}
/>
<br />

<Divider
my="xs"
Expand Down
34 changes: 14 additions & 20 deletions chainforge/react-server/src/LLMListComponent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import ModelSettingsModal, {
} from "./ModelSettingsModal";
import { getDefaultModelSettings } from "./ModelSettingSchemas";
import useStore, { initLLMProviders, initLLMProviderMenu } from "./store";
import { Dict, JSONCompatible, LLMSpec } from "./backend/typing";
import { Dict, JSONCompatible, LLMGroup, LLMSpec } from "./backend/typing";
import { useContextMenu } from "mantine-contextmenu";
import { ContextMenuItemOptions } from "mantine-contextmenu/dist/types";

Expand Down Expand Up @@ -430,31 +430,25 @@ export const LLMListContainer = forwardRef<
);

const menuItems = useMemo(() => {
const res: ContextMenuItemOptions[] = [];
const initModels: Set<string> = new Set<string>();
for (const item of initLLMProviderMenu) {
if (!("group" in item)) {
const convert = (item: LLMSpec | LLMGroup): ContextMenuItemOptions => {
if ("group" in item) {
return {
key: item.group,
title: `${item.emoji} ${item.group}`,
items: item.items.map(convert),
};
} else {
initModels.add(item.base_model);
res.push({
return {
key: item.model,
title: `${item.emoji} ${item.name}`,
onClick: () => handleSelectModel(item.base_model),
});
} else {
res.push({
key: item.group,
title: `${item.emoji} ${item.group}`,
items: item.items.map((k) => {
initModels.add(k.base_model);
return {
key: k.model,
title: `${k.emoji} ${k.name}`,
onClick: () => handleSelectModel(k.base_model),
};
}),
});
};
}
}
};
const res = initLLMProviderMenu.map(convert);

for (const item of AvailableLLMs) {
if (initModels.has(item.base_model)) {
continue;
Expand Down
157 changes: 157 additions & 0 deletions chainforge/react-server/src/ModelSettingSchemas.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2036,6 +2036,160 @@ const BedrockLlama2ChatSettings: ModelSettingsDict = {
},
};

export const TogetherChatSettings: ModelSettingsDict = {
fullName: "Together Chat",
schema: {
type: "object",
required: ["shortname"],
properties: {
shortname: {
type: "string",
title: "Nickname",
description:
"Unique identifier to appear in ChainForge. Keep it short.",
default: "TogetherChat",
},
model: {
type: "string",
title: "model",
description:
"Select a version of Together model to query. For more details on the differences, see the Together API documentation.",
enum: [
"zero-one-ai/Yi-34B-Chat",
"allenai/OLMo-7B-Instruct",
"allenai/OLMo-7B-Twin-2T",
"allenai/OLMo-7B",
"Austism/chronos-hermes-13b",
"cognitivecomputations/dolphin-2.5-mixtral-8x7b",
"databricks/dbrx-instruct",
"deepseek-ai/deepseek-coder-33b-instruct",
"deepseek-ai/deepseek-llm-67b-chat",
"garage-bAInd/Platypus2-70B-instruct",
"google/gemma-2b-it",
"google/gemma-7b-it",
"Gryphe/MythoMax-L2-13b",
"lmsys/vicuna-13b-v1.5",
"lmsys/vicuna-7b-v1.5",
"codellama/CodeLlama-13b-Instruct-hf",
"codellama/CodeLlama-34b-Instruct-hf",
"codellama/CodeLlama-70b-Instruct-hf",
"codellama/CodeLlama-7b-Instruct-hf",
"meta-llama/Llama-2-70b-chat-hf",
"meta-llama/Llama-2-13b-chat-hf",
"meta-llama/Llama-2-7b-chat-hf",
"meta-llama/Llama-3-8b-chat-hf",
"meta-llama/Llama-3-70b-chat-hf",
"mistralai/Mistral-7B-Instruct-v0.1",
"mistralai/Mistral-7B-Instruct-v0.2",
"mistralai/Mixtral-8x7B-Instruct-v0.1",
"mistralai/Mixtral-8x22B-Instruct-v0.1",
"NousResearch/Nous-Capybara-7B-V1p9",
"NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
"NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
"NousResearch/Nous-Hermes-llama-2-7b",
"NousResearch/Nous-Hermes-Llama2-13b",
"NousResearch/Nous-Hermes-2-Yi-34B",
"openchat/openchat-3.5-1210",
"Open-Orca/Mistral-7B-OpenOrca",
"Qwen/Qwen1.5-0.5B-Chat",
"Qwen/Qwen1.5-1.8B-Chat",
"Qwen/Qwen1.5-4B-Chat",
"Qwen/Qwen1.5-7B-Chat",
"Qwen/Qwen1.5-14B-Chat",
"Qwen/Qwen1.5-32B-Chat",
"Qwen/Qwen1.5-72B-Chat",
"Qwen/Qwen1.5-110B-Chat",
"snorkelai/Snorkel-Mistral-PairRM-DPO",
"Snowflake/snowflake-arctic-instruct",
"togethercomputer/alpaca-7b",
"teknium/OpenHermes-2-Mistral-7B",
"teknium/OpenHermes-2p5-Mistral-7B",
"togethercomputer/Llama-2-7B-32K-Instruct",
"togethercomputer/RedPajama-INCITE-Chat-3B-v1",
"togethercomputer/RedPajama-INCITE-7B-Chat",
"togethercomputer/StripedHyena-Nous-7B",
"Undi95/ReMM-SLERP-L2-13B",
"Undi95/Toppy-M-7B",
"WizardLM/WizardLM-13B-V1.2",
"upstage/SOLAR-10.7B-Instruct-v1.0",
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You'll have to add together/ to all of these, too...

],
default: "meta-llama/Llama-2-7b-chat-hf",
},
temperature: {
type: "number",
title: "temperature",
description:
"Amount of randomness injected into the response. Ranges from 0 to 1. Use temp closer to 0 for analytical / multiple choice, and temp closer to 1 for creative and generative tasks.",
default: 1,
minimum: 0,
maximum: 1,
multipleOf: 0.01,
},
max_gen_len: {
type: "integer",
title: "max_tokens",
description:
"The maximum number of tokens to generate for each response.",
default: 1024,
minimum: 1,
},
top_p: {
type: "number",
title: "top_p",
description:
"Does nucleus sampling, in which we compute the cumulative distribution over all the options for each subsequent token in decreasing probability order and cut it off once it reaches a particular probability specified by top_p. Defaults to -1, which disables it. Note that you should either alter temperature or top_p, but not both.",
default: 1,
minimum: 0.01,
maximum: 1,
multipleOf: 0.001,
},
},
},
postprocessors: {
stop_sequences: (str) => {
if (typeof str !== "string" || str.trim().length === 0) return [];
return str
.match(/"((?:[^"\\]|\\.)*)"/g)
?.map((s) => s.substring(1, s.length - 1)); // split on double-quotes but exclude escaped double-quotes inside the group
},
model: (str) => {
if (typeof str !== "string") return str;
return "together/" + str;
},
},

uiSchema: {
"ui:submitButtonOptions": UI_SUBMIT_BUTTON_SPEC,
shortname: {
"ui:autofocus": true,
},
model: {
"ui:help": "Defaults to LlamaChat 13B",
},
temperature: {
"ui:help": "Defaults to 1.0.",
"ui:widget": "range",
},
max_tokens: {
"ui:help": "Defaults to 1024.",
},
num_generations: {
"ui:help": "Defaults to 1.",
},
k: {
"ui:help": "Defaults to 0.",
},
p: {
"ui:help": "Defaults to 1.",
},
stop_sequences: {
"ui:widget": "textarea",
"ui:help": "Defaults to no sequence",
},
},
};

const BedrockLlama3Settings = deepcopy(BedrockLlama2ChatSettings);

BedrockLlama3Settings.schema.properties = {
Expand Down Expand Up @@ -2083,8 +2237,10 @@ export const ModelSettings: Dict<ModelSettingsDict> = {
"br.cohere.command": BedrockCommandTextSettings,
"br.mistral.mistral": MistralSettings,
"br.mistral.mixtral": MixtralSettings,
"br.meta.llama2": MetaLlama2ChatSettings,
"br.meta.llama2": BedrockLlama2ChatSettings,
"br.meta.llama3": BedrockLlama3Settings,
together: TogetherChatSettings,
};

export function getSettingsSchemaForLLM(
Expand All @@ -2103,6 +2259,7 @@ export function getSettingsSchemaForLLM(
[LLMProvider.HuggingFace]: HuggingFaceTextInferenceSettings,
[LLMProvider.Aleph_Alpha]: AlephAlphaLuminousSettings,
[LLMProvider.Ollama]: OllamaSettings,
[LLMProvider.Together]: TogetherChatSettings,
};

if (llm_provider === LLMProvider.Custom) return ModelSettings[llm_name];
Expand Down
63 changes: 63 additions & 0 deletions chainforge/react-server/src/backend/models.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,66 @@ export enum NativeLLM {
Bedrock_Mistral_Mistral = "mistral.mistral-7b-instruct-v0:2",
Bedrock_Mistral_Mistral_Large = "mistral.mistral-large-2402-v1:0",
Bedrock_Mistral_Mixtral = "mistral.mixtral-8x7b-instruct-v0:1",

// Together.ai
Together_ZeroOneAI_01ai_Yi_Chat_34B = "zero-one-ai/Yi-34B-Chat",
Together_AllenAI_OLMo_Instruct_7B = "allenai/OLMo-7B-Instruct",
Together_AllenAI_OLMo_Twin2T_7B = "allenai/OLMo-7B-Twin-2T",
Together_AllenAI_OLMo_7B = "allenai/OLMo-7B",
Together_Austism_Chronos_Hermes_13B = "Austism/chronos-hermes-13b",
Together_cognitivecomputations_Dolphin_2_5_Mixtral_8x7b = "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
Together_databricks_DBRX_Instruct = "databricks/dbrx-instruct",
Together_DeepSeek_Deepseek_Coder_Instruct_33B = "deepseek-ai/deepseek-coder-33b-instruct",
Together_DeepSeek_DeepSeek_LLM_Chat_67B = "deepseek-ai/deepseek-llm-67b-chat",
Together_garagebAInd_Platypus2_Instruct_70B = "garage-bAInd/Platypus2-70B-instruct",
Together_Google_Gemma_Instruct_2B = "google/gemma-2b-it",
Together_Google_Gemma_Instruct_7B = "google/gemma-7b-it",
Together_Gryphe_MythoMaxL2_13B = "Gryphe/MythoMax-L2-13b",
Together_LMSys_Vicuna_v1_5_13B = "lmsys/vicuna-13b-v1.5",
Together_LMSys_Vicuna_v1_5_7B = "lmsys/vicuna-7b-v1.5",
Together_Meta_Code_Llama_Instruct_13B = "codellama/CodeLlama-13b-Instruct-hf",
Together_Meta_Code_Llama_Instruct_34B = "codellama/CodeLlama-34b-Instruct-hf",
Together_Meta_Code_Llama_Instruct_70B = "codellama/CodeLlama-70b-Instruct-hf",
Together_Meta_Code_Llama_Instruct_7B = "codellama/CodeLlama-7b-Instruct-hf",
Together_Meta_LLaMA2_Chat_70B = "meta-llama/Llama-2-70b-chat-hf",
Together_Meta_LLaMA2_Chat_13B = "meta-llama/Llama-2-13b-chat-hf",
Together_Meta_LLaMA2_Chat_7B = "meta-llama/Llama-2-7b-chat-hf",
Together_Meta_LLaMA3_Chat_8B = "meta-llama/Llama-3-8b-chat-hf",
Together_Meta_LLaMA3_Chat_70B = "meta-llama/Llama-3-70b-chat-hf",
Together_mistralai_Mistral_7B_Instruct = "mistralai/Mistral-7B-Instruct-v0.1",
Together_mistralai_Mistral_7B_Instruct_v0_2 = "mistralai/Mistral-7B-Instruct-v0.2",
Together_mistralai_Mixtral8x7B_Instruct_46_7B = "mistralai/Mixtral-8x7B-Instruct-v0.1",
Together_mistralai_Mixtral8x22B_Instruct_141B = "mistralai/Mixtral-8x22B-Instruct-v0.1",
Together_NousResearch_Nous_Capybara_v1_9_7B = "NousResearch/Nous-Capybara-7B-V1p9",
Together_NousResearch_Nous_Hermes_2__Mistral_DPO_7B = "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
Together_NousResearch_Nous_Hermes_2__Mixtral_8x7BDPO_46_7B = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
Together_NousResearch_Nous_Hermes_2__Mixtral_8x7BSFT_46_7B = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
Together_NousResearch_Nous_Hermes_LLaMA2_7B = "NousResearch/Nous-Hermes-llama-2-7b",
Together_NousResearch_Nous_Hermes_Llama2_13B = "NousResearch/Nous-Hermes-Llama2-13b",
Together_NousResearch_Nous_Hermes2_Yi_34B = "NousResearch/Nous-Hermes-2-Yi-34B",
Together_OpenChat_OpenChat_3_5_7B = "openchat/openchat-3.5-1210",
Together_OpenOrca_OpenOrca_Mistral_7B_8K = "Open-Orca/Mistral-7B-OpenOrca",
Together_Qwen_Qwen_1_5_Chat_0_5B = "Qwen/Qwen1.5-0.5B-Chat",
Together_Qwen_Qwen_1_5_Chat_1_8B = "Qwen/Qwen1.5-1.8B-Chat",
Together_Qwen_Qwen_1_5_Chat_4B = "Qwen/Qwen1.5-4B-Chat",
Together_Qwen_Qwen_1_5_Chat_7B = "Qwen/Qwen1.5-7B-Chat",
Together_Qwen_Qwen_1_5_Chat_14B = "Qwen/Qwen1.5-14B-Chat",
Together_Qwen_Qwen_1_5_Chat_32B = "Qwen/Qwen1.5-32B-Chat",
Together_Qwen_Qwen_1_5_Chat_72B = "Qwen/Qwen1.5-72B-Chat",
Together_Qwen_Qwen_1_5_Chat_110B = "Qwen/Qwen1.5-110B-Chat",
Together_SnorkelAI_Snorkel_Mistral_PairRM_DPO_7B = "snorkelai/Snorkel-Mistral-PairRM-DPO",
Together_Snowflake_Snowflake_Arctic_Instruct = "Snowflake/snowflake-arctic-instruct",
Together_Stanford_Alpaca_7B = "togethercomputer/alpaca-7b",
Together_Teknium_OpenHermes2Mistral_7B = "teknium/OpenHermes-2-Mistral-7B",
Together_Teknium_OpenHermes2_5Mistral_7B = "teknium/OpenHermes-2p5-Mistral-7B",
Together_LLaMA27B32KInstruct_7B = "togethercomputer/Llama-2-7B-32K-Instruct",
Together_RedPajamaINCITE_Chat_3B = "togethercomputer/RedPajama-INCITE-Chat-3B-v1",
Together_RedPajamaINCITE_Chat_7B = "togethercomputer/RedPajama-INCITE-7B-Chat",
Together_StripedHyena_Nous_7B = "togethercomputer/StripedHyena-Nous-7B",
Together_Undi95_ReMM_SLERP_L2_13B = "Undi95/ReMM-SLERP-L2-13B",
Together_Undi95_Toppy_M_7B = "Undi95/Toppy-M-7B",
Together_WizardLM_WizardLM_v1_2_13B = "WizardLM/WizardLM-13B-V1.2",
Together_upstage_Upstage_SOLAR_Instruct_v1_11B = "upstage/SOLAR-10.7B-Instruct-v1.0",
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add "together/" in front of all of these? And just extract out the last part in call_together. To not clutter the namespace. Thanks!

(I realize I should've thought harder about this problem when structuring this backend ---it is on my todo list ot restructure it, but it is so central it is not so easy anymore.)

}

export type LLM = string | NativeLLM;
Expand All @@ -138,6 +198,7 @@ export enum LLMProvider {
Aleph_Alpha = "alephalpha",
Ollama = "ollama",
Bedrock = "bedrock",
Together = "together",
Custom = "__custom",
}

Expand All @@ -158,6 +219,7 @@ export function getProvider(llm: LLM): LLMProvider | undefined {
else if (llm_name?.startsWith("Aleph_Alpha")) return LLMProvider.Aleph_Alpha;
else if (llm_name?.startsWith("Ollama")) return LLMProvider.Ollama;
else if (llm_name?.startsWith("Bedrock")) return LLMProvider.Bedrock;
else if (llm_name?.startsWith("Together")) return LLMProvider.Together;
else if (llm.toString().startsWith("__custom/")) return LLMProvider.Custom;

return undefined;
Expand Down Expand Up @@ -212,6 +274,7 @@ export const RATE_LIMIT_BY_MODEL: { [key in LLM]?: number } = {

export const RATE_LIMIT_BY_PROVIDER: { [key in LLMProvider]?: number } = {
[LLMProvider.Anthropic]: 25, // Tier 1 pricing limit is 50 per minute, across all models; we halve this, to be safe.
[LLMProvider.Together]: 30, // Paid tier limit is 60 per minute, across all models; we halve this, to be safe.
};

// Max concurrent requests. Add to this to further constrain the rate limiter.
Expand Down
6 changes: 6 additions & 0 deletions chainforge/react-server/src/backend/typing.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,12 @@ export type LLMSpec = {
progress?: QueryProgress; // only used for front-end to display progress collecting responses for this LLM
};

export type LLMGroup = {
group: string;
emoji: string;
items: LLMSpec[] | LLMGroup[];
};

/** A spec for a user-defined custom LLM provider */
export type CustomLLMProviderSpec = {
name: string;
Expand Down