From a84eb4de7989723dd7ef7ecb955616827e31ff63 Mon Sep 17 00:00:00 2001 From: Can Bal Date: Tue, 14 May 2024 22:26:43 -0700 Subject: [PATCH 1/3] first commit --- .../react-server/src/GlobalSettingsModal.tsx | 6 ++ .../react-server/src/LLMListComponent.tsx | 35 ++++-- .../react-server/src/ModelSettingSchemas.tsx | 93 ++++++++++++++++ chainforge/react-server/src/backend/models.ts | 63 +++++++++++ chainforge/react-server/src/backend/typing.ts | 6 ++ chainforge/react-server/src/backend/utils.ts | 100 ++++++++++++++++++ chainforge/react-server/src/store.tsx | 57 +++++++++- 7 files changed, 346 insertions(+), 14 deletions(-) diff --git a/chainforge/react-server/src/GlobalSettingsModal.tsx b/chainforge/react-server/src/GlobalSettingsModal.tsx index 3129bf25..5c5275bd 100644 --- a/chainforge/react-server/src/GlobalSettingsModal.tsx +++ b/chainforge/react-server/src/GlobalSettingsModal.tsx @@ -406,6 +406,12 @@ const GlobalSettingsModal = forwardRef( {...form.getInputProps("AlephAlpha")} />
+ +
{ const res: ContextMenuItemOptions[] = []; const initModels: Set = new Set(); + const getMenuItems = (group: LLMGroup) => { + const res: ContextMenuItemOptions[] = []; + for (const item of group.items) { + if (!("group" in item)) { + initModels.add(item.base_model); + res.push({ + key: item.model, + title: `${item.emoji} ${item.name}`, + onClick: () => handleSelectModel(item.base_model), + }); + } else { + const a = getMenuItems(item); + res.push({ + key: item.group, + title: `${item.emoji} ${item.group}`, + items: getMenuItems(item), + }); + } + }; + return res; + } + for (const item of initLLMProviderMenu) { if (!("group" in item)) { initModels.add(item.base_model); @@ -444,14 +466,7 @@ export const LLMListContainer = forwardRef< res.push({ key: item.group, title: `${item.emoji} ${item.group}`, - items: item.items.map((k) => { - initModels.add(k.base_model); - return { - key: k.model, - title: `${k.emoji} ${k.name}`, - onClick: () => handleSelectModel(k.base_model), - }; - }), + items: getMenuItems(item), }); } } diff --git a/chainforge/react-server/src/ModelSettingSchemas.tsx b/chainforge/react-server/src/ModelSettingSchemas.tsx index d08b2a6c..7b51744d 100644 --- a/chainforge/react-server/src/ModelSettingSchemas.tsx +++ b/chainforge/react-server/src/ModelSettingSchemas.tsx @@ -2022,6 +2022,97 @@ const MetaLlama2ChatSettings: ModelSettingsDict = { }, }; +const TogetherChatSettings: ModelSettingsDict = { + fullName: "Together Chat", + schema: { + type: "object", + required: ["shortname"], + properties: { + shortname: { + type: "string", + title: "Nickname", + description: + "Unique identifier to appear in ChainForge. Keep it short.", + default: "TogetherChat", + }, + model: { + type: "string", + title: "model", + description: + "Select a version of Together model to query. For more details on the differences, see the Together API documentation.", + enum: ["zero-one-ai/Yi-34B-Chat", "allenai/OLMo-7B-Instruct", "allenai/OLMo-7B-Twin-2T", "allenai/OLMo-7B", "Austism/chronos-hermes-13b", "cognitivecomputations/dolphin-2.5-mixtral-8x7b", "databricks/dbrx-instruct", "deepseek-ai/deepseek-coder-33b-instruct", "deepseek-ai/deepseek-llm-67b-chat", "garage-bAInd/Platypus2-70B-instruct", "google/gemma-2b-it", "google/gemma-7b-it", "Gryphe/MythoMax-L2-13b", "lmsys/vicuna-13b-v1.5", "lmsys/vicuna-7b-v1.5", "codellama/CodeLlama-13b-Instruct-hf", "codellama/CodeLlama-34b-Instruct-hf", "codellama/CodeLlama-70b-Instruct-hf", "codellama/CodeLlama-7b-Instruct-hf", "meta-llama/Llama-2-70b-chat-hf", "meta-llama/Llama-2-13b-chat-hf", "meta-llama/Llama-2-7b-chat-hf", "meta-llama/Llama-3-8b-chat-hf", "meta-llama/Llama-3-70b-chat-hf", "mistralai/Mistral-7B-Instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.2", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x22B-Instruct-v0.1", "NousResearch/Nous-Capybara-7B-V1p9", "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", "NousResearch/Nous-Hermes-llama-2-7b", "NousResearch/Nous-Hermes-Llama2-13b", "NousResearch/Nous-Hermes-2-Yi-34B", "openchat/openchat-3.5-1210", "Open-Orca/Mistral-7B-OpenOrca", "Qwen/Qwen1.5-0.5B-Chat", "Qwen/Qwen1.5-1.8B-Chat", "Qwen/Qwen1.5-4B-Chat", "Qwen/Qwen1.5-7B-Chat", "Qwen/Qwen1.5-14B-Chat", "Qwen/Qwen1.5-32B-Chat", "Qwen/Qwen1.5-72B-Chat", "Qwen/Qwen1.5-110B-Chat", "snorkelai/Snorkel-Mistral-PairRM-DPO", "Snowflake/snowflake-arctic-instruct", "togethercomputer/alpaca-7b", "teknium/OpenHermes-2-Mistral-7B", "teknium/OpenHermes-2p5-Mistral-7B", "togethercomputer/Llama-2-7B-32K-Instruct", "togethercomputer/RedPajama-INCITE-Chat-3B-v1", "togethercomputer/RedPajama-INCITE-7B-Chat", "togethercomputer/StripedHyena-Nous-7B", "Undi95/ReMM-SLERP-L2-13B", "Undi95/Toppy-M-7B", "WizardLM/WizardLM-13B-V1.2", "upstage/SOLAR-10.7B-Instruct-v1.0"], + default: "meta-llama/Llama-2-7b-chat-hf", + }, + temperature: { + type: "number", + title: "temperature", + description: + "Amount of randomness injected into the response. Ranges from 0 to 1. Use temp closer to 0 for analytical / multiple choice, and temp closer to 1 for creative and generative tasks.", + default: 1, + minimum: 0, + maximum: 1, + multipleOf: 0.01, + }, + max_gen_len: { + type: "integer", + title: "max_tokens", + description: + "The maximum number of tokens to generate for each response.", + default: 1024, + minimum: 1, + }, + top_p: { + type: "number", + title: "top_p", + description: + "Does nucleus sampling, in which we compute the cumulative distribution over all the options for each subsequent token in decreasing probability order and cut it off once it reaches a particular probability specified by top_p. Defaults to -1, which disables it. Note that you should either alter temperature or top_p, but not both.", + default: 1, + minimum: 0.01, + maximum: 1, + multipleOf: 0.001, + }, + }, + }, + postprocessors: { + stop_sequences: (str) => { + if (typeof str !== "string" || str.trim().length === 0) return []; + return str + .match(/"((?:[^"\\]|\\.)*)"/g) + ?.map((s) => s.substring(1, s.length - 1)); // split on double-quotes but exclude escaped double-quotes inside the group + }, + }, + + uiSchema: { + "ui:submitButtonOptions": UI_SUBMIT_BUTTON_SPEC, + shortname: { + "ui:autofocus": true, + }, + model: { + "ui:help": "Defaults to LlamaChat 13B", + }, + temperature: { + "ui:help": "Defaults to 1.0.", + "ui:widget": "range", + }, + max_tokens: { + "ui:help": "Defaults to 1024.", + }, + num_generations: { + "ui:help": "Defaults to 1.", + }, + k: { + "ui:help": "Defaults to 0.", + }, + p: { + "ui:help": "Defaults to 1.", + }, + stop_sequences: { + "ui:widget": "textarea", + "ui:help": "Defaults to no sequence", + }, + }, +}; + // A lookup table indexed by base_model. export const ModelSettings: Dict = { "gpt-3.5-turbo": ChatGPTSettings, @@ -2041,6 +2132,7 @@ export const ModelSettings: Dict = { "br.mistral.mistral": MistralSettings, "br.mistral.mixtral": MixtralSettings, "br.meta.llama2": MetaLlama2ChatSettings, + "together": TogetherChatSettings, }; export function getSettingsSchemaForLLM( @@ -2059,6 +2151,7 @@ export function getSettingsSchemaForLLM( [LLMProvider.HuggingFace]: HuggingFaceTextInferenceSettings, [LLMProvider.Aleph_Alpha]: AlephAlphaLuminousSettings, [LLMProvider.Ollama]: OllamaSettings, + [LLMProvider.Together]: TogetherChatSettings, }; if (llm_provider === LLMProvider.Custom) return ModelSettings[llm_name]; diff --git a/chainforge/react-server/src/backend/models.ts b/chainforge/react-server/src/backend/models.ts index 4bea4ad1..8d244065 100644 --- a/chainforge/react-server/src/backend/models.ts +++ b/chainforge/react-server/src/backend/models.ts @@ -106,6 +106,66 @@ export enum NativeLLM { Bedrock_Mistral_Mistral = "mistral.mistral-7b-instruct-v0:2", Bedrock_Mistral_Mistral_Large = "mistral.mistral-large-2402-v1:0", Bedrock_Mistral_Mixtral = "mistral.mixtral-8x7b-instruct-v0:1", + + // Together.ai + Together_ZeroOneAI_01ai_Yi_Chat_34B = "zero-one-ai/Yi-34B-Chat", + Together_AllenAI_OLMo_Instruct_7B = "allenai/OLMo-7B-Instruct", + Together_AllenAI_OLMo_Twin2T_7B = "allenai/OLMo-7B-Twin-2T", + Together_AllenAI_OLMo_7B = "allenai/OLMo-7B", + Together_Austism_Chronos_Hermes_13B = "Austism/chronos-hermes-13b", + Together_cognitivecomputations_Dolphin_2_5_Mixtral_8x7b = "cognitivecomputations/dolphin-2.5-mixtral-8x7b", + Together_databricks_DBRX_Instruct = "databricks/dbrx-instruct", + Together_DeepSeek_Deepseek_Coder_Instruct_33B = "deepseek-ai/deepseek-coder-33b-instruct", + Together_DeepSeek_DeepSeek_LLM_Chat_67B = "deepseek-ai/deepseek-llm-67b-chat", + Together_garagebAInd_Platypus2_Instruct_70B = "garage-bAInd/Platypus2-70B-instruct", + Together_Google_Gemma_Instruct_2B = "google/gemma-2b-it", + Together_Google_Gemma_Instruct_7B = "google/gemma-7b-it", + Together_Gryphe_MythoMaxL2_13B = "Gryphe/MythoMax-L2-13b", + Together_LMSys_Vicuna_v1_5_13B = "lmsys/vicuna-13b-v1.5", + Together_LMSys_Vicuna_v1_5_7B = "lmsys/vicuna-7b-v1.5", + Together_Meta_Code_Llama_Instruct_13B = "codellama/CodeLlama-13b-Instruct-hf", + Together_Meta_Code_Llama_Instruct_34B = "codellama/CodeLlama-34b-Instruct-hf", + Together_Meta_Code_Llama_Instruct_70B = "codellama/CodeLlama-70b-Instruct-hf", + Together_Meta_Code_Llama_Instruct_7B = "codellama/CodeLlama-7b-Instruct-hf", + Together_Meta_LLaMA2_Chat_70B = "meta-llama/Llama-2-70b-chat-hf", + Together_Meta_LLaMA2_Chat_13B = "meta-llama/Llama-2-13b-chat-hf", + Together_Meta_LLaMA2_Chat_7B = "meta-llama/Llama-2-7b-chat-hf", + Together_Meta_LLaMA3_Chat_8B = "meta-llama/Llama-3-8b-chat-hf", + Together_Meta_LLaMA3_Chat_70B = "meta-llama/Llama-3-70b-chat-hf", + Together_mistralai_Mistral_7B_Instruct = "mistralai/Mistral-7B-Instruct-v0.1", + Together_mistralai_Mistral_7B_Instruct_v0_2 = "mistralai/Mistral-7B-Instruct-v0.2", + Together_mistralai_Mixtral8x7B_Instruct_46_7B = "mistralai/Mixtral-8x7B-Instruct-v0.1", + Together_mistralai_Mixtral8x22B_Instruct_141B = "mistralai/Mixtral-8x22B-Instruct-v0.1", + Together_NousResearch_Nous_Capybara_v1_9_7B = "NousResearch/Nous-Capybara-7B-V1p9", + Together_NousResearch_Nous_Hermes_2__Mistral_DPO_7B = "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", + Together_NousResearch_Nous_Hermes_2__Mixtral_8x7BDPO_46_7B = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", + Together_NousResearch_Nous_Hermes_2__Mixtral_8x7BSFT_46_7B = "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", + Together_NousResearch_Nous_Hermes_LLaMA2_7B = "NousResearch/Nous-Hermes-llama-2-7b", + Together_NousResearch_Nous_Hermes_Llama2_13B = "NousResearch/Nous-Hermes-Llama2-13b", + Together_NousResearch_Nous_Hermes2_Yi_34B = "NousResearch/Nous-Hermes-2-Yi-34B", + Together_OpenChat_OpenChat_3_5_7B = "openchat/openchat-3.5-1210", + Together_OpenOrca_OpenOrca_Mistral_7B_8K = "Open-Orca/Mistral-7B-OpenOrca", + Together_Qwen_Qwen_1_5_Chat_0_5B = "Qwen/Qwen1.5-0.5B-Chat", + Together_Qwen_Qwen_1_5_Chat_1_8B = "Qwen/Qwen1.5-1.8B-Chat", + Together_Qwen_Qwen_1_5_Chat_4B = "Qwen/Qwen1.5-4B-Chat", + Together_Qwen_Qwen_1_5_Chat_7B = "Qwen/Qwen1.5-7B-Chat", + Together_Qwen_Qwen_1_5_Chat_14B = "Qwen/Qwen1.5-14B-Chat", + Together_Qwen_Qwen_1_5_Chat_32B = "Qwen/Qwen1.5-32B-Chat", + Together_Qwen_Qwen_1_5_Chat_72B = "Qwen/Qwen1.5-72B-Chat", + Together_Qwen_Qwen_1_5_Chat_110B = "Qwen/Qwen1.5-110B-Chat", + Together_SnorkelAI_Snorkel_Mistral_PairRM_DPO_7B = "snorkelai/Snorkel-Mistral-PairRM-DPO", + Together_Snowflake_Snowflake_Arctic_Instruct = "Snowflake/snowflake-arctic-instruct", + Together_Stanford_Alpaca_7B = "togethercomputer/alpaca-7b", + Together_Teknium_OpenHermes2Mistral_7B = "teknium/OpenHermes-2-Mistral-7B", + Together_Teknium_OpenHermes2_5Mistral_7B = "teknium/OpenHermes-2p5-Mistral-7B", + Together_LLaMA27B32KInstruct_7B = "togethercomputer/Llama-2-7B-32K-Instruct", + Together_RedPajamaINCITE_Chat_3B = "togethercomputer/RedPajama-INCITE-Chat-3B-v1", + Together_RedPajamaINCITE_Chat_7B = "togethercomputer/RedPajama-INCITE-7B-Chat", + Together_StripedHyena_Nous_7B = "togethercomputer/StripedHyena-Nous-7B", + Together_Undi95_ReMM_SLERP_L2_13B = "Undi95/ReMM-SLERP-L2-13B", + Together_Undi95_Toppy_M_7B = "Undi95/Toppy-M-7B", + Together_WizardLM_WizardLM_v1_2_13B = "WizardLM/WizardLM-13B-V1.2", + Together_upstage_Upstage_SOLAR_Instruct_v1_11B = "upstage/SOLAR-10.7B-Instruct-v1.0", } export type LLM = string | NativeLLM; @@ -132,6 +192,7 @@ export enum LLMProvider { Aleph_Alpha = "alephalpha", Ollama = "ollama", Bedrock = "bedrock", + Together = "together", Custom = "__custom", } @@ -152,6 +213,7 @@ export function getProvider(llm: LLM): LLMProvider | undefined { else if (llm_name?.startsWith("Aleph_Alpha")) return LLMProvider.Aleph_Alpha; else if (llm_name?.startsWith("Ollama")) return LLMProvider.Ollama; else if (llm_name?.startsWith("Bedrock")) return LLMProvider.Bedrock; + else if (llm_name?.startsWith("Together")) return LLMProvider.Together; else if (llm.toString().startsWith("__custom/")) return LLMProvider.Custom; return undefined; @@ -202,6 +264,7 @@ export const RATE_LIMIT_BY_MODEL: { [key in LLM]?: number } = { export const RATE_LIMIT_BY_PROVIDER: { [key in LLMProvider]?: number } = { [LLMProvider.Anthropic]: 25, // Tier 1 pricing limit is 50 per minute, across all models; we halve this, to be safe. + [LLMProvider.Together]: 30, // Paid tier limit is 60 per minute, across all models; we halve this, to be safe. }; // Max concurrent requests. Add to this to further constrain the rate limiter. diff --git a/chainforge/react-server/src/backend/typing.ts b/chainforge/react-server/src/backend/typing.ts index 3bbeb14b..d23076c3 100644 --- a/chainforge/react-server/src/backend/typing.ts +++ b/chainforge/react-server/src/backend/typing.ts @@ -122,6 +122,12 @@ export type LLMSpec = { progress?: QueryProgress; // only used for front-end to display progress collecting responses for this LLM }; +export type LLMGroup = { + group: string; + emoji: string; + items: LLMSpec[] | LLMGroup[]; +}; + /** A spec for a user-defined custom LLM provider */ export type CustomLLMProviderSpec = { name: string; diff --git a/chainforge/react-server/src/backend/utils.ts b/chainforge/react-server/src/backend/utils.ts index c2bf99a5..48366faf 100644 --- a/chainforge/react-server/src/backend/utils.ts +++ b/chainforge/react-server/src/backend/utils.ts @@ -155,6 +155,7 @@ let AWS_ACCESS_KEY_ID = get_environ("AWS_ACCESS_KEY_ID"); let AWS_SECRET_ACCESS_KEY = get_environ("AWS_SECRET_ACCESS_KEY"); let AWS_SESSION_TOKEN = get_environ("AWS_SESSION_TOKEN"); let AWS_REGION = get_environ("AWS_REGION"); +let TOGETHER_API_KEY = get_environ("TOGETHER_API_KEY"); /** * Sets the local API keys for the revelant LLM API(s). @@ -186,6 +187,7 @@ export function set_api_keys(api_keys: Dict): void { if (key_is_present("AWS_Session_Token")) AWS_SESSION_TOKEN = api_keys.AWS_Session_Token; if (key_is_present("AWS_Region")) AWS_REGION = api_keys.AWS_Region; + if (key_is_present("Together")) TOGETHER_API_KEY = api_keys.Together; } export function get_azure_openai_api_keys(): [ @@ -1358,6 +1360,101 @@ export async function call_bedrock( return [query, responses]; } +/** + * Calls Together.ai text + chat models via Together's API. + @returns raw query and response JSON dicts. + */ +export async function call_together( + prompt: string, + model: LLM, + n = 1, + temperature = 1.0, + params?: Dict, + should_cancel?: () => boolean, +): Promise<[Dict, Dict]> { + if (!TOGETHER_API_KEY) + throw new Error( + "Could not find an Together API key. Double-check that your API key is set in Settings or in your local environment.", + ); + + const toegetherBaseUrl = "https://api.together.xyz/v1"; + + // Together.ai uses OpenAI's API, so we can use the OpenAI API client to make the call: + const configuration = new OpenAIConfig({ + apiKey: TOGETHER_API_KEY, + basePath: toegetherBaseUrl, + }); + + // Since we are running client-side, we need to remove the user-agent header: + delete configuration.baseOptions.headers["User-Agent"]; + + const together = new OpenAIApi(configuration); + + const modelname: string = model.toString(); + if ( + params?.stop !== undefined && + (!Array.isArray(params.stop) || params.stop.length === 0) + ) + delete params.stop; + if (params?.seed && params.seed.toString().length === 0) delete params?.seed; + if ( + params?.functions !== undefined && + (!Array.isArray(params.functions) || params.functions.length === 0) + ) + delete params?.functions; + if ( + params?.function_call !== undefined && + (!(typeof params.function_call === "string") || + params.function_call.trim().length === 0) + ) + delete params.function_call; + + console.log(`Querying Together model '${model}' with prompt '${prompt}'...`); + + // Determine the system message and whether there's chat history to continue: + const chat_history: ChatHistory | undefined = params?.chat_history; + const system_msg: string = + params?.system_msg !== undefined + ? params.system_msg + : "You are a helpful assistant."; + delete params?.system_msg; + delete params?.chat_history; + + const query: Dict = { + model: modelname, + n, + temperature, + ...params, // 'the rest' of the settings, passed from the front-end settings + }; + + // Create call to chat model + const together_call: any = together.createChatCompletion.bind(together); + + // Carry over chat history, if present: + query.messages = construct_openai_chat_history( + prompt, + chat_history, + system_msg, + ); + + // Try to call OpenAI + let response: Dict = {}; + try { + const completion = await together_call(query); + response = completion.data; + } catch (error: any) { + if (error?.response) { + throw new Error(error.response.data?.error?.message); + // throw new Error(error.response.status); + } else { + console.log(error?.message || error); + throw new Error(error?.message || error); + } + } + + return [query, response]; +} + async function call_custom_provider( prompt: string, model: LLM, @@ -1442,6 +1539,7 @@ export async function call_llm( else if (llm_provider === LLMProvider.Ollama) call_api = call_ollama_provider; else if (llm_provider === LLMProvider.Custom) call_api = call_custom_provider; else if (llm_provider === LLMProvider.Bedrock) call_api = call_bedrock; + else if (llm_provider === LLMProvider.Together) call_api = call_together; if (call_api === undefined) throw new Error( `Adapter for Language model ${llm} and ${llm_provider} not found`, @@ -1617,6 +1715,8 @@ export function extract_responses( return _extract_ollama_responses(response as Dict[]); case LLMProvider.Bedrock: return response as Array; + case LLMProvider.Together: + return _extract_openai_responses(response as Dict[]); default: if ( Array.isArray(response) && diff --git a/chainforge/react-server/src/store.tsx b/chainforge/react-server/src/store.tsx index 7a6f87e5..d119101a 100644 --- a/chainforge/react-server/src/store.tsx +++ b/chainforge/react-server/src/store.tsx @@ -20,6 +20,7 @@ import { DuplicateVariableNameError } from "./backend/errors"; import { Dict, LLMSpec, + LLMGroup, PromptVarType, PromptVarsDict, TemplateVarInfo, @@ -89,7 +90,7 @@ const refreshableOutputNodeTypes = new Set([ export const initLLMProviderMenu: ( | LLMSpec - | { group: string; emoji: string; items: LLMSpec[] } + | LLMGroup )[] = [ { group: "OpenAI", @@ -223,6 +224,39 @@ export const initLLMProviderMenu: ( }, ]; +const togetherModels = ["zero-one-ai/Yi-34B-Chat", "allenai/OLMo-7B-Instruct", "allenai/OLMo-7B-Twin-2T", "allenai/OLMo-7B", "Austism/chronos-hermes-13b", "cognitivecomputations/dolphin-2.5-mixtral-8x7b", "databricks/dbrx-instruct", "deepseek-ai/deepseek-coder-33b-instruct", "deepseek-ai/deepseek-llm-67b-chat", "garage-bAInd/Platypus2-70B-instruct", "google/gemma-2b-it", "google/gemma-7b-it", "Gryphe/MythoMax-L2-13b", "lmsys/vicuna-13b-v1.5", "lmsys/vicuna-7b-v1.5", "codellama/CodeLlama-13b-Instruct-hf", "codellama/CodeLlama-34b-Instruct-hf", "codellama/CodeLlama-70b-Instruct-hf", "codellama/CodeLlama-7b-Instruct-hf", "meta-llama/Llama-2-70b-chat-hf", "meta-llama/Llama-2-13b-chat-hf", "meta-llama/Llama-2-7b-chat-hf", "meta-llama/Llama-3-8b-chat-hf", "meta-llama/Llama-3-70b-chat-hf", "mistralai/Mistral-7B-Instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.2", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x22B-Instruct-v0.1", "NousResearch/Nous-Capybara-7B-V1p9", "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", "NousResearch/Nous-Hermes-llama-2-7b", "NousResearch/Nous-Hermes-Llama2-13b", "NousResearch/Nous-Hermes-2-Yi-34B", "openchat/openchat-3.5-1210", "Open-Orca/Mistral-7B-OpenOrca", "Qwen/Qwen1.5-0.5B-Chat", "Qwen/Qwen1.5-1.8B-Chat", "Qwen/Qwen1.5-4B-Chat", "Qwen/Qwen1.5-7B-Chat", "Qwen/Qwen1.5-14B-Chat", "Qwen/Qwen1.5-32B-Chat", "Qwen/Qwen1.5-72B-Chat", "Qwen/Qwen1.5-110B-Chat", "snorkelai/Snorkel-Mistral-PairRM-DPO", "Snowflake/snowflake-arctic-instruct", "togethercomputer/alpaca-7b", "teknium/OpenHermes-2-Mistral-7B", "teknium/OpenHermes-2p5-Mistral-7B", "togethercomputer/Llama-2-7B-32K-Instruct", "togethercomputer/RedPajama-INCITE-Chat-3B-v1", "togethercomputer/RedPajama-INCITE-7B-Chat", "togethercomputer/StripedHyena-Nous-7B", "Undi95/ReMM-SLERP-L2-13B", "Undi95/Toppy-M-7B", "WizardLM/WizardLM-13B-V1.2", "upstage/SOLAR-10.7B-Instruct-v1.0"] +const togetherGroups = () => { + const groupNames: string[] = []; + const groups: { [key: string]: LLMGroup } = {}; + togetherModels.forEach((model) => { + const [group, name] = model.split("/"); + const spec: LLMSpec = { + name: name, + emoji: "🤝", + model: model, + base_model: "together/" + model, + temp: 0.9, + } + if (!(group in groups)) { + groups[group] = { + group: group, + emoji: "🤝", + items: [spec], + }; + groupNames.push(group); + } else { + (groups[group].items as LLMSpec[]).push(spec); + } + }); + return groupNames.map((group) => groups[group]); +}; +const togetherLLMProviderMenu: LLMGroup = { + group: "Together", + emoji: "🤝", + items: togetherGroups(), +}; +initLLMProviderMenu.push(togetherLLMProviderMenu); + if (APP_IS_RUNNING_LOCALLY()) { initLLMProviderMenu.push({ name: "Ollama", @@ -235,9 +269,24 @@ if (APP_IS_RUNNING_LOCALLY()) { // initLLMProviders.push({ name: "Dalai (Alpaca.7B)", emoji: "🦙", model: "alpaca.7B", base_model: "dalai", temp: 0.5 }); // ------------------------- } -export const initLLMProviders = initLLMProviderMenu - .map((item) => ("group" in item && "items" in item ? item.items : item)) - .flat(); + +function flattenLLMGroup(group: LLMGroup): LLMSpec[] { + return group.items.flatMap((item) => + "group" in item && "items" in item ? flattenLLMGroup(item) : item, + ); +} + +function flattenLLMGroups(groups: LLMGroup[]): LLMSpec[] { + return groups.flatMap((group) => flattenLLMGroup(group)); +} + +function flattenLLMProviders(providers: (LLMSpec | LLMGroup)[]): LLMSpec[] { + return providers.flatMap((item) => + "group" in item && "items" in item ? flattenLLMGroups([item]) : item, + ); +} + +export const initLLMProviders = flattenLLMProviders(initLLMProviderMenu) export interface StoreHandles { // Nodes and edges From 4456a533a38a716d203ecac61c96a7982de0be08 Mon Sep 17 00:00:00 2001 From: Can Bal Date: Tue, 14 May 2024 23:23:38 -0700 Subject: [PATCH 2/3] some cleanup --- .../react-server/src/LLMListComponent.tsx | 47 +++++-------------- .../react-server/src/ModelSettingSchemas.tsx | 2 +- chainforge/react-server/src/backend/utils.ts | 6 +-- chainforge/react-server/src/store.tsx | 29 +++++------- 4 files changed, 30 insertions(+), 54 deletions(-) diff --git a/chainforge/react-server/src/LLMListComponent.tsx b/chainforge/react-server/src/LLMListComponent.tsx index af66addb..63e59322 100644 --- a/chainforge/react-server/src/LLMListComponent.tsx +++ b/chainforge/react-server/src/LLMListComponent.tsx @@ -27,7 +27,7 @@ import { getDefaultModelSettings } from "./ModelSettingSchemas"; import useStore, { initLLMProviders, initLLMProviderMenu } from "./store"; import { Dict, JSONCompatible, LLMGroup, LLMSpec } from "./backend/typing"; import { useContextMenu } from "mantine-contextmenu"; -import { ContextMenuOptions, ContextMenuItemOptions } from "mantine-contextmenu/dist/types"; +import { ContextMenuItemOptions } from "mantine-contextmenu/dist/types"; // The LLM(s) to include by default on a PromptNode whenever one is created. // Defaults to ChatGPT (GPT3.5) when running locally, and HF-hosted falcon-7b for online version since it's free. @@ -430,46 +430,25 @@ export const LLMListContainer = forwardRef< ); const menuItems = useMemo(() => { - const res: ContextMenuItemOptions[] = []; const initModels: Set = new Set(); - const getMenuItems = (group: LLMGroup) => { - const res: ContextMenuItemOptions[] = []; - for (const item of group.items) { - if (!("group" in item)) { - initModels.add(item.base_model); - res.push({ - key: item.model, - title: `${item.emoji} ${item.name}`, - onClick: () => handleSelectModel(item.base_model), - }); - } else { - const a = getMenuItems(item); - res.push({ - key: item.group, - title: `${item.emoji} ${item.group}`, - items: getMenuItems(item), - }); - } - }; - return res; - } - - for (const item of initLLMProviderMenu) { - if (!("group" in item)) { + const convert = (item: LLMSpec | LLMGroup): ContextMenuItemOptions => { + if ("group" in item) { + return { + key: item.group, + title: `${item.emoji} ${item.group}`, + items: item.items.map(convert), + }; + } else { initModels.add(item.base_model); - res.push({ + return { key: item.model, title: `${item.emoji} ${item.name}`, onClick: () => handleSelectModel(item.base_model), - }); - } else { - res.push({ - key: item.group, - title: `${item.emoji} ${item.group}`, - items: getMenuItems(item), - }); + } } } + const res = initLLMProviderMenu.map(convert); + for (const item of AvailableLLMs) { if (initModels.has(item.base_model)) { continue; diff --git a/chainforge/react-server/src/ModelSettingSchemas.tsx b/chainforge/react-server/src/ModelSettingSchemas.tsx index 7b51744d..8910ef4d 100644 --- a/chainforge/react-server/src/ModelSettingSchemas.tsx +++ b/chainforge/react-server/src/ModelSettingSchemas.tsx @@ -2022,7 +2022,7 @@ const MetaLlama2ChatSettings: ModelSettingsDict = { }, }; -const TogetherChatSettings: ModelSettingsDict = { +export const TogetherChatSettings: ModelSettingsDict = { fullName: "Together Chat", schema: { type: "object", diff --git a/chainforge/react-server/src/backend/utils.ts b/chainforge/react-server/src/backend/utils.ts index 48366faf..7f96c764 100644 --- a/chainforge/react-server/src/backend/utils.ts +++ b/chainforge/react-server/src/backend/utils.ts @@ -1377,12 +1377,12 @@ export async function call_together( "Could not find an Together API key. Double-check that your API key is set in Settings or in your local environment.", ); - const toegetherBaseUrl = "https://api.together.xyz/v1"; + const togetherBaseUrl = "https://api.together.xyz/v1"; // Together.ai uses OpenAI's API, so we can use the OpenAI API client to make the call: const configuration = new OpenAIConfig({ apiKey: TOGETHER_API_KEY, - basePath: toegetherBaseUrl, + basePath: togetherBaseUrl, }); // Since we are running client-side, we need to remove the user-agent header: @@ -1437,7 +1437,7 @@ export async function call_together( system_msg, ); - // Try to call OpenAI + // Try to call Together let response: Dict = {}; try { const completion = await together_call(query); diff --git a/chainforge/react-server/src/store.tsx b/chainforge/react-server/src/store.tsx index d119101a..8e214cde 100644 --- a/chainforge/react-server/src/store.tsx +++ b/chainforge/react-server/src/store.tsx @@ -19,14 +19,15 @@ import { import { DuplicateVariableNameError } from "./backend/errors"; import { Dict, - LLMSpec, LLMGroup, + LLMSpec, PromptVarType, PromptVarsDict, TemplateVarInfo, TabularDataColType, TabularDataRowType, } from "./backend/typing"; +import { TogetherChatSettings } from "./ModelSettingSchemas"; // Initial project settings const initialAPIKeys = {}; @@ -224,31 +225,31 @@ export const initLLMProviderMenu: ( }, ]; -const togetherModels = ["zero-one-ai/Yi-34B-Chat", "allenai/OLMo-7B-Instruct", "allenai/OLMo-7B-Twin-2T", "allenai/OLMo-7B", "Austism/chronos-hermes-13b", "cognitivecomputations/dolphin-2.5-mixtral-8x7b", "databricks/dbrx-instruct", "deepseek-ai/deepseek-coder-33b-instruct", "deepseek-ai/deepseek-llm-67b-chat", "garage-bAInd/Platypus2-70B-instruct", "google/gemma-2b-it", "google/gemma-7b-it", "Gryphe/MythoMax-L2-13b", "lmsys/vicuna-13b-v1.5", "lmsys/vicuna-7b-v1.5", "codellama/CodeLlama-13b-Instruct-hf", "codellama/CodeLlama-34b-Instruct-hf", "codellama/CodeLlama-70b-Instruct-hf", "codellama/CodeLlama-7b-Instruct-hf", "meta-llama/Llama-2-70b-chat-hf", "meta-llama/Llama-2-13b-chat-hf", "meta-llama/Llama-2-7b-chat-hf", "meta-llama/Llama-3-8b-chat-hf", "meta-llama/Llama-3-70b-chat-hf", "mistralai/Mistral-7B-Instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.2", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x22B-Instruct-v0.1", "NousResearch/Nous-Capybara-7B-V1p9", "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", "NousResearch/Nous-Hermes-llama-2-7b", "NousResearch/Nous-Hermes-Llama2-13b", "NousResearch/Nous-Hermes-2-Yi-34B", "openchat/openchat-3.5-1210", "Open-Orca/Mistral-7B-OpenOrca", "Qwen/Qwen1.5-0.5B-Chat", "Qwen/Qwen1.5-1.8B-Chat", "Qwen/Qwen1.5-4B-Chat", "Qwen/Qwen1.5-7B-Chat", "Qwen/Qwen1.5-14B-Chat", "Qwen/Qwen1.5-32B-Chat", "Qwen/Qwen1.5-72B-Chat", "Qwen/Qwen1.5-110B-Chat", "snorkelai/Snorkel-Mistral-PairRM-DPO", "Snowflake/snowflake-arctic-instruct", "togethercomputer/alpaca-7b", "teknium/OpenHermes-2-Mistral-7B", "teknium/OpenHermes-2p5-Mistral-7B", "togethercomputer/Llama-2-7B-32K-Instruct", "togethercomputer/RedPajama-INCITE-Chat-3B-v1", "togethercomputer/RedPajama-INCITE-7B-Chat", "togethercomputer/StripedHyena-Nous-7B", "Undi95/ReMM-SLERP-L2-13B", "Undi95/Toppy-M-7B", "WizardLM/WizardLM-13B-V1.2", "upstage/SOLAR-10.7B-Instruct-v1.0"] +const togetherModels = TogetherChatSettings['schema']['properties']['model']['enum'] as string[]; const togetherGroups = () => { const groupNames: string[] = []; const groups: { [key: string]: LLMGroup } = {}; togetherModels.forEach((model) => { - const [group, name] = model.split("/"); + const [groupName, modelName] = model.split("/"); const spec: LLMSpec = { - name: name, + name: modelName, emoji: "🤝", model: model, base_model: "together/" + model, temp: 0.9, } - if (!(group in groups)) { - groups[group] = { - group: group, + if (groupName in groups) { + (groups[groupName].items as LLMSpec[]).push(spec); + } else { + groups[groupName] = { + group: groupName, emoji: "🤝", items: [spec], }; - groupNames.push(group); - } else { - (groups[group].items as LLMSpec[]).push(spec); + groupNames.push(groupName); } }); - return groupNames.map((group) => groups[group]); + return groupNames.map((name) => groups[name]); }; const togetherLLMProviderMenu: LLMGroup = { group: "Together", @@ -276,13 +277,9 @@ function flattenLLMGroup(group: LLMGroup): LLMSpec[] { ); } -function flattenLLMGroups(groups: LLMGroup[]): LLMSpec[] { - return groups.flatMap((group) => flattenLLMGroup(group)); -} - function flattenLLMProviders(providers: (LLMSpec | LLMGroup)[]): LLMSpec[] { return providers.flatMap((item) => - "group" in item && "items" in item ? flattenLLMGroups([item]) : item, + "group" in item && "items" in item ? flattenLLMGroup(item) : item, ); } From ccf2b6110898b40781974634560cd7b3f6c55c07 Mon Sep 17 00:00:00 2001 From: Can Bal Date: Wed, 15 May 2024 11:22:21 -0700 Subject: [PATCH 3/3] linter changes --- .../react-server/src/LLMListComponent.tsx | 4 +- .../react-server/src/ModelSettingSchemas.tsx | 63 ++++++++++++++++++- chainforge/react-server/src/store.tsx | 14 ++--- 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/chainforge/react-server/src/LLMListComponent.tsx b/chainforge/react-server/src/LLMListComponent.tsx index 63e59322..a550e089 100644 --- a/chainforge/react-server/src/LLMListComponent.tsx +++ b/chainforge/react-server/src/LLMListComponent.tsx @@ -444,9 +444,9 @@ export const LLMListContainer = forwardRef< key: item.model, title: `${item.emoji} ${item.name}`, onClick: () => handleSelectModel(item.base_model), - } + }; } - } + }; const res = initLLMProviderMenu.map(convert); for (const item of AvailableLLMs) { diff --git a/chainforge/react-server/src/ModelSettingSchemas.tsx b/chainforge/react-server/src/ModelSettingSchemas.tsx index 8910ef4d..cd77bcb1 100644 --- a/chainforge/react-server/src/ModelSettingSchemas.tsx +++ b/chainforge/react-server/src/ModelSettingSchemas.tsx @@ -2040,7 +2040,66 @@ export const TogetherChatSettings: ModelSettingsDict = { title: "model", description: "Select a version of Together model to query. For more details on the differences, see the Together API documentation.", - enum: ["zero-one-ai/Yi-34B-Chat", "allenai/OLMo-7B-Instruct", "allenai/OLMo-7B-Twin-2T", "allenai/OLMo-7B", "Austism/chronos-hermes-13b", "cognitivecomputations/dolphin-2.5-mixtral-8x7b", "databricks/dbrx-instruct", "deepseek-ai/deepseek-coder-33b-instruct", "deepseek-ai/deepseek-llm-67b-chat", "garage-bAInd/Platypus2-70B-instruct", "google/gemma-2b-it", "google/gemma-7b-it", "Gryphe/MythoMax-L2-13b", "lmsys/vicuna-13b-v1.5", "lmsys/vicuna-7b-v1.5", "codellama/CodeLlama-13b-Instruct-hf", "codellama/CodeLlama-34b-Instruct-hf", "codellama/CodeLlama-70b-Instruct-hf", "codellama/CodeLlama-7b-Instruct-hf", "meta-llama/Llama-2-70b-chat-hf", "meta-llama/Llama-2-13b-chat-hf", "meta-llama/Llama-2-7b-chat-hf", "meta-llama/Llama-3-8b-chat-hf", "meta-llama/Llama-3-70b-chat-hf", "mistralai/Mistral-7B-Instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.2", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mixtral-8x22B-Instruct-v0.1", "NousResearch/Nous-Capybara-7B-V1p9", "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", "NousResearch/Nous-Hermes-llama-2-7b", "NousResearch/Nous-Hermes-Llama2-13b", "NousResearch/Nous-Hermes-2-Yi-34B", "openchat/openchat-3.5-1210", "Open-Orca/Mistral-7B-OpenOrca", "Qwen/Qwen1.5-0.5B-Chat", "Qwen/Qwen1.5-1.8B-Chat", "Qwen/Qwen1.5-4B-Chat", "Qwen/Qwen1.5-7B-Chat", "Qwen/Qwen1.5-14B-Chat", "Qwen/Qwen1.5-32B-Chat", "Qwen/Qwen1.5-72B-Chat", "Qwen/Qwen1.5-110B-Chat", "snorkelai/Snorkel-Mistral-PairRM-DPO", "Snowflake/snowflake-arctic-instruct", "togethercomputer/alpaca-7b", "teknium/OpenHermes-2-Mistral-7B", "teknium/OpenHermes-2p5-Mistral-7B", "togethercomputer/Llama-2-7B-32K-Instruct", "togethercomputer/RedPajama-INCITE-Chat-3B-v1", "togethercomputer/RedPajama-INCITE-7B-Chat", "togethercomputer/StripedHyena-Nous-7B", "Undi95/ReMM-SLERP-L2-13B", "Undi95/Toppy-M-7B", "WizardLM/WizardLM-13B-V1.2", "upstage/SOLAR-10.7B-Instruct-v1.0"], + enum: [ + "zero-one-ai/Yi-34B-Chat", + "allenai/OLMo-7B-Instruct", + "allenai/OLMo-7B-Twin-2T", + "allenai/OLMo-7B", + "Austism/chronos-hermes-13b", + "cognitivecomputations/dolphin-2.5-mixtral-8x7b", + "databricks/dbrx-instruct", + "deepseek-ai/deepseek-coder-33b-instruct", + "deepseek-ai/deepseek-llm-67b-chat", + "garage-bAInd/Platypus2-70B-instruct", + "google/gemma-2b-it", + "google/gemma-7b-it", + "Gryphe/MythoMax-L2-13b", + "lmsys/vicuna-13b-v1.5", + "lmsys/vicuna-7b-v1.5", + "codellama/CodeLlama-13b-Instruct-hf", + "codellama/CodeLlama-34b-Instruct-hf", + "codellama/CodeLlama-70b-Instruct-hf", + "codellama/CodeLlama-7b-Instruct-hf", + "meta-llama/Llama-2-70b-chat-hf", + "meta-llama/Llama-2-13b-chat-hf", + "meta-llama/Llama-2-7b-chat-hf", + "meta-llama/Llama-3-8b-chat-hf", + "meta-llama/Llama-3-70b-chat-hf", + "mistralai/Mistral-7B-Instruct-v0.1", + "mistralai/Mistral-7B-Instruct-v0.2", + "mistralai/Mixtral-8x7B-Instruct-v0.1", + "mistralai/Mixtral-8x22B-Instruct-v0.1", + "NousResearch/Nous-Capybara-7B-V1p9", + "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", + "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", + "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", + "NousResearch/Nous-Hermes-llama-2-7b", + "NousResearch/Nous-Hermes-Llama2-13b", + "NousResearch/Nous-Hermes-2-Yi-34B", + "openchat/openchat-3.5-1210", + "Open-Orca/Mistral-7B-OpenOrca", + "Qwen/Qwen1.5-0.5B-Chat", + "Qwen/Qwen1.5-1.8B-Chat", + "Qwen/Qwen1.5-4B-Chat", + "Qwen/Qwen1.5-7B-Chat", + "Qwen/Qwen1.5-14B-Chat", + "Qwen/Qwen1.5-32B-Chat", + "Qwen/Qwen1.5-72B-Chat", + "Qwen/Qwen1.5-110B-Chat", + "snorkelai/Snorkel-Mistral-PairRM-DPO", + "Snowflake/snowflake-arctic-instruct", + "togethercomputer/alpaca-7b", + "teknium/OpenHermes-2-Mistral-7B", + "teknium/OpenHermes-2p5-Mistral-7B", + "togethercomputer/Llama-2-7B-32K-Instruct", + "togethercomputer/RedPajama-INCITE-Chat-3B-v1", + "togethercomputer/RedPajama-INCITE-7B-Chat", + "togethercomputer/StripedHyena-Nous-7B", + "Undi95/ReMM-SLERP-L2-13B", + "Undi95/Toppy-M-7B", + "WizardLM/WizardLM-13B-V1.2", + "upstage/SOLAR-10.7B-Instruct-v1.0", + ], default: "meta-llama/Llama-2-7b-chat-hf", }, temperature: { @@ -2132,7 +2191,7 @@ export const ModelSettings: Dict = { "br.mistral.mistral": MistralSettings, "br.mistral.mixtral": MixtralSettings, "br.meta.llama2": MetaLlama2ChatSettings, - "together": TogetherChatSettings, + together: TogetherChatSettings, }; export function getSettingsSchemaForLLM( diff --git a/chainforge/react-server/src/store.tsx b/chainforge/react-server/src/store.tsx index 8e214cde..0ae39afa 100644 --- a/chainforge/react-server/src/store.tsx +++ b/chainforge/react-server/src/store.tsx @@ -89,10 +89,7 @@ const refreshableOutputNodeTypes = new Set([ "split", ]); -export const initLLMProviderMenu: ( - | LLMSpec - | LLMGroup -)[] = [ +export const initLLMProviderMenu: (LLMSpec | LLMGroup)[] = [ { group: "OpenAI", emoji: "🤖", @@ -225,7 +222,8 @@ export const initLLMProviderMenu: ( }, ]; -const togetherModels = TogetherChatSettings['schema']['properties']['model']['enum'] as string[]; +const togetherModels = TogetherChatSettings.schema.properties.model + .enum as string[]; const togetherGroups = () => { const groupNames: string[] = []; const groups: { [key: string]: LLMGroup } = {}; @@ -234,10 +232,10 @@ const togetherGroups = () => { const spec: LLMSpec = { name: modelName, emoji: "🤝", - model: model, + model, base_model: "together/" + model, temp: 0.9, - } + }; if (groupName in groups) { (groups[groupName].items as LLMSpec[]).push(spec); } else { @@ -283,7 +281,7 @@ function flattenLLMProviders(providers: (LLMSpec | LLMGroup)[]): LLMSpec[] { ); } -export const initLLMProviders = flattenLLMProviders(initLLMProviderMenu) +export const initLLMProviders = flattenLLMProviders(initLLMProviderMenu); export interface StoreHandles { // Nodes and edges