Merge pull request #278 from helixml/bigger-70b

switch to 4bit quant of llama3-70b
helixml · May 2, 2024 · d72b619 · d72b619
2 parents ac49ea2 + 726594e
commit d72b619
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 5 deletions.
diff --git a/api/pkg/model/models.go b/api/pkg/model/models.go
@@ -34,7 +34,7 @@ func GetModels() (map[types.ModelName]Model, error) {
 
 	// Llama3
 	models[types.Model_Ollama_Llama3_8b] = NewOllamaGenericText(types.Model_Ollama_Llama3_8b.String(), MB*5349)
-	models[types.Model_Ollama_Llama3_70b] = NewOllamaGenericText(types.Model_Ollama_Llama3_70b.String(), GB*40)
+	models[types.Model_Ollama_Llama3_70b] = NewOllamaGenericText(types.Model_Ollama_Llama3_70b.String(), GB*80)
 
 	return models, nil
 }

diff --git a/api/pkg/types/models.go b/api/pkg/types/models.go
@@ -21,7 +21,7 @@ const (
 	Model_Ollama_NousHermes2Pro ModelName = "adrienbrault/nous-hermes2pro:Q5_K_S"
 
 	Model_Ollama_Llama3_8b  ModelName = "llama3:instruct"
-	Model_Ollama_Llama3_70b ModelName = "llama3:70b-instruct-q2_K"
+	Model_Ollama_Llama3_70b ModelName = "llama3:70b"
 )
 
 func NewModel(name string) ModelName {

diff --git a/charts/helix-runner/values.yaml b/charts/helix-runner/values.yaml
@@ -19,7 +19,7 @@ runner:
   host: https://app.tryhelix.ai
   token: oh-hallo
   # Models to pull on start
-  models: mistral:7b-instruct,llama3:instruct,llama3:70b-instruct-q2_K,mixtral:instruct
+  models: mistral:7b-instruct,llama3:instruct,llama3:70b,mixtral:instruct
   # Available memory for the runner
   memory: 24GB
   # IPC mode

diff --git a/frontend/src/types.ts b/frontend/src/types.ts
@@ -35,12 +35,12 @@ export const CLONE_INTERACTION_MODE_JUST_DATA: ICloneInteractionMode = 'just_dat
 export const CLONE_INTERACTION_MODE_WITH_QUESTIONS: ICloneInteractionMode = 'with_questions'
 export const CLONE_INTERACTION_MODE_ALL: ICloneInteractionMode = 'all'
 
-export type IModelName = 'mistralai/Mistral-7B-Instruct-v0.1' | 'stabilityai/stable-diffusion-xl-base-1.0' | 'mistral:7b-instruct' | 'mixtral:instruct' | 'llama3:instruct' | 'llama3:70b-instruct-q2_K'
+export type IModelName = 'mistralai/Mistral-7B-Instruct-v0.1' | 'stabilityai/stable-diffusion-xl-base-1.0' | 'mistral:7b-instruct' | 'mixtral:instruct' | 'llama3:instruct' | 'llama3:70b'
 export const MODEL_NAME_MISTRAL: IModelName = 'mistralai/Mistral-7B-Instruct-v0.1'
 export const MODEL_NAME_SDXL: IModelName = 'stabilityai/stable-diffusion-xl-base-1.0'
 export const MODEL_NAME_OLLAMA_MISTRAL: IModelName = 'mistral:7b-instruct'
 export const MODEL_NAME_OLLAMA_LLAMA3_8B: IModelName = 'llama3:instruct'
-export const MODEL_NAME_OLLAMA_LLAMA3_70B: IModelName = 'llama3:70b-instruct-q2_K'
+export const MODEL_NAME_OLLAMA_LLAMA3_70B: IModelName = 'llama3:70b'
 export const MODEL_NAME_OLLAMA_MIXTRAL: IModelName = 'mixtral:instruct'
 
 export type ITextDataPrepStage = '' | 'edit_files' | 'extract_text' | 'index_rag' | 'generate_questions' | 'edit_questions' | 'finetune' | 'complete'