diff --git a/config/model/task.json b/config/model/task.json new file mode 100644 index 00000000..9385dffe --- /dev/null +++ b/config/model/task.json @@ -0,0 +1,493 @@ +{ + "$defs": { + "chat_message": { + "properties": { + "content": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json#/$defs/instill_types/multi_modal_content", + "description": "The message content", + "instillUIOrder": 1, + "title": "Content" + }, + "role": { + "description": "The message role, i.e. 'system', 'user' or 'assistant'", + "instillFormat": "string", + "instillUIOrder": 0, + "title": "Role", + "type": "string" + } + }, + "required": ["role", "content"], + "title": "Chat Message", + "type": "object" + }, + "common": { + "description": "Input", + "instillUIOrder": 0, + "properties": { + "image_base64": { + "description": "Image base64", + "instillUIOrder": 2, + "title": "Image", + "type": "string" + }, + "model_name": { + "description": "The Instill Model model to be used.", + "instillUIOrder": 0, + "title": "Model Name", + "type": "string" + } + }, + "required": ["image_base64", "model_name"], + "title": "Input", + "type": "object" + }, + "extra_params": { + "description": "Extra Parameters", + "instillUIOrder": 3, + "required": [], + "title": "Extra Parameters", + "type": "object" + } + }, + "TASK_CLASSIFICATION": { + "instillShortDescription": "Classify images into predefined categories.", + "input": { + "$ref": "#/$defs/common", + "type": "object" + }, + "output": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json##/$defs/instill_types/classification", + "description": "Output", + "instillUIOrder": 0, + "title": "Output", + "type": "object" + } + }, + "TASK_DETECTION": { + "instillShortDescription": "Detect and localize multiple objects in images.", + "input": { + "$ref": "#/$defs/common", + "type": "object" + }, + "output": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json##/$defs/instill_types/detection", + "description": "Output", + "instillUIOrder": 0, + "title": "Output", + "type": "object" + } + }, + "TASK_IMAGE_TO_IMAGE": { + "instillShortDescription": "Generate image from input text prompt and image.", + "input": { + "description": "Input", + "instillUIOrder": 0, + "properties": { + "cfg_scale": { + "description": "The guidance scale, default is 7.5", + "instillUIOrder": 4, + "title": "CFG Scale", + "type": "number" + }, + "extra_params": { + "$ref": "#/$defs/extra_params" + }, + "image_base64": { + "description": "The prompt image", + "instillUIOrder": 3, + "title": "Prompt Image", + "type": "string" + }, + "model_name": { + "description": "The Instill Model model to be used.", + "instillUIOrder": 0, + "title": "Model Name", + "type": "string" + }, + "prompt": { + "description": "The prompt text", + "instillUIMultiline": true, + "instillUIOrder": 2, + "title": "Prompt", + "type": "string" + }, + "samples": { + "description": "The number of generated samples, default is 1", + "instillUIOrder": 5, + "title": "Samples", + "type": "integer" + }, + "seed": { + "description": "The seed", + "instillUIOrder": 4, + "title": "Seed", + "type": "integer" + }, + "top_k": { + "default": 10, + "description": "Top k for sampling", + "instillUIOrder": 5, + "title": "Top K", + "type": "integer" + } + }, + "required": ["prompt", "image_base64", "model_name"], + "title": "Input", + "type": "object" + }, + "output": { + "description": "Output", + "instillUIOrder": 0, + "properties": { + "images": { + "description": "Images", + "instillUIOrder": 0, + "instillFormat": "array:image/jpeg", + "items": { + "instillFormat": "image/jpeg", + "title": "Image", + "type": "string" + }, + "title": "Images", + "type": "array" + } + }, + "required": ["images"], + "title": "Output", + "type": "object" + } + }, + "TASK_INSTANCE_SEGMENTATION": { + "instillShortDescription": "Detect, localize and delineate multiple objects in images.", + "input": { + "$ref": "#/$defs/common", + "type": "object" + }, + "output": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json##/$defs/instill_types/instance_segmentation", + "description": "Output", + "instillUIOrder": 0, + "title": "Output", + "type": "object" + } + }, + "TASK_KEYPOINT": { + "instillShortDescription": "Detect and localize multiple keypoints of objects in images.", + "input": { + "$ref": "#/$defs/common", + "type": "object" + }, + "output": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json##/$defs/instill_types/keypoint", + "description": "Output", + "instillUIOrder": 0, + "title": "Output", + "type": "object" + } + }, + "TASK_OCR": { + "instillShortDescription": "Detect and recognize text in images.", + "input": { + "$ref": "#/$defs/common", + "type": "object" + }, + "output": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json##/$defs/instill_types/ocr", + "description": "Output", + "instillUIOrder": 0, + "title": "Output", + "type": "object" + } + }, + "TASK_SEMANTIC_SEGMENTATION": { + "instillShortDescription": "Classify image pixels into predefined categories.", + "input": { + "$ref": "#/$defs/common", + "type": "object" + }, + "output": { + "$ref": "https://raw.githubusercontent.com/instill-ai/model-backend/5f3aa3e/config/model/schema.json##/$defs/instill_types/semantic_segmentation", + "description": "Output", + "instillUIOrder": 0, + "title": "Output", + "type": "object" + } + }, + "TASK_TEXT_GENERATION": { + "instillShortDescription": "Generate texts from input text prompts.", + "input": { + "description": "Input", + "instillUIOrder": 0, + "properties": { + "chat_history": { + "description": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillShortDescription": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillUIOrder": 4, + "items": { + "$ref": "#/$defs/chat_message" + }, + "title": "Chat history", + "type": "array" + }, + "extra_params": { + "$ref": "#/$defs/extra_params" + }, + "max_new_tokens": { + "default": 50, + "description": "The maximum number of tokens for model to generate", + "instillUIOrder": 6, + "title": "Max new tokens", + "type": "integer" + }, + "model_name": { + "description": "The Instill Model model to be used.", + "instillUIOrder": 0, + "title": "Model Name", + "type": "string" + }, + "prompt": { + "description": "The prompt text", + "instillUIMultiline": true, + "instillUIOrder": 2, + "title": "Prompt", + "type": "string" + }, + "prompt_images": { + "description": "The prompt images", + "instillUIOrder": 3, + "items": { + "type": "string" + }, + "title": "Prompt Images", + "type": "array" + }, + "seed": { + "description": "The seed", + "instillUIOrder": 4, + "title": "Seed", + "type": "integer" + }, + "system_message": { + "default": "You are a helpful assistant.", + "description": "The system message helps set the behavior of the assistant. For example, you can modify the personality of the assistant or provide specific instructions about how it should behave throughout the conversation. By default, the model’s behavior is using a generic message as \"You are a helpful assistant.\"", + "instillShortDescription": "The system message helps set the behavior of the assistant", + "instillUIMultiline": true, + "instillUIOrder": 2, + "title": "System message", + "type": "string" + }, + "temperature": { + "default": 0.7, + "description": "The temperature for sampling", + "instillUIOrder": 5, + "title": "Temperature", + "type": "number" + }, + "top_k": { + "default": 10, + "description": "Top k for sampling", + "instillUIOrder": 5, + "title": "Top K", + "type": "integer" + } + }, + "required": ["prompt", "model_name"], + "title": "Input", + "type": "object" + }, + "output": { + "description": "Output", + "instillUIOrder": 0, + "properties": { + "text": { + "description": "Text", + "instillFormat": "string", + "instillUIMultiline": true, + "instillUIOrder": 0, + "title": "Text", + "type": "string" + } + }, + "required": ["text"], + "title": "Output", + "type": "object" + } + }, + "TASK_TEXT_GENERATION_CHAT": { + "instillShortDescription": "Generate texts from input text prompts and chat history.", + "$ref": "#/TASK_TEXT_GENERATION" + }, + "TASK_TEXT_TO_IMAGE": { + "instillShortDescription": "Generate images from input text prompts.", + "input": { + "description": "Input", + "instillUIOrder": 0, + "properties": { + "cfg_scale": { + "description": "The guidance scale, default is 7.5", + "instillUIOrder": 4, + "title": "CFG Scale", + "type": "number" + }, + "extra_params": { + "$ref": "#/$defs/extra_params" + }, + "model_name": { + "description": "The Instill Model model to be used.", + "instillUIOrder": 0, + "title": "Model Name", + "type": "string" + }, + "prompt": { + "description": "The prompt text", + "instillUIMultiline": true, + "instillUIOrder": 2, + "title": "Prompt", + "type": "string" + }, + "samples": { + "description": "The number of generated samples, default is 1", + "instillUIOrder": 5, + "title": "Samples", + "type": "integer" + }, + "seed": { + "description": "The seed, default is 0", + "instillUIOrder": 6, + "title": "Seed", + "type": "integer" + }, + "steps": { + "description": "The steps, default is 5", + "instillUIOrder": 7, + "title": "Steps", + "type": "integer" + } + }, + "required": ["prompt", "model_name"], + "title": "Input", + "type": "object" + }, + "output": { + "description": "Output", + "instillUIOrder": 0, + "properties": { + "images": { + "description": "Images", + "instillUIOrder": 0, + "instillFormat": "array:image/jpeg", + "items": { + "instillFormat": "image/jpeg", + "title": "Image", + "type": "string" + }, + "title": "Images", + "type": "array" + } + }, + "required": ["images"], + "title": "Output", + "type": "object" + } + }, + "TASK_VISUAL_QUESTION_ANSWERING": { + "instillShortDescription": "Answer questions based on a prompt and an image.", + "input": { + "description": "Input", + "instillUIOrder": 0, + "properties": { + "chat_history": { + "description": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillShortDescription": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillUIOrder": 4, + "items": { + "$ref": "#/$defs/chat_message" + }, + "title": "Chat history", + "type": "array" + }, + "extra_params": { + "$ref": "#/$defs/extra_params" + }, + "max_new_tokens": { + "default": 50, + "description": "The maximum number of tokens for model to generate", + "instillUIOrder": 6, + "title": "Max new tokens", + "type": "integer" + }, + "model_name": { + "description": "The Instill Model model to be used.", + "instillUIOrder": 0, + "title": "Model Name", + "type": "string" + }, + "prompt": { + "description": "The prompt text", + "instillUIMultiline": true, + "instillUIOrder": 2, + "title": "Prompt", + "type": "string" + }, + "prompt_images": { + "description": "The prompt images", + "instillUIOrder": 3, + "items": { + "type": "string" + }, + "title": "Prompt Images", + "type": "array" + }, + "seed": { + "description": "The seed", + "instillUIOrder": 4, + "title": "Seed", + "type": "integer" + }, + "system_message": { + "default": "You are a helpful assistant.", + "description": "The system message helps set the behavior of the assistant. For example, you can modify the personality of the assistant or provide specific instructions about how it should behave throughout the conversation. By default, the model’s behavior is using a generic message as \"You are a helpful assistant.\"", + "instillShortDescription": "The system message helps set the behavior of the assistant", + "instillUIMultiline": true, + "instillUIOrder": 2, + "title": "System message", + "type": "string" + }, + "temperature": { + "default": 0.7, + "description": "The temperature for sampling", + "instillUIOrder": 5, + "title": "Temperature", + "type": "number" + }, + "top_k": { + "default": 10, + "description": "Top k for sampling", + "instillUIOrder": 5, + "title": "Top K", + "type": "integer" + } + }, + "required": ["prompt", "prompt_images", "model_name"], + "title": "Input", + "type": "object" + }, + "output": { + "description": "Output", + "instillUIOrder": 0, + "properties": { + "text": { + "description": "Text", + "instillFormat": "string", + "instillUIMultiline": true, + "instillUIOrder": 0, + "title": "Text", + "type": "string" + } + }, + "required": ["text"], + "title": "Output", + "type": "object" + } + } +} diff --git a/config/model/task_input.json b/config/model/task_input.json new file mode 100644 index 00000000..1c85bffa --- /dev/null +++ b/config/model/task_input.json @@ -0,0 +1,622 @@ +{ + "TASK_CLASSIFICATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Classification", + "instillShortDescription": "Classify images into predefined categories.", + "properties": { + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillUIOrder": 0, + "instillAcceptFormats": [ + "image/*" + ], + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + } + }, + "required": [ + "image_base64" + ], + "type": "object" + }, + "TASK_DETECTION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Detection", + "instillShortDescription": "Detect and localize multiple objects in images.", + "properties": { + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillUIOrder": 0, + "instillAcceptFormats": [ + "image/*" + ], + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + } + }, + "required": [ + "image_base64" + ], + "type": "object" + }, + "TASK_OCR": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Ocr", + "instillShortDescription": "Detect and recognize text in images.", + "properties": { + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillUIOrder": 0, + "instillAcceptFormats": [ + "image/*" + ], + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + } + }, + "required": [ + "image_base64" + ], + "type": "object" + }, + "TASK_SEMANTIC_SEGMENTATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Semantic Segmentation", + "instillShortDescription": "Classify image pixels into predefined categories.", + "properties": { + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillUIOrder": 0, + "instillAcceptFormats": [ + "image/*" + ], + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + } + }, + "required": [ + "image_base64" + ], + "type": "object" + }, + "TASK_KEYPOINT": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Keypoint", + "instillShortDescription": "Detect and localize multiple keypoints of objects in images.", + "properties": { + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillUIOrder": 0, + "instillAcceptFormats": [ + "image/*" + ], + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + } + }, + "required": [ + "image_base64" + ], + "type": "object" + }, + "TASK_INSTANCE_SEGMENTATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Instance Segmentation", + "instillShortDescription": "Detect, localize and delineate multiple objects in images.", + "properties": { + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillUIOrder": 0, + "instillAcceptFormats": [ + "image/*" + ], + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + } + }, + "required": [ + "image_base64" + ], + "type": "object" + }, + "TASK_TEXT_TO_IMAGE": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Text To Image", + "instillShortDescription": "Generate images from input text prompts.", + "properties": { + "cfg_scale": { + "type": "number", + "description": "The guidance scale, default is 7.5", + "instillAcceptFormats": [ + "number", + "integer" + ], + "default": 7.5, + "instillShortDescription": "The guidance scale, default is 7.5", + "instillUIOrder": 2, + "title": "CFG Scale" + }, + "extra_params": { + "type": "string", + "description": "Extra Parameters", + "instillAcceptFormats": [ + "semi-structured/object" + ], + "instillShortDescription": "Extra Parameters", + "instillUIOrder": 6, + "title": "Extra Parameters" + }, + "prompt": { + "instillUIMultiline": true, + "type": "string", + "description": "The prompt text", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The prompt text", + "instillUIOrder": 1, + "title": "Prompt" + }, + "samples": { + "type": "integer", + "description": "The number of generated samples, default is 1", + "instillAcceptFormats": [ + "integer" + ], + "default": 1, + "instillShortDescription": "The number of generated samples, default is 1", + "instillUIOrder": 5, + "title": "Samples" + }, + "seed": { + "type": "integer", + "description": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0, + "instillShortDescription": "The seed, default is 0", + "instillUIOrder": 4, + "title": "Seed" + }, + "steps": { + "type": "integer", + "description": "The steps, default is 5", + "instillAcceptFormats": [ + "integer" + ], + "default": 5, + "instillShortDescription": "The steps, default is 5", + "instillUIOrder": 3, + "title": "Steps" + } + }, + "required": [ + "prompt" + ], + "type": "object" + }, + "TASK_IMAGE_TO_IMAGE": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Image To Image", + "instillShortDescription": "Generate image from input text prompt and image.", + "properties": { + "cfg_scale": { + "type": "number", + "description": "The guidance scale, default is 7.5", + "instillAcceptFormats": [ + "number", + "integer" + ], + "default": 7.5, + "instillShortDescription": "The guidance scale, default is 7.5", + "instillUIOrder": 3, + "title": "CFG Scale" + }, + "extra_params": { + "type": "string", + "description": "Extra Parameters", + "instillAcceptFormats": [ + "semi-structured/object" + ], + "instillShortDescription": "Extra Parameters", + "instillUIOrder": 6, + "title": "Extra Parameters" + }, + "image_base64": { + "description": "Image base64", + "instillShortDescription": "Image base64", + "instillAcceptFormats": [ + "image/*" + ], + "instillUIOrder": 2, + "title": "Image base64", + "pattern": "^[A-Za-z0-9+/]*={0,2}$", + "type": "string" + }, + "prompt": { + "instillUIMultiline": true, + "type": "string", + "description": "The prompt text", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The prompt text", + "instillUIOrder": 1, + "title": "Prompt" + }, + "samples": { + "type": "integer", + "description": "The number of generated samples, default is 1", + "instillAcceptFormats": [ + "integer" + ], + "default": 1, + "instillShortDescription": "The number of generated samples, default is 1", + "instillUIOrder": 5, + "title": "Samples" + }, + "seed": { + "type": "integer", + "description": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0, + "instillShortDescription": "The seed, default is 0", + "instillUIOrder": 4, + "title": "Seed" + }, + "top_k": { + "type": "integer", + "description": "Top k for sampling", + "instillAcceptFormats": [ + "integer" + ], + "default": 10, + "instillShortDescription": "Top k for sampling", + "instillUIOrder": 5, + "title": "Top K" + } + }, + "required": [ + "prompt", + "image_base64" + ], + "type": "object" + }, + "TASK_TEXT_GENERATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Text Generation", + "instillShortDescription": "Generate texts from input text prompts.", + "properties": { + "chat_history": { + "type": "string", + "description": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillAcceptFormats": [ + "structured/chat_messages" + ], + "instillShortDescription": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillUIOrder": 4, + "title": "Chat history" + }, + "extra_params": { + "type": "string", + "description": "Extra Parameters", + "instillAcceptFormats": [ + "semi-structured/object" + ], + "instillShortDescription": "Extra Parameters", + "instillUIOrder": 8, + "title": "Extra Parameters" + }, + "max_new_tokens": { + "type": "integer", + "default": 50, + "description": "The maximum number of tokens for model to generate", + "instillAcceptFormats": [ + "integer" + ], + "instillShortDescription": "The maximum number of tokens for model to generate", + "instillUIOrder": 7, + "title": "Max new tokens" + }, + "prompt": { + "instillUIMultiline": true, + "type": "string", + "description": "The prompt text", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The prompt text", + "instillUIOrder": 1, + "title": "Prompt" + }, + "prompt_images": { + "description": "The prompt images", + "instillAcceptFormats": [ + "array:image/*" + ], + "instillShortDescription": "The prompt images", + "instillUIOrder": 2, + "title": "Prompt Images" + }, + "seed": { + "type": "integer", + "description": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0, + "instillShortDescription": "The seed, default is 0", + "instillUIOrder": 6, + "title": "Seed" + }, + "system_message": { + "instillUIMultiline": true, + "type": "string", + "description": "The system message helps set the behavior of the assistant. For example, you can modify the personality of the assistant or provide specific instructions about how it should behave throughout the conversation. By default, the model’s behavior is using a generic message as \"You are a helpful assistant.\"", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The system message helps set the behavior of the assistant", + "instillUIOrder": 3, + "default": "You are a helpful assistant.", + "title": "System message" + }, + "temperature": { + "type": "number", + "default": 0.7, + "description": "The temperature for sampling", + "instillAcceptFormats": [ + "number" + ], + "instillShortDescription": "The temperature for sampling", + "instillUIOrder": 5, + "title": "Temperature" + }, + "top_k": { + "type": "integer", + "description": "Top k for sampling", + "instillAcceptFormats": [ + "integer" + ], + "default": 10, + "instillShortDescription": "Top k for sampling", + "instillUIOrder": 4, + "title": "Top K" + } + }, + "required": [ + "prompt" + ], + "type": "object" + }, + "TASK_TEXT_GENERATION_CHAT": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Text Generation Chat", + "instillShortDescription": "Generate texts from input text prompts and chat history.", + "properties": { + "chat_history": { + "type": "string", + "description": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillAcceptFormats": [ + "structured/chat_messages" + ], + "instillShortDescription": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillUIOrder": 4, + "title": "Chat history" + }, + "extra_params": { + "type": "string", + "description": "Extra Parameters", + "instillAcceptFormats": [ + "semi-structured/object" + ], + "instillShortDescription": "Extra Parameters", + "instillUIOrder": 8, + "title": "Extra Parameters" + }, + "max_new_tokens": { + "type": "integer", + "default": 50, + "description": "The maximum number of tokens for model to generate", + "instillAcceptFormats": [ + "integer" + ], + "instillShortDescription": "The maximum number of tokens for model to generate", + "instillUIOrder": 7, + "title": "Max new tokens" + }, + "prompt": { + "instillUIMultiline": true, + "type": "string", + "description": "The prompt text", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The prompt text", + "instillUIOrder": 1, + "title": "Prompt" + }, + "prompt_images": { + "description": "The prompt images", + "instillAcceptFormats": [ + "array:image/*" + ], + "instillShortDescription": "The prompt images", + "instillUIOrder": 2, + "title": "Prompt Images" + }, + "seed": { + "type": "integer", + "description": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0, + "instillShortDescription": "The seed, default is 0", + "instillUIOrder": 6, + "title": "Seed" + }, + "system_message": { + "instillUIMultiline": true, + "type": "string", + "description": "The system message helps set the behavior of the assistant. For example, you can modify the personality of the assistant or provide specific instructions about how it should behave throughout the conversation. By default, the model’s behavior is using a generic message as \"You are a helpful assistant.\"", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The system message helps set the behavior of the assistant", + "instillUIOrder": 3, + "default": "You are a helpful assistant.", + "title": "System message" + }, + "temperature": { + "type": "number", + "default": 0.7, + "description": "The temperature for sampling", + "instillAcceptFormats": [ + "number" + ], + "instillShortDescription": "The temperature for sampling", + "instillUIOrder": 5, + "title": "Temperature" + }, + "top_k": { + "type": "integer", + "description": "Top k for sampling", + "instillAcceptFormats": [ + "integer" + ], + "default": 10, + "instillShortDescription": "Top k for sampling", + "instillUIOrder": 4, + "title": "Top K" + } + }, + "required": [ + "prompt" + ], + "type": "object" + }, + "TASK_VISUAL_QUESTION_ANSWERING": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Visual Question Answering", + "instillShortDescription": "Answer questions based on a prompt and an image.", + "properties": { + "chat_history": { + "type": "string", + "description": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillAcceptFormats": [ + "structured/chat_messages" + ], + "instillShortDescription": "Incorporate external chat history, specifically previous messages within the conversation. Please note that System Message will be ignored and will not have any effect when this field is populated. Each message should adhere to the format: : {\"role\": \"The message role, i.e. 'system', 'user' or 'assistant'\", \"content\": \"message content\"}.", + "instillUIOrder": 4, + "title": "Chat history" + }, + "extra_params": { + "type": "string", + "description": "Extra Parameters", + "instillAcceptFormats": [ + "semi-structured/object" + ], + "instillShortDescription": "Extra Parameters", + "instillUIOrder": 8, + "title": "Extra Parameters" + }, + "max_new_tokens": { + "type": "integer", + "default": 50, + "description": "The maximum number of tokens for model to generate", + "instillAcceptFormats": [ + "integer" + ], + "instillShortDescription": "The maximum number of tokens for model to generate", + "instillUIOrder": 7, + "title": "Max new tokens" + }, + "prompt": { + "instillUIMultiline": true, + "type": "string", + "description": "The prompt text", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The prompt text", + "instillUIOrder": 1, + "title": "Prompt" + }, + "prompt_images": { + "description": "The prompt images", + "instillAcceptFormats": [ + "array:image/*" + ], + "instillShortDescription": "The prompt images", + "instillUIOrder": 2, + "title": "Prompt Images" + }, + "seed": { + "type": "integer", + "description": "The seed, default is 0", + "instillAcceptFormats": [ + "integer" + ], + "default": 0, + "instillShortDescription": "The seed, default is 0", + "instillUIOrder": 6, + "title": "Seed" + }, + "system_message": { + "instillUIMultiline": true, + "type": "string", + "description": "The system message helps set the behavior of the assistant. For example, you can modify the personality of the assistant or provide specific instructions about how it should behave throughout the conversation. By default, the model’s behavior is using a generic message as \"You are a helpful assistant.\"", + "instillAcceptFormats": [ + "string" + ], + "instillShortDescription": "The system message helps set the behavior of the assistant", + "instillUIOrder": 3, + "default": "You are a helpful assistant.", + "title": "System message" + }, + "temperature": { + "type": "number", + "default": 0.7, + "description": "The temperature for sampling", + "instillAcceptFormats": [ + "number" + ], + "instillShortDescription": "The temperature for sampling", + "instillUIOrder": 5, + "title": "Temperature" + }, + "top_k": { + "type": "integer", + "description": "Top k for sampling", + "instillAcceptFormats": [ + "integer" + ], + "default": 10, + "instillShortDescription": "Top k for sampling", + "instillUIOrder": 4, + "title": "Top K" + } + }, + "required": [ + "prompt", + "prompt_images" + ], + "type": "object" + } +} \ No newline at end of file diff --git a/config/model/task_output.json b/config/model/task_output.json new file mode 100644 index 00000000..4e0b7217 --- /dev/null +++ b/config/model/task_output.json @@ -0,0 +1,631 @@ +{ + "TASK_CLASSIFICATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Classification", + "instillShortDescription": "Classify images into predefined categories.", + "instillUIOrder": 0, + "properties": { + "category": { + "description": "The predicted category of the input.", + "instillFormat": "string", + "instillShortDescription": "The predicted category of the input.", + "instillUIOrder": 0, + "title": "Category", + "type": "string" + }, + "score": { + "description": "The confidence score of the predicted category of the input.", + "instillFormat": "number", + "instillShortDescription": "The confidence score of the predicted category of the input.", + "instillUIOrder": 1, + "title": "Score", + "type": "number" + } + }, + "required": [ + "category", + "score" + ], + "type": "object" + }, + "TASK_DETECTION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Detection", + "instillShortDescription": "Detect and localize multiple objects in images.", + "instillUIOrder": 0, + "properties": { + "objects": { + "description": "A list of detected objects.", + "instillFormat": "array:structured/detection_object", + "instillUIOrder": 0, + "items": { + "additionalProperties": false, + "instillFormat": "structured/detection_object", + "instillUIOrder": 0, + "properties": { + "bounding_box": { + "additionalProperties": false, + "description": "The detected bounding box in (left, top, width, height) format.", + "instillFormat": "structured/bounding_box", + "instillUIOrder": 1, + "properties": { + "height": { + "description": "Bounding box height value", + "instillFormat": "number", + "instillShortDescription": "Bounding box height value", + "instillUIOrder": 3, + "title": "Height", + "type": "number" + }, + "left": { + "description": "Bounding box left x-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box left x-axis value", + "instillUIOrder": 0, + "title": "Left", + "type": "number" + }, + "top": { + "description": "Bounding box top y-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box top y-axis value", + "instillUIOrder": 1, + "title": "Top", + "type": "number" + }, + "width": { + "description": "Bounding box width value", + "instillFormat": "number", + "instillShortDescription": "Bounding box width value", + "instillUIOrder": 2, + "title": "Width", + "type": "number" + } + }, + "required": [ + "left", + "top", + "width", + "height" + ], + "title": "Bounding box", + "type": "object" + }, + "category": { + "description": "The predicted category of the bounding box.", + "instillFormat": "string", + "instillShortDescription": "The predicted category of the bounding box.", + "instillUIOrder": 2, + "title": "Category", + "type": "string" + }, + "score": { + "description": "The confidence score of the predicted category of the bounding box.", + "instillFormat": "number", + "instillShortDescription": "The confidence score of the predicted category of the bounding box.", + "instillUIOrder": 3, + "title": "Score", + "type": "number" + } + }, + "required": [ + "bounding_box", + "category", + "score" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "TASK_OCR": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Ocr", + "instillShortDescription": "Detect and recognize text in images.", + "instillUIOrder": 0, + "properties": { + "objects": { + "description": "A list of detected bounding boxes.", + "instillFormat": "array:structured/ocr_object", + "instillUIOrder": 0, + "items": { + "instillFormat": "structured/ocr_object", + "instillUIOrder": 0, + "properties": { + "bounding_box": { + "additionalProperties": false, + "description": "The detected bounding box in (left, top, width, height) format.", + "instillFormat": "structured/bounding_box", + "instillUIOrder": 0, + "properties": { + "height": { + "description": "Bounding box height value", + "instillFormat": "number", + "instillShortDescription": "Bounding box height value", + "instillUIOrder": 3, + "title": "Height", + "type": "number" + }, + "left": { + "description": "Bounding box left x-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box left x-axis value", + "instillUIOrder": 0, + "title": "Left", + "type": "number" + }, + "top": { + "description": "Bounding box top y-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box top y-axis value", + "instillUIOrder": 1, + "title": "Top", + "type": "number" + }, + "width": { + "description": "Bounding box width value", + "instillFormat": "number", + "instillShortDescription": "Bounding box width value", + "instillUIOrder": 2, + "title": "Width", + "type": "number" + } + }, + "required": [ + "left", + "top", + "width", + "height" + ], + "title": "Bounding Box", + "type": "object" + }, + "score": { + "description": "The confidence score of the predicted object.", + "instillFormat": "number", + "instillShortDescription": "The confidence score of the predicted object.", + "instillUIOrder": 2, + "title": "Score", + "type": "number" + }, + "text": { + "description": "Text string recognised per bounding box.", + "instillFormat": "string", + "instillShortDescription": "Text string recognised per bounding box.", + "instillUIOrder": 1, + "title": "Text", + "type": "string" + } + }, + "required": [ + "bounding_box", + "text", + "score" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "TASK_SEMANTIC_SEGMENTATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Semantic Segmentation", + "instillShortDescription": "Classify image pixels into predefined categories.", + "instillUIOrder": 0, + "properties": { + "stuffs": { + "description": "A list of RLE binary masks.", + "instillFormat": "array:structured/semantic_segmentation_stuff", + "instillUIOrder": 0, + "items": { + "instillFormat": "structured/semantic_segmentation_stuff", + "instillUIOrder": 0, + "properties": { + "category": { + "description": "Category text string corresponding to each stuff mask.", + "instillFormat": "string", + "instillShortDescription": "Category text string corresponding to each stuff mask.", + "instillUIOrder": 1, + "title": "Category", + "type": "string" + }, + "rle": { + "description": "Run Length Encoding (RLE) of each stuff mask within the image.", + "instillFormat": "string", + "instillShortDescription": "Run Length Encoding (RLE) of each stuff mask within the image.", + "instillUIOrder": 0, + "title": "RLE", + "type": "string" + } + }, + "required": [ + "rle", + "category" + ], + "title": "Object", + "type": "object" + }, + "title": "Stuffs", + "type": "array" + } + }, + "required": [ + "stuffs" + ], + "type": "object" + }, + "TASK_KEYPOINT": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Keypoint", + "instillShortDescription": "Detect and localize multiple keypoints of objects in images.", + "instillUIOrder": 0, + "properties": { + "objects": { + "description": "A list of keypoint objects, a keypoint object includes all the pre-defined keypoints of a detected object.", + "instillFormat": "array:structured/keypoint_object", + "instillUIOrder": 0, + "items": { + "instillFormat": "structured/keypoint_object", + "instillUIOrder": 0, + "properties": { + "bounding_box": { + "additionalProperties": false, + "description": "The detected bounding box in (left, top, width, height) format.", + "instillFormat": "structured/bounding_box", + "instillUIOrder": 2, + "properties": { + "height": { + "description": "Bounding box height value", + "instillFormat": "number", + "instillShortDescription": "Bounding box height value", + "instillUIOrder": 3, + "title": "Height", + "type": "number" + }, + "left": { + "description": "Bounding box left x-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box left x-axis value", + "instillUIOrder": 0, + "title": "Left", + "type": "number" + }, + "top": { + "description": "Bounding box top y-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box top y-axis value", + "instillUIOrder": 1, + "title": "Top", + "type": "number" + }, + "width": { + "description": "Bounding box width value", + "instillFormat": "number", + "instillShortDescription": "Bounding box width value", + "instillUIOrder": 2, + "title": "Width", + "type": "number" + } + }, + "required": [ + "left", + "top", + "width", + "height" + ], + "title": "Bounding Box", + "type": "object" + }, + "keypoints": { + "description": "A keypoint group is composed of a list of pre-defined keypoints of a detected object.", + "instillUIOrder": 0, + "items": { + "instillUIOrder": 0, + "properties": { + "v": { + "description": "visibility score of the keypoint.", + "instillFormat": "number", + "instillShortDescription": "visibility score of the keypoint.", + "instillUIOrder": 2, + "title": "", + "type": "number" + }, + "x": { + "description": "x coordinate of the keypoint.", + "instillFormat": "number", + "instillShortDescription": "x coordinate of the keypoint.", + "instillUIOrder": 0, + "title": "", + "type": "number" + }, + "y": { + "description": "y coordinate of the keypoint.", + "instillFormat": "number", + "instillShortDescription": "y coordinate of the keypoint.", + "instillUIOrder": 1, + "title": "", + "type": "number" + } + }, + "required": [ + "x", + "y", + "v" + ], + "title": "Keypoints", + "type": "object" + }, + "title": "Keypoints", + "type": "array" + }, + "score": { + "description": "The confidence score of the predicted object.", + "instillFormat": "number", + "instillShortDescription": "The confidence score of the predicted object.", + "instillUIOrder": 1, + "title": "Score", + "type": "number" + } + }, + "required": [ + "keypoints", + "score", + "bounding_box" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "TASK_INSTANCE_SEGMENTATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Instance Segmentation", + "instillShortDescription": "Detect, localize and delineate multiple objects in images.", + "instillUIOrder": 0, + "properties": { + "objects": { + "description": "A list of detected instance bounding boxes.", + "instillFormat": "array:structured/instance_segmentation_object", + "instillUIOrder": 0, + "items": { + "instillEditOnNodeFields": [ + "rle", + "bounding_box", + "category", + "score" + ], + "instillFormat": "structured/instance_segmentation_object", + "instillUIOrder": 0, + "properties": { + "bounding_box": { + "additionalProperties": false, + "description": "The detected bounding box in (left, top, width, height) format.", + "instillEditOnNodeFields": [ + "left", + "top", + "width", + "height" + ], + "instillFormat": "structured/bounding_box", + "instillUIOrder": 1, + "properties": { + "height": { + "description": "Bounding box height value", + "instillFormat": "number", + "instillShortDescription": "Bounding box height value", + "instillUIOrder": 3, + "title": "Height", + "type": "number" + }, + "left": { + "description": "Bounding box left x-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box left x-axis value", + "instillUIOrder": 0, + "title": "Left", + "type": "number" + }, + "top": { + "description": "Bounding box top y-axis value", + "instillFormat": "number", + "instillShortDescription": "Bounding box top y-axis value", + "instillUIOrder": 1, + "title": "Top", + "type": "number" + }, + "width": { + "description": "Bounding box width value", + "instillFormat": "number", + "instillShortDescription": "Bounding box width value", + "instillUIOrder": 2, + "title": "Width", + "type": "number" + } + }, + "required": [ + "left", + "top", + "width", + "height" + ], + "title": "Bounding Box", + "type": "object" + }, + "category": { + "description": "The predicted category of the bounding box.", + "instillFormat": "string", + "instillShortDescription": "The predicted category of the bounding box.", + "instillUIOrder": 2, + "title": "Category", + "type": "string" + }, + "rle": { + "description": "Run Length Encoding (RLE) of instance mask within the bounding box.", + "instillFormat": "string", + "instillShortDescription": "Run Length Encoding (RLE) of instance mask within the bounding box.", + "instillUIOrder": 0, + "title": "RLE", + "type": "string" + }, + "score": { + "description": "The confidence score of the predicted instance object.", + "instillFormat": "number", + "instillShortDescription": "The confidence score of the predicted instance object.", + "instillUIOrder": 3, + "title": "Score", + "type": "number" + } + }, + "required": [ + "rle", + "bounding_box", + "category", + "score" + ], + "title": "Object", + "type": "object" + }, + "title": "Objects", + "type": "array" + } + }, + "required": [ + "objects" + ], + "type": "object" + }, + "TASK_TEXT_TO_IMAGE": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Text To Image", + "instillShortDescription": "Generate images from input text prompts.", + "instillUIOrder": 0, + "properties": { + "images": { + "description": "Images", + "instillFormat": "array:image/jpeg", + "instillUIOrder": 0, + "items": { + "description": "", + "instillFormat": "image/jpeg", + "instillShortDescription": "", + "instillUIOrder": 0, + "title": "Image", + "type": "string" + }, + "title": "Images", + "type": "array" + } + }, + "required": [ + "images" + ], + "type": "object" + }, + "TASK_IMAGE_TO_IMAGE": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Image To Image", + "instillShortDescription": "Generate image from input text prompt and image.", + "instillUIOrder": 0, + "properties": { + "images": { + "description": "Images", + "instillFormat": "array:image/jpeg", + "instillUIOrder": 0, + "items": { + "description": "", + "instillFormat": "image/jpeg", + "instillShortDescription": "", + "instillUIOrder": 0, + "title": "Image", + "type": "string" + }, + "title": "Images", + "type": "array" + } + }, + "required": [ + "images" + ], + "type": "object" + }, + "TASK_TEXT_GENERATION": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Text Generation", + "instillShortDescription": "Generate texts from input text prompts.", + "instillUIOrder": 0, + "properties": { + "text": { + "description": "Text", + "instillFormat": "string", + "instillShortDescription": "Text", + "instillUIOrder": 0, + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "type": "object" + }, + "TASK_TEXT_GENERATION_CHAT": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Text Generation Chat", + "instillShortDescription": "Generate texts from input text prompts and chat history.", + "instillUIOrder": 0, + "properties": { + "text": { + "description": "Text", + "instillFormat": "string", + "instillShortDescription": "Text", + "instillUIOrder": 0, + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "type": "object" + }, + "TASK_VISUAL_QUESTION_ANSWERING": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Visual Question Answering", + "instillShortDescription": "Answer questions based on a prompt and an image.", + "properties": { + "text": { + "description": "Text", + "instillFormat": "string", + "instillUIMultiline": true, + "instillUIOrder": 0, + "title": "Text", + "type": "string" + } + }, + "required": [ + "text" + ], + "type": "object" + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index f36828f5..42b02902 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 github.com/iancoleman/strcase v0.3.0 - github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240521161711-af99b826b207 + github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240522171446-6efac14a674e github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a github.com/instill-ai/x v0.4.0-alpha github.com/knadh/koanf v1.5.0 diff --git a/go.sum b/go.sum index 5cc911e1..0bc83fc7 100644 --- a/go.sum +++ b/go.sum @@ -1316,6 +1316,8 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240521161711-af99b826b207 h1:HIICqyngAMGcFTJwa10AOmAGSDEtTmFvPGfPXOzLeyU= github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240521161711-af99b826b207/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ= +github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240522171446-6efac14a674e h1:RIAH7ibosC8H6yC6MutFvjVdFyvSsV2gFwqQWlrqzmM= +github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240522171446-6efac14a674e/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ= github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw= github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw= github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg= diff --git a/pkg/datamodel/jsonschema.go b/pkg/datamodel/jsonschema.go index 48d7cc34..fe78ae03 100644 --- a/pkg/datamodel/jsonschema.go +++ b/pkg/datamodel/jsonschema.go @@ -31,6 +31,9 @@ var GCSServiceAccountJSONSchema *jsonschema.Schema var RegionHardwareJSON RegionHardware +var TaskInputJSON map[string]any +var TaskOutputJSON map[string]any + type RegionHardware struct { Properties struct { Region struct { @@ -152,6 +155,22 @@ func InitJSONSchema(ctx context.Context) { if err != nil { logger.Fatal(fmt.Sprintf("%#v\n", err.Error())) } + + taskInputJSONFile, err := os.ReadFile("config/model/task_input.json") + if err != nil { + logger.Fatal(fmt.Sprintf("%#v\n", err.Error())) + } + if err := json.Unmarshal(taskInputJSONFile, &TaskInputJSON); err != nil { + logger.Fatal(fmt.Sprintf("%#v\n", err.Error())) + } + + taskOutputJSONFile, err := os.ReadFile("config/model/task_output.json") + if err != nil { + logger.Fatal(fmt.Sprintf("%#v\n", err.Error())) + } + if err := json.Unmarshal(taskOutputJSONFile, &TaskOutputJSON); err != nil { + logger.Fatal(fmt.Sprintf("%#v\n", err.Error())) + } } // ValidateJSONSchema validates the Protobuf message data diff --git a/pkg/ray/rayserver/ray_server.proto b/pkg/ray/rayserver/ray_server.proto new file mode 100644 index 00000000..b48d55ae --- /dev/null +++ b/pkg/ray/rayserver/ray_server.proto @@ -0,0 +1,306 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +option cc_enable_arenas = true; + +package ray.serve; + +option go_package = "./rayserver"; + + +// Configuration options for Serve's replica autoscaler. +message AutoscalingConfig { + // Minimal number of replicas, must be a non-negative integer. + uint32 min_replicas = 1; + + // Maximal number of replicas, must be a non-negative integer and greater or equals + // to min_replicas. + uint32 max_replicas = 2; + + // Target number of in flight requests per replicas. This is the primary configuration + // knob for replica autoscaler. Lower the number, the more rapidly will the replicas + // being scaled up. Must be a non-negative integer. + double target_num_ongoing_requests_per_replica = 3; + + // The frequency of how long does each replica sending metrics to autoscaler. + double metrics_interval_s = 4; + + // The window (in seconds) for autoscaler to calculate rolling average of metrics on. + double look_back_period_s = 5; + + // The multiplicative "gain" factor to limit scaling decisions. + double smoothing_factor = 6; + + // How long to wait before scaling down replicas. + double downscale_delay_s = 7; + + // How long to wait before scaling up replicas. + double upscale_delay_s = 8; + + // Initial number of replicas deployment should start with. Must be non-negative. + optional uint32 initial_replicas = 9; + + // The multiplicative "gain" factor to limit upscale. + optional double upscale_smoothing_factor = 10; + + // The multiplicative "gain" factor to limit downscale. + optional double downscale_smoothing_factor = 11; + + // The cloudpickled policy definition. + bytes serialized_policy_def = 12; + + // The import path of the policy if user passed a string. Will be the concatenation + // of the policy module and the policy name if user passed a callable. + string policy = 13; +} + +//[Begin] LOGGING CONFIG +// Encoding type +enum EncodingType { + TEXT = 0; + JSON = 1; +} + +message LoggingConfig { + EncodingType encoding = 1; + string log_level = 2; + string logs_dir = 3; + bool enable_access_log = 4; +} + +//[End] Logging Config + +// Configuration options for a deployment, to be set by the user. +message DeploymentConfig { + // The number of processes to start up that will handle requests to this deployment. + // Defaults to 1. + int32 num_replicas = 1; + + // The maximum number of queries that will be sent to a replica of this deployment + // without receiving a response. Defaults to 100. + int32 max_concurrent_queries = 2; + + // Arguments to pass to the reconfigure method of the deployment. The reconfigure method + // is called if user_config is not None. + bytes user_config = 3; + + // Duration that deployment replicas will wait until there is no more work to be done + // before shutting down. + double graceful_shutdown_wait_loop_s = 4; + + // Controller waits for this duration to forcefully kill the replica for shutdown. + double graceful_shutdown_timeout_s = 5; + + // Frequency at which the controller health checks replicas. + double health_check_period_s = 6; + + // Timeout after which a replica is marked unhealthy without a response. + double health_check_timeout_s = 7; + + // Is the construction of deployment is cross language? + bool is_cross_language = 8; + + // The deployment's programming language. + DeploymentLanguage deployment_language = 9; + + // The deployment's autoscaling configuration. + AutoscalingConfig autoscaling_config = 10; + + string version = 11; + + repeated string user_configured_option_names = 12; + + LoggingConfig logging_config = 13; +} + +// Deployment language. +enum DeploymentLanguage { + PYTHON = 0; + JAVA = 1; +} + +message RequestMetadata { + string request_id = 1; + + string endpoint = 2; + + string call_method = 3; + + map context = 4; + + string multiplexed_model_id = 5; + + string route = 6; +} + +message RequestWrapper { + bytes body = 1; +} + +message UpdatedObject { + bytes object_snapshot = 1; + int32 snapshot_id = 2; +} + +message LongPollRequest { + map keys_to_snapshot_ids = 1; +} + +message LongPollResult { + map updated_objects = 1; +} + +message EndpointInfo { + string endpoint_name = 1; + string route = 2; + map config = 3; +} + +message EndpointSet { + map endpoints = 1; +} + +// Now Actor handle can be transfered across language through ray call, but the list of +// Actor handles can't. So we use this message wrapped a Actor name list to pass actor +// list across language. When Actor handle list supports across language, this message can +// be replaced. +message ActorNameList { + repeated string names = 1; +} + +message DeploymentVersion { + string code_version = 1; + DeploymentConfig deployment_config = 2; + string ray_actor_options = 3; + string placement_group_bundles = 4; + string placement_group_strategy = 5; + int32 max_replicas_per_node = 6; +} + +message ReplicaConfig { + string deployment_def_name = 1; + bytes deployment_def = 2; + bytes init_args = 3; + bytes init_kwargs = 4; + string ray_actor_options = 5; + string placement_group_bundles = 6; + string placement_group_strategy = 7; + int32 max_replicas_per_node = 8; +} + +enum TargetCapacityDirection { + UNSET = 0; + UP = 1; + DOWN = 2; +} + +message DeploymentInfo { + string name = 1; + DeploymentConfig deployment_config = 2; + ReplicaConfig replica_config = 3; + int64 start_time_ms = 4; + string actor_name = 5; + string version = 6; + int64 end_time_ms = 7; + double target_capacity = 8; + TargetCapacityDirection target_capacity_direction = 9; +} + +// Wrap DeploymentInfo and route. The "" route value need to be convert to None/null. +message DeploymentRoute { + DeploymentInfo deployment_info = 1; + string route = 2; +} + +// Wrap a list for DeploymentRoute. +message DeploymentRouteList { + repeated DeploymentRoute deployment_routes = 1; +} + +enum DeploymentStatus { + // Keep frontend code of ServeDeploymentStatus in dashboard/client/src/type/serve.ts + // in sync with this enum + DEPLOYMENT_STATUS_UPDATING = 0; + DEPLOYMENT_STATUS_HEALTHY = 1; + DEPLOYMENT_STATUS_UNHEALTHY = 2; + DEPLOYMENT_STATUS_UPSCALING = 3; + DEPLOYMENT_STATUS_DOWNSCALING = 4; +} + +enum DeploymentStatusTrigger { + DEPLOYMENT_STATUS_TRIGGER_UNSPECIFIED = 0; + DEPLOYMENT_STATUS_TRIGGER_CONFIG_UPDATE_STARTED = 1; + DEPLOYMENT_STATUS_TRIGGER_CONFIG_UPDATE_COMPLETED = 2; + DEPLOYMENT_STATUS_TRIGGER_UPSCALE_COMPLETED = 3; + DEPLOYMENT_STATUS_TRIGGER_DOWNSCALE_COMPLETED = 4; + DEPLOYMENT_STATUS_TRIGGER_AUTOSCALING = 5; + DEPLOYMENT_STATUS_TRIGGER_REPLICA_STARTUP_FAILED = 6; + DEPLOYMENT_STATUS_TRIGGER_HEALTH_CHECK_FAILED = 7; + DEPLOYMENT_STATUS_TRIGGER_INTERNAL_ERROR = 8; + DEPLOYMENT_STATUS_TRIGGER_DELETING = 9; +} + +message DeploymentStatusInfo { + string name = 1; + DeploymentStatus status = 2; + string message = 3; + DeploymentStatusTrigger status_trigger = 4; +} + +// Wrap a list for DeploymentStatusInfo. +message DeploymentStatusInfoList { + repeated DeploymentStatusInfo deployment_status_infos = 1; +} + +enum ApplicationStatus { + // Keep frontend code of ServeApplicationStatus in dashboard/client/src/type/serve.ts + // in sync with this enum + APPLICATION_STATUS_DEPLOYING = 0; + APPLICATION_STATUS_RUNNING = 1; + APPLICATION_STATUS_DEPLOY_FAILED = 2; + APPLICATION_STATUS_DELETING = 3; + APPLICATION_STATUS_NOT_STARTED = 5; + APPLICATION_STATUS_UNHEALTHY = 6; +} + +message ApplicationStatusInfo { + ApplicationStatus status = 1; + string message = 2; + double deployment_timestamp = 3; +} + +message StatusOverview { + ApplicationStatusInfo app_status = 1; + DeploymentStatusInfoList deployment_statuses = 2; + string name = 3; +} + +// Used for gRPC proxy health check +message ListApplicationsRequest {} + +message ListApplicationsResponse { + repeated string application_names = 1; +} + +message HealthzRequest {} + +message HealthzResponse { + string message = 1; +} + +service RayServeAPIService { + rpc ListApplications(ListApplicationsRequest) returns (ListApplicationsResponse); + rpc Healthz(HealthzRequest) returns (HealthzResponse); +} diff --git a/pkg/service/convert.go b/pkg/service/convert.go index 75c84627..ceeea1f6 100644 --- a/pkg/service/convert.go +++ b/pkg/service/convert.go @@ -29,6 +29,7 @@ import ( custom_logger "github.com/instill-ai/model-backend/pkg/logger" commonPB "github.com/instill-ai/protogen-go/common/task/v1alpha" + mgmtPB "github.com/instill-ai/protogen-go/core/mgmt/v1beta" modelPB "github.com/instill-ai/protogen-go/model/model/v1alpha" ) @@ -142,17 +143,13 @@ func (s *service) PBToDBModel(ctx context.Context, ns resource.Namespace, pbMode }, nil } -func (s *service) DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefinition, dbModel *datamodel.Model, checkPermission bool) (*modelPB.Model, error) { +func (s *service) DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefinition, dbModel *datamodel.Model, view modelPB.View, checkPermission bool) (*modelPB.Model, error) { logger, _ := custom_logger.GetZapLogger(ctx) ownerName, err := s.ConvertOwnerPermalinkToName(dbModel.Owner) if err != nil { return nil, err } - owner, err := s.FetchOwnerWithPermalink(dbModel.Owner) - if err != nil { - return nil, err - } ctxUserUID := resource.GetRequestSingleHeader(ctx, constant.HeaderUserUIDKey) @@ -186,7 +183,6 @@ func (s *service) DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefi return nil }(), OwnerName: ownerName, - Owner: owner, Region: dbModel.Region, Hardware: dbModel.Hardware, SourceUrl: &dbModel.SourceURL.String, @@ -197,7 +193,7 @@ func (s *service) DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefi } var wg sync.WaitGroup - wg.Add(2) + wg.Add(3) pbModel.Permission = &modelPB.Permission{} go func() { defer wg.Done() @@ -231,15 +227,48 @@ func (s *service) DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefi } pbModel.Permission.CanTrigger = canTrigger }() + go func() { + defer wg.Done() + var owner *mgmtPB.Owner + if view > modelPB.View_VIEW_BASIC { + owner, err = s.FetchOwnerWithPermalink(dbModel.Owner) + if err != nil { + return + } + pbModel.Owner = owner + } + }() - wg.Wait() + if view > modelPB.View_VIEW_BASIC { - appendSampleInputOutput(&pbModel) + inputTaskSchema, ok := datamodel.TaskInputJSON[pbModel.Task.String()].(map[string]any) + if !ok { + return nil, fmt.Errorf("value for key %s is not a JSON object", pbModel.Task.String()) + } + outputTaskSchema, ok := datamodel.TaskOutputJSON[pbModel.Task.String()].(map[string]any) + if !ok { + return nil, fmt.Errorf("value for key %s is not a JSON object", pbModel.Task.String()) + } + inputStruct, err := structpb.NewStruct(inputTaskSchema) + if err != nil { + return nil, err + } + outputStruct, err := structpb.NewStruct(outputTaskSchema) + if err != nil { + return nil, err + } + pbModel.InputSchema = inputStruct + pbModel.OutputSchema = outputStruct + + appendSampleInputOutput(&pbModel) + } + + wg.Wait() return &pbModel, nil } -func (s *service) DBToPBModels(ctx context.Context, dbModels []*datamodel.Model, checkPermission bool) ([]*modelPB.Model, error) { +func (s *service) DBToPBModels(ctx context.Context, dbModels []*datamodel.Model, view modelPB.View, checkPermission bool) ([]*modelPB.Model, error) { pbModels := make([]*modelPB.Model, len(dbModels)) @@ -253,6 +282,7 @@ func (s *service) DBToPBModels(ctx context.Context, dbModels []*datamodel.Model, ctx, modelDef, dbModels[idx], + view, checkPermission, ) if err != nil { @@ -512,7 +542,6 @@ func appendSampleInputOutput(pbModel *modelPB.Model) { TextToImage: &modelPB.TextToImageOutput{ Images: []string{ "/9j/4AAQSkZJRgABAQAAAQABAAD/...", - "/9j/2wCEAAEBAQEBAQEFKEPOFAFD...", }, }, } diff --git a/pkg/service/service.go b/pkg/service/service.go index ab670c3a..600df4d8 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -51,8 +51,8 @@ type Service interface { ConvertOwnerNameToPermalink(name string) (string, error) ConvertRepositoryNameToRscName(repositoryName string) (string, error) PBToDBModel(ctx context.Context, ns resource.Namespace, pbModel *modelPB.Model) (*datamodel.Model, error) - DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefinition, dbModel *datamodel.Model, checkPermission bool) (*modelPB.Model, error) - DBToPBModels(ctx context.Context, dbModels []*datamodel.Model, checkPermission bool) ([]*modelPB.Model, error) + DBToPBModel(ctx context.Context, modelDef *datamodel.ModelDefinition, dbModel *datamodel.Model, view modelPB.View, checkPermission bool) (*modelPB.Model, error) + DBToPBModels(ctx context.Context, dbModels []*datamodel.Model, view modelPB.View, checkPermission bool) ([]*modelPB.Model, error) DBToPBModelDefinition(ctx context.Context, dbModelDefinition *datamodel.ModelDefinition) (*modelPB.ModelDefinition, error) DBToPBModelDefinitions(ctx context.Context, dbModelDefinitions []*datamodel.ModelDefinition) ([]*modelPB.ModelDefinition, error) @@ -273,7 +273,7 @@ func (s *service) GetModelByUID(ctx context.Context, modelUID uuid.UUID, view mo return nil, err } - return s.DBToPBModel(ctx, modelDef, dbModel, true) + return s.DBToPBModel(ctx, modelDef, dbModel, view, true) } func (s *service) GetModelByIDAdmin(ctx context.Context, ns resource.Namespace, modelID string, view modelPB.View) (*modelPB.Model, error) { @@ -288,7 +288,7 @@ func (s *service) GetModelByIDAdmin(ctx context.Context, ns resource.Namespace, return nil, err } - return s.DBToPBModel(ctx, modelDef, dbModel, true) + return s.DBToPBModel(ctx, modelDef, dbModel, view, true) } func (s *service) GetModelByUIDAdmin(ctx context.Context, modelUID uuid.UUID, view modelPB.View) (*modelPB.Model, error) { @@ -303,7 +303,7 @@ func (s *service) GetModelByUIDAdmin(ctx context.Context, modelUID uuid.UUID, vi return nil, err } - return s.DBToPBModel(ctx, modelDef, dbModel, true) + return s.DBToPBModel(ctx, modelDef, dbModel, view, true) } func (s *service) GetNamespaceModelByID(ctx context.Context, ns resource.Namespace, modelID string, view modelPB.View) (*modelPB.Model, error) { @@ -326,7 +326,7 @@ func (s *service) GetNamespaceModelByID(ctx context.Context, ns resource.Namespa return nil, err } - return s.DBToPBModel(ctx, modelDef, dbModel, true) + return s.DBToPBModel(ctx, modelDef, dbModel, view, true) } func (s *service) CreateNamespaceModel(ctx context.Context, ns resource.Namespace, model *datamodel.Model) error { @@ -598,7 +598,7 @@ func (s *service) ListModels(ctx context.Context, pageSize int32, pageToken stri if err != nil { return nil, 0, "", err } - pbModels, err := s.DBToPBModels(ctx, dbModels, true) + pbModels, err := s.DBToPBModels(ctx, dbModels, view, true) return pbModels, int32(totalSize), nextPageToken, err } @@ -640,7 +640,7 @@ func (s *service) ListNamespaceModels(ctx context.Context, ns resource.Namespace return nil, 0, "", err } - pbModels, err := s.DBToPBModels(ctx, dbModels, true) + pbModels, err := s.DBToPBModels(ctx, dbModels, view, true) return pbModels, int32(ps), pt, err } @@ -695,7 +695,7 @@ func (s *service) ListModelsAdmin(ctx context.Context, pageSize int32, pageToken return nil, 0, "", err } - pbModels, err := s.DBToPBModels(ctx, dbModels, true) + pbModels, err := s.DBToPBModels(ctx, dbModels, view, true) return pbModels, int32(totalSize), nextPageToken, err } @@ -778,7 +778,7 @@ func (s *service) RenameNamespaceModelByID(ctx context.Context, ns resource.Name return nil, err } - return s.DBToPBModel(ctx, modelDef, updatedDBModel, true) + return s.DBToPBModel(ctx, modelDef, updatedDBModel, modelPB.View_VIEW_BASIC, true) } func (s *service) UpdateNamespaceModelByID(ctx context.Context, ns resource.Namespace, modelID string, toUpdateModel *modelPB.Model, reDeploy bool) (*modelPB.Model, error) { @@ -845,7 +845,7 @@ func (s *service) UpdateNamespaceModelByID(ctx context.Context, ns resource.Name } } - return s.DBToPBModel(ctx, modelDef, updatedDBModel, true) + return s.DBToPBModel(ctx, modelDef, updatedDBModel, modelPB.View_VIEW_BASIC, true) } func (s *service) GetModelDefinition(ctx context.Context, id string) (*modelPB.ModelDefinition, error) {