From 87ea7b7845431092f3dd4f5b25ac12b9105890f3 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Sun, 30 Jun 2024 03:04:18 +0800 Subject: [PATCH 1/2] feat(ray): unify cli commands --- instill/helpers/build.py | 144 ----------- instill/helpers/cli.py | 229 ++++++++++++++++++ .../{ => init-templates}/.dockerignore | 0 .../helpers/{ => init-templates}/Dockerfile | 0 instill/helpers/init-templates/instill.yaml | 20 ++ instill/helpers/init-templates/model.py | 78 ++++++ instill/helpers/push.py | 53 ---- poetry.lock | 13 +- pyproject.toml | 3 +- 9 files changed, 334 insertions(+), 206 deletions(-) delete mode 100644 instill/helpers/build.py create mode 100644 instill/helpers/cli.py rename instill/helpers/{ => init-templates}/.dockerignore (100%) rename instill/helpers/{ => init-templates}/Dockerfile (100%) create mode 100644 instill/helpers/init-templates/instill.yaml create mode 100644 instill/helpers/init-templates/model.py delete mode 100644 instill/helpers/push.py diff --git a/instill/helpers/build.py b/instill/helpers/build.py deleted file mode 100644 index 5b5aa3e..0000000 --- a/instill/helpers/build.py +++ /dev/null @@ -1,144 +0,0 @@ -import argparse -import hashlib -import os -import platform -import shutil -import subprocess -import tempfile - -import ray -import yaml - -import instill -from instill.helpers.const import DEFAULT_DEPENDENCIES -from instill.helpers.errors import ModelConfigException -from instill.utils.logger import Logger - - -def config_check_required_fields(c): - if "build" not in c or c["build"] is None: - raise ModelConfigException("build") - if "gpu" not in c["build"] or c["build"]["gpu"] is None: - raise ModelConfigException("gpu") - if "python_version" not in c["build"] or c["build"]["python_version"] is None: - raise ModelConfigException("python_version") - if "repo" not in c or c["repo"] is None: - raise ModelConfigException("repo") - - -def build_image(): - if platform.machine() in ("i386", "AMD64", "x86_64"): - default_platform = "amd64" - else: - default_platform = platform.machine() - parser = argparse.ArgumentParser() - parser.add_argument( - "-t", - "--tag", - help="tag for the model image", - default=hashlib.sha256().hexdigest(), - required=False, - ) - parser.add_argument( - "--no-cache", - help="build the image without cache", - action="store_true", - required=False, - ) - parser.add_argument( - "--target-arch", - help="target platform architecture for the model image, default to host", - default=default_platform, - choices=["arm64", "amd64"], - required=False, - ) - - args = parser.parse_args() - try: - Logger.i("[Instill Builder] Loading config file...") - with open("instill.yaml", "r", encoding="utf8") as f: - Logger.i("[Instill Builder] Parsing config file...") - config = yaml.safe_load(f) - - config_check_required_fields(config) - - build = config["build"] - repo = config["repo"] - - python_version = build["python_version"].replace(".", "") - ray_version = ray.__version__ - instill_version = instill.__version__ - - if not build["gpu"]: - cuda_suffix = "" - elif "cuda_version" in build and not build["cuda_version"] is None: - cuda_suffix = f'-cu{build["cuda_version"].replace(".", "")}' - else: - cuda_suffix = "-gpu" - - system_str = "" - if "system_packages" in build and not build["system_packages"] is None: - for p in build["system_packages"]: - system_str += p + " " - - packages_str = "" - if "python_packages" in build and not build["python_packages"] is None: - for p in build["python_packages"]: - packages_str += p + " " - for p in DEFAULT_DEPENDENCIES: - packages_str += p + " " - packages_str += f"instill-sdk=={instill_version}" - - with tempfile.TemporaryDirectory() as tmpdir: - shutil.copyfile( - __file__.replace("build.py", "Dockerfile"), f"{tmpdir}/Dockerfile" - ) - shutil.copyfile( - __file__.replace("build.py", ".dockerignore"), f"{tmpdir}/.dockerignore" - ) - shutil.copytree(os.getcwd(), tmpdir, dirs_exist_ok=True) - - target_arch_suffix = "-aarch64" if args.target_arch == "arm64" else "" - - Logger.i("[Instill Builder] Building model image...") - command = [ - "docker", - "buildx", - "build", - "--build-arg", - f"TARGET_ARCH_SUFFIX={target_arch_suffix}", - "--build-arg", - f"RAY_VERSION={ray_version}", - "--build-arg", - f"PYTHON_VERSION={python_version}", - "--build-arg", - f"CUDA_SUFFIX={cuda_suffix}", - "--build-arg", - f"PACKAGES={packages_str}", - "--build-arg", - f"SYSTEM_PACKAGES={system_str}", - "--platform", - f"linux/{args.target_arch}", - "-t", - f"{repo}:{args.tag}", - tmpdir, - "--load", - ] - if args.no_cache: - command.append("--no-cache") - subprocess.run( - command, - check=True, - ) - Logger.i(f"[Instill Builder] {repo}:{args.tag} built") - except subprocess.CalledProcessError: - Logger.e("[Instill Builder] Build failed") - except Exception as e: - Logger.e("[Instill Builder] Prepare failed") - Logger.e(e) - finally: - Logger.i("[Instill Builder] Done") - - -if __name__ == "__main__": - build_image() diff --git a/instill/helpers/cli.py b/instill/helpers/cli.py new file mode 100644 index 0000000..46d2c5f --- /dev/null +++ b/instill/helpers/cli.py @@ -0,0 +1,229 @@ +import argparse + +# import hashlib +import os +import platform +import shutil +import subprocess +import tempfile + +import ray +import yaml + +import instill +from instill.helpers.const import DEFAULT_DEPENDENCIES +from instill.helpers.errors import ModelConfigException +from instill.utils.logger import Logger + + +def config_check_required_fields(c): + if "build" not in c or c["build"] is None: + raise ModelConfigException("build") + if "gpu" not in c["build"] or c["build"]["gpu"] is None: + raise ModelConfigException("gpu") + if "python_version" not in c["build"] or c["build"]["python_version"] is None: + raise ModelConfigException("python_version") + + +def cli(): + if platform.machine() in ("i386", "AMD64", "x86_64"): + default_platform = "amd64" + else: + default_platform = platform.machine() + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(required=True) + + # init + init_parser = subcommands.add_parser("init", help="Initialize model directory") + init_parser.set_defaults(func=init) + + # build + build_parser = subcommands.add_parser("build", help="Build model image") + build_parser.set_defaults(func=build) + build_parser.add_argument( + "name", + help="user and model namespace, in the format of /", + ) + build_parser.add_argument( + "-t", + "--tag", + help="tag for the model image, default to `latest`", + # default=hashlib.sha256().hexdigest(), + default="latest", + required=False, + ) + build_parser.add_argument( + "--no-cache", + help="build the image without cache", + action="store_true", + required=False, + ) + build_parser.add_argument( + "--target-arch", + help="target platform architecture for the model image, default to host", + default=default_platform, + choices=["arm64", "amd64"], + required=False, + ) + + # push + push_parser = subcommands.add_parser("push", help="Push model image") + push_parser.set_defaults(func=push) + push_parser.add_argument( + "name", + help="user and model namespace, in the format of /", + ) + push_parser.add_argument( + "-u", + "--url", + help="image registry url, in the format of host:port, default to api.instill.tech", + default="api.instill.tech", + required=False, + ) + push_parser.add_argument( + "-t", + "--tag", + help="tag for the model image, default to `latest`", + default="latest", + required=False, + ) + + args = parser.parse_args() + args.func(args) + + +def init(_): + shutil.copyfile( + __file__.replace("cli.py", "init-templates/instill.yaml"), + f"{os.getcwd()}/instill.yaml", + ) + shutil.copyfile( + __file__.replace("cli.py", "init-templates/model.py"), + f"{os.getcwd()}/model.py", + ) + shutil.copyfile( + __file__.replace("cli.py", "init-templates/.dockerignore"), + f"{os.getcwd()}/.dockerignore", + ) + +def build(args): + try: + Logger.i("[Instill Builder] Loading config file...") + with open("instill.yaml", "r", encoding="utf8") as f: + Logger.i("[Instill Builder] Parsing config file...") + config = yaml.safe_load(f) + + config_check_required_fields(config) + + build_params = config["build"] + + python_version = build_params["python_version"].replace(".", "") + ray_version = ray.__version__ + instill_version = instill.__version__ + + if not build_params["gpu"]: + cuda_suffix = "" + elif ( + "cuda_version" in build_params and not build_params["cuda_version"] is None + ): + cuda_suffix = f'-cu{build_params["cuda_version"].replace(".", "")}' + else: + cuda_suffix = "-gpu" + + system_str = "" + if ( + "system_packages" in build_params + and not build_params["system_packages"] is None + ): + for p in build_params["system_packages"]: + system_str += p + " " + + packages_str = "" + if ( + "python_packages" in build_params + and not build_params["python_packages"] is None + ): + for p in build_params["python_packages"]: + packages_str += p + " " + for p in DEFAULT_DEPENDENCIES: + packages_str += p + " " + packages_str += f"instill-sdk=={instill_version}" + + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copyfile( + __file__.replace("cli.py", "init-templates/Dockerfile"), + f"{tmpdir}/Dockerfile", + ) + shutil.copytree(os.getcwd(), tmpdir, dirs_exist_ok=True) + + target_arch_suffix = "-aarch64" if args.target_arch == "arm64" else "" + + Logger.i("[Instill Builder] Building model image...") + command = [ + "docker", + "buildx", + "build", + "--build-arg", + f"TARGET_ARCH_SUFFIX={target_arch_suffix}", + "--build-arg", + f"RAY_VERSION={ray_version}", + "--build-arg", + f"PYTHON_VERSION={python_version}", + "--build-arg", + f"CUDA_SUFFIX={cuda_suffix}", + "--build-arg", + f"PACKAGES={packages_str}", + "--build-arg", + f"SYSTEM_PACKAGES={system_str}", + "--platform", + f"linux/{args.target_arch}", + "-t", + f"{args.name}:{args.tag}", + tmpdir, + "--load", + ] + if args.no_cache: + command.append("--no-cache") + subprocess.run( + command, + check=True, + ) + Logger.i(f"[Instill Builder] {args.name}:{args.tag} built") + except subprocess.CalledProcessError: + Logger.e("[Instill Builder] Build failed") + except Exception as e: + Logger.e("[Instill Builder] Prepare failed") + Logger.e(e) + finally: + Logger.i("[Instill Builder] Done") + + +def push(args): + try: + registry = args.url + + subprocess.run( + [ + "docker", + "tag", + f"{args.name}:{args.tag}", + f"{registry}/{args.name}:{args.tag}", + ], + check=True, + ) + Logger.i("[Instill Builder] Pushing model image...") + subprocess.run( + ["docker", "push", f"{registry}/{args.name}:{args.tag}"], check=True + ) + Logger.i(f"[Instill Builder] {registry}/{args.name}:{args.tag} pushed") + except subprocess.CalledProcessError: + Logger.e("[Instill Builder] Push failed") + except Exception as e: + Logger.e("[Instill Builder] Prepare failed") + Logger.e(e) + finally: + Logger.i("[Instill Builder] Done") + + +if __name__ == "__main__": + cli() diff --git a/instill/helpers/.dockerignore b/instill/helpers/init-templates/.dockerignore similarity index 100% rename from instill/helpers/.dockerignore rename to instill/helpers/init-templates/.dockerignore diff --git a/instill/helpers/Dockerfile b/instill/helpers/init-templates/Dockerfile similarity index 100% rename from instill/helpers/Dockerfile rename to instill/helpers/init-templates/Dockerfile diff --git a/instill/helpers/init-templates/instill.yaml b/instill/helpers/init-templates/instill.yaml new file mode 100644 index 0000000..e53974a --- /dev/null +++ b/instill/helpers/init-templates/instill.yaml @@ -0,0 +1,20 @@ +build: + # set to true if your model requires GPU + gpu: true + + # python version, currently only support 3.11 + python_version: "3.11" + + # cuda version if `gpu` is set to true + # support 11.5, 11.6 ,11.7 ,11.7 ,12.1 + cuda_version: "12.1" + + # a list of python packages in the format of {package-name}=={version} + # python_packages: + # - torch==2.3.1 + # - transformers==4.41.2 + + # a list of system packages from apt package manager + # system_packages: + # - htop + # - libglib2.0-0 diff --git a/instill/helpers/init-templates/model.py b/instill/helpers/init-templates/model.py new file mode 100644 index 0000000..f7be702 --- /dev/null +++ b/instill/helpers/init-templates/model.py @@ -0,0 +1,78 @@ +from instill.helpers.const import TextGenerationChatInput +from instill.helpers.ray_io import StandardTaskIO +from instill.helpers.ray_config import instill_deployment, InstillDeployable +from instill.helpers import ( + construct_text_generation_chat_infer_response, + construct_text_generation_chat_metadata_response, +) + + +@instill_deployment +class Phimini: + + def __init__(self): + """Load model into memory""" + # model = AutoModelForCausalLM.from_pretrained( + # "microsoft/Phi-3-mini-4k-instruct", + # device_map="cuda", + # torch_dtype="auto", + # trust_remote_code=True, + # ) + # tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct") + + # self.pipeline = pipeline( + # "text-generation", + # model=model, + # tokenizer=tokenizer, + # ) + + def ModelMetadata(self, req): + """Define model input and output shape base on task type""" + # return construct_text_generation_chat_metadata_response(req=req) + + async def __call__(self, request): + """Run inference logic""" + + # deserialize input base on task type + # task_text_generation_chat_input: TextGenerationChatInput = ( + # StandardTaskIO.parse_task_text_generation_chat_input(request=request) + # ) + + # preprocess + # conv = [ + # { + # "role": "system", + # "content": task_text_generation_chat_input.system_message, + # }, + # { + # "role": "user", + # "content": task_text_generation_chat_input.prompt, + # }, + # ] + # generation_args = { + # "max_new_tokens": task_text_generation_chat_input.max_new_tokens, + # "return_full_text": False, + # "temperature": task_text_generation_chat_input.temperature, + # "top_k": task_text_generation_chat_input.top_k, + # "top_p": 0.95, + # "do_sample": False, + # } + + # inference + # sequences = self.pipeline(conv, **generation_args) + + # convert the model output into response output using StandardTaskIO + # task_text_generation_chat_output = ( + # StandardTaskIO.parse_task_text_generation_chat_output(sequences=sequences) + # ) + + # return response + # return construct_text_generation_chat_infer_response( + # req=request, + # # specify the output dimension + # shape=[1, len(sequences)], + # raw_outputs=[task_text_generation_chat_output], + # ) + +# define model deployment entrypoint +entrypoint = InstillDeployable(Phimini).get_deployment_handle() diff --git a/instill/helpers/push.py b/instill/helpers/push.py deleted file mode 100644 index 4cc0166..0000000 --- a/instill/helpers/push.py +++ /dev/null @@ -1,53 +0,0 @@ -import argparse -import subprocess - -import yaml - -from instill.utils.logger import Logger - - -def push_image(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-u", - "--url", - help="image registry url, in the format of host:port, default to docker.io", - default="docker.io", - required=False, - ) - parser.add_argument( - "-t", - "--tag", - help="tag for the model image", - required=True, - ) - - try: - args = parser.parse_args() - - Logger.i("[Instill Builder] Loading config file...") - with open("instill.yaml", "r", encoding="utf8") as f: - Logger.i("[Instill Builder] Parsing config file...") - config = yaml.safe_load(f) - - registry = args.url - repo = config["repo"] - - subprocess.run( - ["docker", "tag", f"{repo}:{args.tag}", f"{registry}/{repo}:{args.tag}"], - check=True, - ) - Logger.i("[Instill Builder] Pushing model image...") - subprocess.run(["docker", "push", f"{registry}/{repo}:{args.tag}"], check=True) - Logger.i(f"[Instill Builder] {registry}/{repo}:{args.tag} pushed") - except subprocess.CalledProcessError: - Logger.e("[Instill Builder] Push failed") - except Exception as e: - Logger.e("[Instill Builder] Prepare failed") - Logger.e(e) - finally: - Logger.i("[Instill Builder] Done") - - -if __name__ == "__main__": - push_image() diff --git a/poetry.lock b/poetry.lock index 6adb484..cfd1077 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -211,8 +211,8 @@ files = [ lazy-object-proxy = ">=1.4.0" typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} wrapt = [ - {version = ">=1.11,<2", markers = "python_version < \"3.11\""}, {version = ">=1.14,<2", markers = "python_version >= \"3.11\""}, + {version = ">=1.11,<2", markers = "python_version < \"3.11\""}, ] [[package]] @@ -720,10 +720,10 @@ isort = ">=4.3.21,<6.0" jinja2 = ">=2.10.1,<4.0" packaging = "*" pydantic = [ - {version = ">=1.5.1,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version < \"3.10\""}, - {version = ">=1.9.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, {version = ">=1.10.0,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.12\" and python_version < \"4.0\""}, {version = ">=1.10.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, + {version = ">=1.5.1,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version < \"3.10\""}, + {version = ">=1.9.0,<2.4.0 || >2.4.0,<3.0", extras = ["email"], markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] pyyaml = ">=6.0.1" toml = {version = ">=0.10.0,<1.0.0", markers = "python_version < \"3.11\""} @@ -2962,8 +2962,8 @@ files = [ astroid = ">=2.12.13,<=2.14.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ - {version = ">=0.2", markers = "python_version < \"3.11\""}, {version = ">=0.3.6", markers = "python_version >= \"3.11\""}, + {version = ">=0.2", markers = "python_version < \"3.11\""}, ] isort = ">=4.2.5,<6" mccabe = ">=0.6,<0.8" @@ -3168,7 +3168,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -3350,8 +3349,8 @@ fastapi = {version = "*", optional = true, markers = "extra == \"serve\""} filelock = "*" frozenlist = "*" grpcio = [ - {version = ">=1.32.0", optional = true, markers = "python_version < \"3.10\" and extra == \"serve\""}, {version = ">=1.42.0", optional = true, markers = "python_version >= \"3.10\" and extra == \"serve\""}, + {version = ">=1.32.0", optional = true, markers = "python_version < \"3.10\" and extra == \"serve\""}, ] jsonschema = "*" memray = {version = "*", optional = true, markers = "sys_platform != \"win32\" and extra == \"serve\""} diff --git a/pyproject.toml b/pyproject.toml index cd8b42c..ff24483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,8 +86,7 @@ jsonref = "^1.1.0" twine = "^4.0.2" [tool.poetry.scripts] -instill-build = "instill.helpers.build:build_image" -instill-push = "instill.helpers.push:push_image" +instill = "instill.helpers.cli:cli" [tool.black] From cf3913ef48d92bb63f45a2c789320f25d801c572 Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Sun, 30 Jun 2024 03:18:28 +0800 Subject: [PATCH 2/2] chore: fix format --- instill/helpers/cli.py | 1 + instill/helpers/init-templates/model.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/instill/helpers/cli.py b/instill/helpers/cli.py index 46d2c5f..fa0a101 100644 --- a/instill/helpers/cli.py +++ b/instill/helpers/cli.py @@ -106,6 +106,7 @@ def init(_): f"{os.getcwd()}/.dockerignore", ) + def build(args): try: Logger.i("[Instill Builder] Loading config file...") diff --git a/instill/helpers/init-templates/model.py b/instill/helpers/init-templates/model.py index f7be702..ecb25c6 100644 --- a/instill/helpers/init-templates/model.py +++ b/instill/helpers/init-templates/model.py @@ -1,14 +1,16 @@ -from instill.helpers.const import TextGenerationChatInput -from instill.helpers.ray_io import StandardTaskIO -from instill.helpers.ray_config import instill_deployment, InstillDeployable +# type: ignore from instill.helpers import ( construct_text_generation_chat_infer_response, construct_text_generation_chat_metadata_response, ) +from instill.helpers.const import TextGenerationChatInput +from instill.helpers.ray_config import InstillDeployable, instill_deployment +from instill.helpers.ray_io import StandardTaskIO @instill_deployment class Phimini: + """Custom model implementation""" def __init__(self): """Load model into memory""" @@ -74,5 +76,6 @@ async def __call__(self, request): # raw_outputs=[task_text_generation_chat_output], # ) + # define model deployment entrypoint entrypoint = InstillDeployable(Phimini).get_deployment_handle()