Skip to content
This repository was archived by the owner on Sep 10, 2025. It is now read-only.

Commit ce94bfb

Browse files
Jack-Khuufduwjj
Jack-Khuu
authored and
fduwjj
committed
Fix help generate (#891)
* Fixing the help mode of the download subcommand * Initial Addition of subparsers for generation * Move compile out of generation exclusive * typo * Fix test by removing temperature, which is a field eval doesn't use or expect * Typo Generater => Generator
1 parent d2f40df commit ce94bfb

File tree

2 files changed

+104
-81
lines changed

2 files changed

+104
-81
lines changed

‎.ci/scripts/validate.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ function eval_model_sanity_check() {
284284
echo "*************************************************"
285285
if [ "$DTYPE" != "float16" ]; then
286286
python3 -W ignore export.py --dtype ${DTYPE} --quant "$QUANT_OPTIONS" --checkpoint-path "$CHECKPOINT_PATH" --output-dso-path ${MODEL_DIR}/${MODEL_NAME}.so --device "$TARGET_DEVICE" || exit 1
287-
python3 -W ignore eval.py --dtype ${DTYPE} --checkpoint-path "$CHECKPOINT_PATH" --temperature 0 --dso-path ${MODEL_DIR}/${MODEL_NAME}.so --device "$TARGET_DEVICE" --limit 5 > "$MODEL_DIR/output_eval_aoti" || exit 1
287+
python3 -W ignore eval.py --dtype ${DTYPE} --checkpoint-path "$CHECKPOINT_PATH" --dso-path ${MODEL_DIR}/${MODEL_NAME}.so --device "$TARGET_DEVICE" --limit 5 > "$MODEL_DIR/output_eval_aoti" || exit 1
288288
cat "$MODEL_DIR/output_eval_aoti"
289289
fi;
290290

‎cli.py‎

Lines changed: 103 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -61,27 +61,14 @@ def add_arguments_for_verb(parser, verb: str) -> None:
6161
help="Model name for well-known models",
6262
)
6363

64-
parser.add_argument(
65-
"--chat",
66-
action="store_true",
67-
help="Whether to start an interactive chat session",
68-
)
64+
if verb in ["browser", "chat", "generate"]:
65+
_add_generation_args(parser)
66+
6967
parser.add_argument(
7068
"--distributed",
7169
action="store_true",
7270
help="Whether to enable distributed inference",
7371
)
74-
parser.add_argument(
75-
"--gui",
76-
action="store_true",
77-
help="Whether to use a web UI for an interactive chat session",
78-
)
79-
parser.add_argument(
80-
"--prompt",
81-
type=str,
82-
default="Hello, my name is",
83-
help="Input prompt",
84-
)
8572
parser.add_argument(
8673
"--is-chat-model",
8774
action="store_true",
@@ -93,54 +80,17 @@ def add_arguments_for_verb(parser, verb: str) -> None:
9380
default=None,
9481
help="Initialize torch seed",
9582
)
96-
parser.add_argument(
97-
"--num-samples",
98-
type=int,
99-
default=1,
100-
help="Number of samples",
101-
)
102-
parser.add_argument(
103-
"--max-new-tokens",
104-
type=int,
105-
default=200,
106-
help="Maximum number of new tokens",
107-
)
108-
parser.add_argument(
109-
"--top-k",
110-
type=int,
111-
default=200,
112-
help="Top-k for sampling",
113-
)
114-
parser.add_argument(
115-
"--temperature", type=float, default=0.8, help="Temperature for sampling"
116-
)
11783
parser.add_argument(
11884
"--compile",
11985
action="store_true",
12086
help="Whether to compile the model with torch.compile",
12187
)
122-
parser.add_argument(
123-
"--compile-prefill",
124-
action="store_true",
125-
help="Whether to compile the prefill. Improves prefill perf, but has higher compile times.",
126-
)
127-
parser.add_argument(
128-
"--sequential-prefill",
129-
action="store_true",
130-
help="Whether to perform prefill sequentially. Only used for model debug.",
131-
)
13288
parser.add_argument(
13389
"--profile",
13490
type=Path,
13591
default=None,
13692
help="Profile path.",
13793
)
138-
parser.add_argument(
139-
"--speculate-k",
140-
type=int,
141-
default=5,
142-
help="Speculative execution depth",
143-
)
14494
parser.add_argument(
14595
"--draft-checkpoint-path",
14696
type=Path,
@@ -177,30 +127,10 @@ def add_arguments_for_verb(parser, verb: str) -> None:
177127
default=None,
178128
help="Use the specified model tokenizer file",
179129
)
180-
parser.add_argument(
181-
"--output-pte-path",
182-
type=str,
183-
default=None,
184-
help="Output to the specified ExecuTorch .pte model file",
185-
)
186-
parser.add_argument(
187-
"--output-dso-path",
188-
type=str,
189-
default=None,
190-
help="Output to the specified AOT Inductor .dso model file",
191-
)
192-
parser.add_argument(
193-
"--dso-path",
194-
type=Path,
195-
default=None,
196-
help="Use the specified AOT Inductor .dso model file",
197-
)
198-
parser.add_argument(
199-
"--pte-path",
200-
type=Path,
201-
default=None,
202-
help="Use the specified ExecuTorch .pte model file",
203-
)
130+
131+
_add_exported_model_input_args(parser)
132+
_add_export_output_path_args(parser)
133+
204134
parser.add_argument(
205135
"--dtype",
206136
default="fast",
@@ -265,6 +195,40 @@ def add_arguments_for_verb(parser, verb: str) -> None:
265195
_add_cli_metadata_args(parser)
266196

267197

198+
# Add CLI Args representing user provided exported model files
199+
def _add_export_output_path_args(parser) -> None:
200+
output_path_parser = parser.add_argument_group("Export Output Path Args", "Specify the output path for the exported model files")
201+
output_path_parser.add_argument(
202+
"--output-pte-path",
203+
type=str,
204+
default=None,
205+
help="Output to the specified ExecuTorch .pte model file",
206+
)
207+
output_path_parser.add_argument(
208+
"--output-dso-path",
209+
type=str,
210+
default=None,
211+
help="Output to the specified AOT Inductor .dso model file",
212+
)
213+
214+
215+
# Add CLI Args representing user provided exported model files
216+
def _add_exported_model_input_args(parser) -> None:
217+
exported_model_path_parser = parser.add_argument_group("Exported Model Path Args", "Specify the path of the exported model files to ingest")
218+
exported_model_path_parser.add_argument(
219+
"--dso-path",
220+
type=Path,
221+
default=None,
222+
help="Use the specified AOT Inductor .dso model file",
223+
)
224+
exported_model_path_parser.add_argument(
225+
"--pte-path",
226+
type=Path,
227+
default=None,
228+
help="Use the specified ExecuTorch .pte model file",
229+
)
230+
231+
268232
# Add CLI Args that are relevant to any subcommand execution
269233
def _add_cli_metadata_args(parser) -> None:
270234
parser.add_argument(
@@ -303,22 +267,81 @@ def _configure_artifact_inventory_args(parser, verb: str) -> None:
303267
)
304268

305269

270+
# Add CLI Args specific to user prompted generation
271+
def _add_generation_args(parser) -> None:
272+
generator_parser = parser.add_argument_group("Generation Args", "Configs for generating output based on provided prompt")
273+
generator_parser.add_argument(
274+
"--prompt",
275+
type=str,
276+
default="Hello, my name is",
277+
help="Input prompt for manual output generation",
278+
)
279+
generator_parser.add_argument(
280+
"--chat",
281+
action="store_true",
282+
help="Whether to start an interactive chat session",
283+
)
284+
generator_parser.add_argument(
285+
"--gui",
286+
action="store_true",
287+
help="Whether to use a web UI for an interactive chat session",
288+
)
289+
generator_parser.add_argument(
290+
"--num-samples",
291+
type=int,
292+
default=1,
293+
help="Number of samples",
294+
)
295+
generator_parser.add_argument(
296+
"--max-new-tokens",
297+
type=int,
298+
default=200,
299+
help="Maximum number of new tokens",
300+
)
301+
generator_parser.add_argument(
302+
"--top-k",
303+
type=int,
304+
default=200,
305+
help="Top-k for sampling",
306+
)
307+
generator_parser.add_argument(
308+
"--temperature", type=float, default=0.8, help="Temperature for sampling"
309+
)
310+
generator_parser.add_argument(
311+
"--compile-prefill",
312+
action="store_true",
313+
help="Whether to compile the prefill. Improves prefill perf, but has higher compile times.",
314+
)
315+
generator_parser.add_argument(
316+
"--sequential-prefill",
317+
action="store_true",
318+
help="Whether to perform prefill sequentially. Only used for model debug.",
319+
)
320+
generator_parser.add_argument(
321+
"--speculate-k",
322+
type=int,
323+
default=5,
324+
help="Speculative execution depth",
325+
)
326+
327+
306328
# Add CLI Args specific to Model Evaluation
307329
def _add_evaluation_args(parser) -> None:
308-
parser.add_argument(
330+
eval_parser = parser.add_argument_group("Evaluation Args", "Configs for evaluating model performance")
331+
eval_parser.add_argument(
309332
"--tasks",
310333
nargs="+",
311334
type=str,
312335
default=["wikitext"],
313336
help="List of lm-eluther tasks to evaluate. Usage: --tasks task1 task2",
314337
)
315-
parser.add_argument(
338+
eval_parser.add_argument(
316339
"--limit",
317340
type=int,
318341
default=None,
319342
help="Number of samples to evaluate",
320343
)
321-
parser.add_argument(
344+
eval_parser.add_argument(
322345
"--max-seq-length",
323346
type=int,
324347
default=None,

0 commit comments

Comments
 (0)