-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into adithyare/unfused_lora
- Loading branch information
Showing
38 changed files
with
2,782 additions
and
268 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
46 changes: 46 additions & 0 deletions
46
examples/nlp/language_modeling/conf/megatron_retro_inference_legacy.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
# (This inferencing script for native NeMo RETRO will be soon deprecated. For new inferencing script for mcore RETRO, see ./megatron_retro_inference.yaml) | ||
|
||
inference: | ||
greedy: False # Whether or not to use sampling ; use greedy decoding otherwise | ||
top_k: 0 # The number of highest probability vocabulary tokens to keep for top-k-filtering. | ||
top_p: 0.9 # If set to float < 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation. | ||
temperature: 1.0 # sampling temperature | ||
add_BOS: True # add the bos token at the begining of the prompt | ||
tokens_to_generate: 30 # The minimum length of the sequence to be generated. | ||
all_probs: False # whether return the log prob for all the tokens in vocab | ||
repetition_penalty: 1.2 # The parameter for repetition penalty. 1.0 means no penalty. | ||
min_tokens_to_generate: 0 # The minimum length of the sequence to be generated. | ||
compute_logprob: False # a flag used to compute logprob of all the input text, a very special case of running inference, default False | ||
|
||
|
||
trainer: | ||
devices: 1 | ||
num_nodes: 1 | ||
accelerator: gpu | ||
logger: False # logger provided by exp_manager | ||
precision: 16 # 16, 32, or bf16 | ||
|
||
inference_batch_size: 2 | ||
tensor_model_parallel_size: -1 | ||
pipeline_model_parallel_size: -1 | ||
pipeline_model_parallel_split_rank: -1 # used for encoder and decoder model (0 for others) | ||
retro_model_file: null # RETRO nemo file path | ||
|
||
use_predict_method: False # whether to use the predict method | ||
|
||
prompts: # prompts for RETRO model inference | ||
- "hello," | ||
- "good morning," | ||
- "good afternoon," | ||
- "good evening," | ||
|
||
########### Faiss service parameters ######## | ||
retrieval_service: | ||
strategy: RetroModelTextGenerationStrategy # choose customized inference strategy | ||
neighbors: 4 | ||
frequent_query: False # for the current token generation, frequently update the retrieval context. If false, update it every 64 tokens | ||
pad_tokens: True # pad the tokens at the beginning to make it minimum of 64 tokens for retrieving at least once | ||
store_retrieved: False # whether store the retrieved documents, so it can be checked | ||
combo_service: | ||
service_ip: '0.0.0.0' | ||
service_port: 17181 |
40 changes: 40 additions & 0 deletions
40
examples/nlp/language_modeling/conf/megatron_retro_qatask.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
inference: | ||
greedy: False # Whether or not to use sampling ; use greedy decoding otherwise | ||
top_k: 0 # The number of highest probability vocabulary tokens to keep for top-k-filtering. | ||
top_p: 0.9 # If set to float < 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation. | ||
temperature: 1.0 # sampling temperature | ||
add_BOS: False # add the bos token at the begining of the prompt | ||
tokens_to_generate: 30 # The minimum length of the sequence to be generated. | ||
all_probs: False # whether return the log prob for all the tokens in vocab | ||
repetition_penalty: 1.2 # The parameter for repetition penalty. 1.0 means no penalty. | ||
min_tokens_to_generate: 0 # The minimum length of the sequence to be generated. | ||
compute_logprob: False # a flag used to compute logprob of all the input text, a very special case of running inference, default False | ||
end_strings: ["<|endoftext|>"] # generation will stop when one of these tokens is generated | ||
# RETRO-specific arguments | ||
retro_inference: | ||
retro_gpt_retrieved_length: 128 | ||
retro_num_neighbors: 2 | ||
ft_neighbours: 0 | ||
reuse_top: False | ||
|
||
trainer: | ||
devices: 1 | ||
num_nodes: 1 | ||
accelerator: gpu | ||
logger: False # logger provided by exp_manager | ||
precision: 32 # 16, 32, or bf16 | ||
use_distributed_sampler: False | ||
|
||
tensor_model_parallel_size: -1 | ||
pipeline_model_parallel_size: -1 | ||
pipeline_model_parallel_split_rank: -1 # used for encoder and decoder model (0 for others) | ||
megatron_amp_O2: False # Enable O2-level automatic mixed precision to save memory | ||
|
||
retro_model_file: null # Retro nemo file path | ||
checkpoint_dir: null # checkpoint file dir. This is used to load the PTL checkpoint generated during the Retro training | ||
checkpoint_name: null # PTL checkpoint file name, only used for PTL checkpoint loading | ||
hparams_file: null # model configuration file, only used for PTL checkpoint loading | ||
|
||
# qa tasks | ||
qa_file_path: null | ||
pred_file_path: null |
Oops, something went wrong.