Skip to content

all the llamas in canary #1803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

all the llamas in canary #1803

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
167962e
all the llamas
msaroufim Aug 2, 2023
ed70153
add fsdp comment
msaroufim Aug 2, 2023
5b8d4da
bla
msaroufim Aug 2, 2023
65f6515
remove imagebind
msaroufim Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions torchbenchmark/canary_models/llama_v2_13b/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from torchbenchmark.tasks import NLP
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just curious, these models are in canary_models as opposed to models dir. Does this effect how these models are pulled in either from torchbench or dyanmorunner and if so in what way?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This error message explains it super well, should be no difference

  File "/var/lib/jenkins/workspace/benchmarks/dynamo/torchbench.py", line 302, in load_model
    raise ImportError(f"could not import any of {candidates}")
ImportError: could not import any of ['torchbenchmark.models.stable_diffusion', 'torchbenchmark.canary_models.stable_diffusion', 'torchbenchmark.models.fb.stable_diffusion']
ERROR

from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel, HuggingFaceAuthMixin

class Model(HuggingFaceModel, HuggingFaceAuthMixin):
task = NLP.LANGUAGE_MODELING
DEFAULT_TRAIN_BSIZE = 1
DEFAULT_EVAL_BSIZE = 1
DEEPCOPY = False

def __init__(self, test, device, batch_size=None, extra_args=[]):
HuggingFaceAuthMixin.__init__(self)
super().__init__(name="llama_v2_13b", test=test, device=device, batch_size=batch_size, extra_args=extra_args)

def train(self):
return NotImplementedError("FSDP should implement a training loop")
9 changes: 9 additions & 0 deletions torchbenchmark/canary_models/llama_v2_13b/install.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@

import subprocess
import sys
import os
from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model

if __name__ == '__main__':
model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
cache_model(model_name)
12 changes: 12 additions & 0 deletions torchbenchmark/canary_models/llama_v2_13b/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
devices:
NVIDIA A100-SXM4-40GB:
eval_batch_size: 1
eval_benchmark: false
eval_deterministic: false
eval_nograd: true
not_implemented:
- device: cpu
- device: cuda
test: train
train_benchmark: false
train_deterministic: false
17 changes: 17 additions & 0 deletions torchbenchmark/canary_models/llama_v2_70b/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from torchbenchmark.tasks import NLP
from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel, HuggingFaceAuthMixin

class Model(HuggingFaceModel, HuggingFaceAuthMixin):
task = NLP.LANGUAGE_MODELING
DEFAULT_TRAIN_BSIZE = 1
DEFAULT_EVAL_BSIZE = 1
DEEPCOPY = False

def __init__(self, test, device, batch_size=None, extra_args=[]):
HuggingFaceAuthMixin.__init__(self)
super().__init__(name="llama_v2_70b", test=test, device=device, batch_size=batch_size, extra_args=extra_args)


def train(self):
return NotImplementedError("FSDP should implement a training loop")

9 changes: 9 additions & 0 deletions torchbenchmark/canary_models/llama_v2_70b/install.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@

import subprocess
import sys
import os
from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model

if __name__ == '__main__':
model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
cache_model(model_name)
12 changes: 12 additions & 0 deletions torchbenchmark/canary_models/llama_v2_70b/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
devices:
NVIDIA A100-SXM4-40GB:
eval_batch_size: 1
eval_benchmark: false
eval_deterministic: false
eval_nograd: true
not_implemented:
- device: cpu
- device: cuda
test: train
train_benchmark: false
train_deterministic: false
17 changes: 17 additions & 0 deletions torchbenchmark/canary_models/llama_v2_7b/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from torchbenchmark.tasks import NLP
from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel, HuggingFaceAuthMixin

class Model(HuggingFaceModel, HuggingFaceAuthMixin):
task = NLP.LANGUAGE_MODELING
DEFAULT_TRAIN_BSIZE = 1
DEFAULT_EVAL_BSIZE = 1
DEEPCOPY = False

def __init__(self, test, device, batch_size=None, extra_args=[]):
HuggingFaceAuthMixin.__init__(self)
super().__init__(name="llama_v2_7b", test=test, device=device, batch_size=batch_size, extra_args=extra_args)


def train(self):
return NotImplementedError("FSDP should implement a training loop")

9 changes: 9 additions & 0 deletions torchbenchmark/canary_models/llama_v2_7b/install.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@

import subprocess
import sys
import os
from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model

if __name__ == '__main__':
model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
cache_model(model_name)
12 changes: 12 additions & 0 deletions torchbenchmark/canary_models/llama_v2_7b/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
devices:
NVIDIA A100-SXM4-40GB:
eval_batch_size: 1
eval_benchmark: false
eval_deterministic: false
eval_nograd: true
not_implemented:
- device: cpu
- device: cuda
test: train
train_benchmark: false
train_deterministic: false
3 changes: 3 additions & 0 deletions torchbenchmark/util/framework/huggingface/model_factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
'hf_Whisper': (1024, 1024, 'WhisperConfig()', 'AutoModelForAudioClassification'),
# default num_hidden_layers=32 but that OOMs, feel free to change this config to something more real
'llama_v2_7b_16h' : (512,512, 'LlamaConfig(num_hidden_layers=16)', 'AutoModelForCausalLM'),
'llama_v2_7b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-7b-hf")', 'AutoModelForCausalLM'),
'llama_v2_13b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-13b-hf")', 'AutoModelForCausalLM'),
'llama_v2_70b' : (512, 512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-70b-hf")', 'AutoModelForMaskedLM'),
}

cpu_input_slice = {
Expand Down