Skip to content

remove SkipInputShape flag #29615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

remove SkipInputShape flag #29615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b60b51e
change softmax_test input shape
mingzhe09088 Nov 12, 2019
503d86c
remove SkipInputShape flag
mingzhe09088 Nov 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions benchmarks/operator_benchmark/benchmark_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,7 @@ def _build_test(configs, bench_op, OperatorTestCase, run_backward, op_name_funct
op.set_module_name(op_name)

op._set_backward_test(run_backward)
try:
op.init(**init_dict)
except SkipInputShape:
print("Skipping: Config<{}> is not valid for op<{}>".format(input_config, op.module_name()))
continue
op.init(**init_dict)

input_name = None

Expand Down
123 changes: 59 additions & 64 deletions benchmarks/operator_benchmark/benchmark_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,29 +33,29 @@ def str2bool(v):

def numpy_random(dtype, *shapes):
""" Return a random numpy tensor of the provided dtype.
Args:
Args:
shapes: int or a sequence of ints to defining the shapes of the tensor
dtype: use the dtypes from numpy
dtype: use the dtypes from numpy
(https://docs.scipy.org/doc/numpy/user/basics.types.html)
Return:
numpy tensor of dtype
Return:
numpy tensor of dtype
"""
# TODO: consider more complex/custom dynamic ranges for
# comprehensive test coverage.
return np.random.rand(*shapes).astype(dtype)


def set_omp_threads(num_threads):
def set_omp_threads(num_threads):
existing_value = os.environ.get('OMP_NUM_THREADS', '')
if existing_value != '':
if existing_value != '':
print("Overwriting existing OMP_NUM_THREADS value: {}; Setting it to {}.".format(
existing_value, num_threads))
os.environ["OMP_NUM_THREADS"] = str(num_threads)


def set_mkl_threads(num_threads):
existing_value = os.environ.get('MKL_NUM_THREADS', '')
if existing_value != '':
if existing_value != '':
print("Overwriting existing MKL_NUM_THREADS value: {}; Setting it to {}.".format(
existing_value, num_threads))
os.environ["MKL_NUM_THREADS"] = str(num_threads)
Expand Down Expand Up @@ -113,7 +113,7 @@ def cross_product_configs(**configs):
tmp_results = [{key : value} for value in values]
configs_attrs_list.append(tmp_results)

# TODO(mingzhe0908) remove the conversion to list.
# TODO(mingzhe0908) remove the conversion to list.
# itertools.product produces an iterator that produces element on the fly
# while converting to a list produces everything at the same time.
generated_configs = list(itertools.product(*configs_attrs_list))
Expand All @@ -123,17 +123,17 @@ def cross_product_configs(**configs):
def config_list(**configs):
""" Generate configs based on the list of input shapes.
This function will take input shapes specified in a list from user. Besides
that, all other parameters will be cross producted first and each of the
generated list will be merged with the input shapes list.
that, all other parameters will be cross producted first and each of the
generated list will be merged with the input shapes list.

Reserved Args:
attr_names(reserved): a list of names for input shapes.
attrs(reserved): a list of values for each input shape.
corss_product: a dictionary of attributes which will be
cross producted with the input shapes.
tags(reserved): a tag used to filter inputs.
Reserved Args:
attr_names(reserved): a list of names for input shapes.
attrs(reserved): a list of values for each input shape.
corss_product: a dictionary of attributes which will be
cross producted with the input shapes.
tags(reserved): a tag used to filter inputs.

Here is an example:
Here is an example:
attrs = [
[1, 2],
[4, 5],
Expand All @@ -150,31 +150,31 @@ def config_list(**configs):
"""
generated_configs = []
reserved_names = ['attrs', 'attr_names', 'tags']
if any(attr not in configs for attr in reserved_names):
if any(attr not in configs for attr in reserved_names):
raise ValueError("Missing attrs in configs")

cross_configs = None
if 'cross_product_configs' in configs:
if 'cross_product_configs' in configs:
cross_configs = cross_product_configs(**configs['cross_product_configs'])

for inputs in configs['attrs']:
tmp_result = [{configs['attr_names'][i] : input_value}
tmp_result = [{configs['attr_names'][i] : input_value}
for i, input_value in enumerate(inputs)]
# TODO(mingzhe0908):
# TODO(mingzhe0908):
# If multiple 'tags' were provided, do they get concat?
# If a config has both ['short', 'medium'], it should match
# If a config has both ['short', 'medium'], it should match
# both 'short' and 'medium' tag-filter?
tmp_result.append({'tags' : '_'.join(configs['tags'])})
if cross_configs:
if cross_configs:
generated_configs += [tmp_result + list(config) for config in cross_configs]
else:
else:
generated_configs.append(tmp_result)

return generated_configs


def attr_probs(**probs):
""" return the inputs in a dictionary
""" return the inputs in a dictionary
"""
return probs

Expand All @@ -186,7 +186,7 @@ def __init__(self, configs):
self.configs = configs

def _distribution_func(self, key, weights):
""" this is a cumulative distribution function used for random sampling inputs
""" this is a cumulative distribution function used for random sampling inputs
"""
if key in self.saved_cum_distribution:
return self.saved_cum_distribution[key]
Expand All @@ -201,22 +201,22 @@ def _distribution_func(self, key, weights):
return result

def _random_sample(self, key, values, weights):
""" given values and weights, this function randomly sample values based their weights
""" given values and weights, this function randomly sample values based their weights
"""
# TODO(mingzhe09088): cache the results to avoid recalculation overhead
# TODO(mingzhe09088): cache the results to avoid recalculation overhead
assert len(values) == len(weights)
_distribution_func_vals = self._distribution_func(key, weights)
x = random.random()
idx = bisect.bisect(_distribution_func_vals, x)

assert idx <= len(values), "Wrong index value is returned"
# Due to numerical property, the last value in cumsum could be slightly
# Due to numerical property, the last value in cumsum could be slightly
# smaller than 1, and lead to the (index == len(values)).
if idx == len(values):
idx -= 1
return values[idx]

def get_one_set_of_inputs(self):
def get_one_set_of_inputs(self):
tmp_attr_list = []
for key, values in self.configs.items():
if key in _reserved_keywords:
Expand All @@ -227,47 +227,47 @@ def get_one_set_of_inputs(self):
return (tmp_attr_list)


def random_sample_configs(**configs):
def random_sample_configs(**configs):
"""
This function randomly sample <total_samples> values from the given inputs based on
their weights.
Here is an example showing what are the expected inputs and outpus from this function:
This function randomly sample <total_samples> values from the given inputs based on
their weights.
Here is an example showing what are the expected inputs and outpus from this function:
M = [1, 2],
N = [4, 5],
K = [7, 8],
probs = attr_probs(
probs = attr_probs(
M = [0.7, 0.2],
N = [0.5, 0.2],
K = [0.6, 0.2],
),
total_samples=10,
this function will generate
total_samples=10,
this function will generate
[
[{'K': 7}, {'M': 1}, {'N': 4}],
[{'K': 7}, {'M': 2}, {'N': 5}],
[{'K': 7}, {'M': 1}, {'N': 4}],
[{'K': 7}, {'M': 2}, {'N': 5}],
[{'K': 8}, {'M': 2}, {'N': 4}],
...
]
Note:
The probs is optional. Without them, it implies everything is 1. The probs doesn't
have to reflect the actual normalized probability, the implementation will
Note:
The probs is optional. Without them, it implies everything is 1. The probs doesn't
have to reflect the actual normalized probability, the implementation will
normalize it.
TODO (mingzhe09088):
TODO (mingzhe09088):
(1): a lambda that accepts or rejects a config as a sample. For example: for matmul
with M, N, and K, this function could get rid of (M * N * K > 1e8) to filter out
with M, N, and K, this function could get rid of (M * N * K > 1e8) to filter out
very slow benchmarks.
(2): Make sure each sample is unique. If the number of samples are larger than the
total combinations, just return the cross product. Otherwise, if the number of samples
is close to the number of cross-products, it is numerical safer to generate the list
that you don't want, and remove them.
(2): Make sure each sample is unique. If the number of samples are larger than the
total combinations, just return the cross product. Otherwise, if the number of samples
is close to the number of cross-products, it is numerical safer to generate the list
that you don't want, and remove them.
"""
if "probs" not in configs:
if "probs" not in configs:
raise ValueError("probs is missing. Consider adding probs or"
"using other config functions")

configs_attrs_list = []
randomsample = RandomSample(configs)
for i in range(configs["total_samples"]):
for i in range(configs["total_samples"]):
tmp_attr_list = randomsample.get_one_set_of_inputs()
tmp_attr_list.append({"tags" : '_'.join(configs["tags"])})
configs_attrs_list.append(tmp_attr_list)
Expand All @@ -276,13 +276,13 @@ def random_sample_configs(**configs):

def op_list(**configs):
"""Generate a list of ops organized in a specific format.
It takes two parameters which are "attr_names" and "attr".
attrs stores the name and function of operators.
Args:
configs: key-value pairs including the name and function of
operators. attrs and attr_names must be present in configs.
Return:
a sequence of dictionaries which stores the name and function
It takes two parameters which are "attr_names" and "attr".
attrs stores the name and function of operators.
Args:
configs: key-value pairs including the name and function of
operators. attrs and attr_names must be present in configs.
Return:
a sequence of dictionaries which stores the name and function
of ops in a specifal format
Example:
attrs = [
Expand All @@ -291,7 +291,7 @@ def op_list(**configs):
]
attr_names = ["op_name", "op"].

With those two examples,
With those two examples,
we will generate (({"op_name": "abs"}, {"op" : torch.abs}),
({"op_name": "abs_"}, {"op" : torch.abs_}))
"""
Expand All @@ -315,11 +315,6 @@ def is_pytorch_enabled(framework_arg):

def process_arg_list(arg_list):
if arg_list == 'None':
return None
return None

return [fr.strip() for fr in arg_list.split(',') if len(fr.strip()) > 0]


class SkipInputShape(Exception):
"""Used when a test case should be skipped"""
pass
4 changes: 2 additions & 2 deletions benchmarks/operator_benchmark/pt/conv_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
'in_c', 'out_c', 'kernel', 'stride', 'N', 'L'
],
attrs=[
[256, 256, 3, 1, 1, 64],
[256, 256, 3, 2, 16, 128],
[128, 256, 3, 1, 1, 64],
[256, 256, 3, 2, 4, 64],
],
cross_product_configs={
'device': ['cpu', 'cuda'],
Expand Down
Loading