From 7e5ebb63cfa52237a150504d30b5a17e4dc23864 Mon Sep 17 00:00:00 2001 From: mttk Date: Tue, 24 Apr 2018 16:32:15 +0200 Subject: [PATCH 1/7] Sparse initialization speedup --- torch/nn/init.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/torch/nn/init.py b/torch/nn/init.py index 5ebfc400b12af..b6c441d6eb4ef 100644 --- a/torch/nn/init.py +++ b/torch/nn/init.py @@ -361,16 +361,13 @@ def sparse_(tensor, sparsity, std=0.01): rows, cols = tensor.shape num_zeros = int(math.ceil(rows * sparsity)) - with torch.no_grad(): tensor.normal_(0, std) for col_idx in range(cols): - row_indices = list(range(rows)) - random.shuffle(row_indices) + row_indices = torch.randperm(rows) zero_indices = row_indices[:num_zeros] - for row_idx in zero_indices: - tensor[row_idx, col_idx] = 0 - + t = tensor[:, col_idx] + t[zero_indices] = 0 return tensor From cb2a6624bba36be5a04ae5d82c3534b9fa040c8f Mon Sep 17 00:00:00 2001 From: mttk Date: Tue, 24 Apr 2018 16:34:00 +0200 Subject: [PATCH 2/7] +empty line --- torch/nn/init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torch/nn/init.py b/torch/nn/init.py index b6c441d6eb4ef..cf1ab1c0d5161 100644 --- a/torch/nn/init.py +++ b/torch/nn/init.py @@ -361,6 +361,7 @@ def sparse_(tensor, sparsity, std=0.01): rows, cols = tensor.shape num_zeros = int(math.ceil(rows * sparsity)) + with torch.no_grad(): tensor.normal_(0, std) for col_idx in range(cols): From de6ccdea0aa5194f7faee5039069be979a292f01 Mon Sep 17 00:00:00 2001 From: mttk Date: Tue, 24 Apr 2018 16:51:25 +0200 Subject: [PATCH 3/7] simplify indexing --- torch/nn/init.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torch/nn/init.py b/torch/nn/init.py index cf1ab1c0d5161..fe5e200744348 100644 --- a/torch/nn/init.py +++ b/torch/nn/init.py @@ -367,8 +367,7 @@ def sparse_(tensor, sparsity, std=0.01): for col_idx in range(cols): row_indices = torch.randperm(rows) zero_indices = row_indices[:num_zeros] - t = tensor[:, col_idx] - t[zero_indices] = 0 + tensor[zero_indices, col_idx] = 0 return tensor From 5cb4de70fbefc33e5607e7eaca238f01b39eaf06 Mon Sep 17 00:00:00 2001 From: mttk Date: Thu, 3 May 2018 12:30:11 +0200 Subject: [PATCH 4/7] Can't reproduce locally... --- test/test_nn.py | 2 +- torch/nn/init.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_nn.py b/test/test_nn.py index 4b56c89ae9576..315dd93cbcc4c 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -5121,7 +5121,7 @@ def test_sparse_default_std(self): for col_idx in range(input_tensor.size(1)): column = input_tensor[:, col_idx] - assert column[column == 0].nelement() >= math.ceil(sparsity * cols) + assert column[column == 0].nelement() >= math.ceil(sparsity * cols), "{} : {}".format(column[column == 0].nelement(), math.ceil(sparsity * cols)) assert self._is_normal(input_tensor[input_tensor != 0], 0, std) diff --git a/torch/nn/init.py b/torch/nn/init.py index fe5e200744348..99eebb2291c0a 100644 --- a/torch/nn/init.py +++ b/torch/nn/init.py @@ -360,7 +360,7 @@ def sparse_(tensor, sparsity, std=0.01): raise ValueError("Only tensors with 2 dimensions are supported") rows, cols = tensor.shape - num_zeros = int(math.ceil(rows * sparsity)) + num_zeros = int(math.ceil(sparsity * rows)) with torch.no_grad(): tensor.normal_(0, std) From b13b0d41202efb394302f4914307c93f8680bcad Mon Sep 17 00:00:00 2001 From: mttk Date: Thu, 3 May 2018 13:25:59 +0200 Subject: [PATCH 5/7] Can't reproduce locally...+ --- test/test_nn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_nn.py b/test/test_nn.py index 315dd93cbcc4c..d89488483a722 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -5121,7 +5121,8 @@ def test_sparse_default_std(self): for col_idx in range(input_tensor.size(1)): column = input_tensor[:, col_idx] - assert column[column == 0].nelement() >= math.ceil(sparsity * cols), "{} : {}".format(column[column == 0].nelement(), math.ceil(sparsity * cols)) + assert column[column == 0].nelement() >= math.ceil(sparsity * cols), + "{} : {}, {}".format(column[column == 0].nelement(), sparsity, cols) assert self._is_normal(input_tensor[input_tensor != 0], 0, std) From 9a8100ce2f7005da52b12b01ba6b9eacc0fc4398 Mon Sep 17 00:00:00 2001 From: mttk Date: Thu, 3 May 2018 13:38:35 +0200 Subject: [PATCH 6/7] Can't reproduce locally...+ --- test/test_nn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_nn.py b/test/test_nn.py index d89488483a722..59581ec7080ef 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -5121,8 +5121,7 @@ def test_sparse_default_std(self): for col_idx in range(input_tensor.size(1)): column = input_tensor[:, col_idx] - assert column[column == 0].nelement() >= math.ceil(sparsity * cols), - "{} : {}, {}".format(column[column == 0].nelement(), sparsity, cols) + assert column[column == 0].nelement() >= math.ceil(sparsity * cols), "{} : {}, {}".format(column[column == 0].nelement(), sparsity, cols) assert self._is_normal(input_tensor[input_tensor != 0], 0, std) From f866f5d9afcb74a23f1f4fafd84f03354a393863 Mon Sep 17 00:00:00 2001 From: mttk Date: Thu, 3 May 2018 13:55:58 +0200 Subject: [PATCH 7/7] Fix test, cleanup --- test/test_nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_nn.py b/test/test_nn.py index 59581ec7080ef..bd549550c183b 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -5121,7 +5121,7 @@ def test_sparse_default_std(self): for col_idx in range(input_tensor.size(1)): column = input_tensor[:, col_idx] - assert column[column == 0].nelement() >= math.ceil(sparsity * cols), "{} : {}, {}".format(column[column == 0].nelement(), sparsity, cols) + assert column[column == 0].nelement() >= math.ceil(sparsity * rows) assert self._is_normal(input_tensor[input_tensor != 0], 0, std)