Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement assign function #244

Merged
merged 19 commits into from Apr 23, 2019
Merged
Changes from 17 commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -467,3 +467,6 @@ IFFT:
Prune:
float: [float]
Half: [Half]
Assign:
float: [float]
half: [Half]
@@ -3195,6 +3195,39 @@ Array Manipulation:
function_ids:
Empty: 85
c_runtime: support
Assign:
snake_name: assign
doc: |2
Assign source array to destination array just like `tf.assign`.
This is useful to synchronize or manually update parameters.
.. code-block:: python
dst = nn.Variable((2, 3, 4))
src = nn.Variable((2, 3, 4))
assign = F.assign(dst, src)
assign.forward()
assert np.allclose(dst.d, src.d) # dst and src have identical values.
assert np.allclose(assign.d dst.d) # returned Variable is also identical to dst.
Unlike TensorFlow, the returned Variable has a backward path to `dst`:
.. math::
g_{dst} = g_{y}
inputs:
dst:
doc: A destination N-D array
src:
doc: A source N-D array
outputs:
y:
doc: An assigned array
c_runtime: not support
function_ids:
Empty: 248
Signal Processing:
Interpolate:
snake_name: interpolate
@@ -181,6 +181,7 @@ Array Manipulation
.. autofunction:: sort
.. autofunction:: reshape
.. autofunction:: one_hot
.. autofunction:: assign


Stochasticity
@@ -0,0 +1,72 @@
// Copyright (c) 2017 Sony Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#ifndef NBLA_FUNCTION_ASSIGN_HPP
#define NBLA_FUNCTION_ASSIGN_HPP

#include <nbla/cpu.hpp>
#include <nbla/function.hpp>
#include <nbla/function_registry.hpp>

namespace nbla {

NBLA_REGISTER_FUNCTION_HEADER(Assign);

/** Assign source array to destination array
The function is defined as
@f[
y_i = x_i
@f]
Inputs:
- destination N-D array
- source N-D array
Outputs:
- N-D array identical to source array
\ingroup FunctionImplGrp
*/
template <typename T> class Assign : public BaseFunction<> {
protected:
public:
Assign(const Context &ctx) : BaseFunction(ctx)
{}
virtual ~Assign() {}
virtual shared_ptr<Function> copy() const {
return create_Assign(ctx_);
}
virtual int min_inputs() { return 2; }
virtual int min_outputs() { return 1; }
virtual vector<dtypes> in_types() {
return vector<dtypes>{get_dtype<T>(), get_dtype<T>()};
}
virtual vector<dtypes> out_types() {
return vector<dtypes>{get_dtype<T>()};
}
virtual vector<string> allowed_array_classes() {
return SingletonManager::get<Cpu>()->array_classes();
}
virtual string name() { return "Assign"; }

protected:
NBLA_API virtual void setup_impl(const Variables &inputs, const Variables &outputs);
NBLA_API virtual void forward_impl(const Variables &inputs, const Variables &outputs);
NBLA_API virtual void backward_impl(const Variables &inputs, const Variables &outputs,
const vector<bool> &propagate_down,
const vector<bool> &accum);
};
}
#endif
@@ -0,0 +1,54 @@
# Copyright (c) 2017 Sony Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This is still left unlike other *2 operation (sub2, mul2, ...) because
it has cudnn implementation.
"""
import pytest
import numpy as np
import nnabla as nn
import nnabla.functions as F
from nbla_test_utils import list_context

ctxs = list_context('Assign')


@pytest.mark.parametrize("ctx, func_name", ctxs)
@pytest.mark.parametrize("seed", [314])
def test_assign_forward_backward(seed, ctx, func_name):
rng = np.random.RandomState(seed)
dst = nn.Variable((2, 3, 4), need_grad=True)
src = nn.Variable((2, 3, 4), need_grad=True)

assign = F.assign(dst, src)

src.d = np.random.random((2, 3, 4))
assign.forward()

# destination variable should be equal to source variable
assert np.allclose(dst.d, src.d)
# output variable of assign function should be equal to soure variable
assert np.allclose(assign.d, src.d)

dummy = assign + np.random.random()

dst.grad.zero()
src.grad.zero()
dummy.forward()
dummy.backward()

# gradients at destination are identical to gradients at assign operation
assert np.all(dst.g == dummy.g)
assert np.all(src.g == np.zeros((2, 3, 4)))
@@ -0,0 +1,66 @@
// Copyright (c) 2017 Sony Corporation. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.


#include <nbla/array.hpp>
#include <nbla/common.hpp>
#include <nbla/function/assign.hpp>
#include <nbla/function/add2.hpp>
#include <nbla/variable.hpp>

namespace nbla {

NBLA_REGISTER_FUNCTION_SOURCE(Assign);

template <typename T>
void Assign<T>::setup_impl(const Variables &inputs,
const Variables &outputs) {
NBLA_CHECK(inputs[0]->shape() == inputs[1]->shape(), error_code::value,
"Dimensions of inputs must match. "
"inputs[0]: %s != inputs[1]: %s.",
string_join(inputs[0]->shape(), string(", ")).c_str(),
string_join(inputs[1]->shape(), string(", ")).c_str());
outputs[0]->reshape(inputs[0]->shape(), true);
}

template <typename T>
void Assign<T>::forward_impl(const Variables &inputs,
const Variables &outputs) {
Array *dst = inputs[0]->data()->cast(get_dtype<T>(), this->ctx_, true);
const Array *src = inputs[1]->data()->get(get_dtype<T>(), this->ctx_);
Array *y = outputs[0]->data()->cast(get_dtype<T>(), this->ctx_, true);
dst->copy_from(src);
y->copy_from(src);
}


template <typename T>
void Assign<T>::backward_impl(const Variables &inputs,
const Variables &outputs,
const vector<bool> &propagate_down,
const vector<bool> &accum) {
if (!propagate_down[0])
return;

auto gy = make_shared<Variable>(outputs[0]->grad());
This conversation was marked as resolved by TE-TakuyaNarihira

This comment has been minimized.

Copy link
@TE-TakuyaNarihira

TE-TakuyaNarihira Apr 18, 2019

Contributor

You don't have to use a shared pointer here because it's not shared with any other object. You can simply write;

Variable gy(outputs[0]->grad());

and pass a raw pointer to a Variables by &gy.

That is also the case for gx.

auto gx = make_shared<Variable>(inputs[0]->grad());
auto f_add = create_Add2(this->ctx_, true);
f_add->setup(Variables{gx.get(), gy.get()}, Variables{gx.get()});

if (!accum[0])
gx->data()->zero();
This conversation was marked as resolved by TE-TakuyaNarihira

This comment has been minimized.

Copy link
@TE-TakuyaNarihira

TE-TakuyaNarihira Apr 18, 2019

Contributor

This should not be required because the same thing is actually performed in the parent class function Function::backward.

If we consider more performance improvement, we should copy gy values to gx values if accum is false. The current implementation with accum is False, the zeroing kernel is called, then the add kernel is called. Calling small kernels multiple times leads to some overhead. You can use either Identity class in a similar way to Add2 or Array::copy_from function as in the forward function.


f_add->forward(Variables{gx.get(), gy.get()}, Variables{gx.get()});
}
}
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.