From bb20f6b664f574e48db415ea5493b4cef79853c5 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 2 Dec 2019 17:38:23 -0500 Subject: [PATCH 01/30] Create Setup.py, And Folder Structure --- .gitignore | 1 + README.md | 31 +++++++- neuraxle_tensorflow/__init__.py | 1 + neuraxle_tensorflow/tensorflow_v1.py | 97 +++++++++++++++++++++++ requirements.txt | 1 + setup.py | 114 +++++++++++++++++++++++++++ 6 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 neuraxle_tensorflow/__init__.py create mode 100644 neuraxle_tensorflow/tensorflow_v1.py create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 894a44c..7a93673 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,4 @@ venv.bak/ # mypy .mypy_cache/ +.idea/* diff --git a/README.md b/README.md index f0ef1f3..593a661 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ # Neuraxle-TensorFlow -TensorFlow steps, savers, and utilities for Neuraxle. + +TensorFlow steps, savers, and utilities for [Neuraxle](https://github.com/Neuraxio/Neuraxle). + +Neuraxle is a Machine Learning (ML) library for building neat pipelines, providing the right abstractions to both ease research, development, and deployment of your ML applications. + +## Installation for tensorflow>=1.15 + +``` +neuraxle-tensorflow[tf] +``` + +## Installation for tensorflow-gpu>=1.15 + +``` +neuraxle-tensorflow[tf_gpu] +``` + +## Usage example + +```python +class YourTensorflowModelWrapper(BaseStep): + def __init__(self): + BaseStep.__init__( + self, + hyperparams=HYPERPARAMS, + savers=[TensorflowV1StepSaver()] + ) +``` + +[See also a complete example](https://github.com/Neuraxio/LSTM-Human-Activity-Recognition/blob/neuraxle-refactor/steps/lstm_rnn_tensorflow_model_wrapper.py) diff --git a/neuraxle_tensorflow/__init__.py b/neuraxle_tensorflow/__init__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/neuraxle_tensorflow/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py new file mode 100644 index 0000000..7b89ab4 --- /dev/null +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -0,0 +1,97 @@ +""" +Neuraxle Tensorflow V1 Utility classes +==================================== +Neuraxle utility classes for tensorflow v1. +.. + Copyright 2019, Neuraxio Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +""" +import os +import tensorflow as tf + +from neuraxle.base import BaseSaver + + +class TensorflowV1StepSaver(BaseSaver): + """ + Step saver for a tensorflow Session using tf.train.Saver(). + It saves, or restores the tf.Session() checkpoint at the context path using the step name as file name. + .. seealso:: + `Using the saved model format `_ + """ + + def save_step(self, step: 'BaseStep', context: 'ExecutionContext') -> 'BaseStep': + """ + Save a step that is using tf.train.Saver(). + :param step: step to save + :type step: BaseStep + :param context: execution context to save from + :type context: ExecutionContext + :return: saved step + """ + with step.get_graph().as_default(): + saver = tf.train.Saver() + saver.save( + step.get_session(), + self._get_saved_model_path(context, step) + ) + + step.strip() + + return step + + def load_step(self, step: 'BaseStep', context: 'ExecutionContext') -> 'BaseStep': + """ + Load a step that is using tensorflow using tf.train.Saver(). + :param step: step to load + :type step: BaseStep + :param context: execution context to load from + :type context: ExecutionContext + :return: loaded step + """ + step.is_initialized = False + step.setup() + with step.get_graph().as_default(): + saver = tf.train.Saver() + saver.restore( + step.get_session(), + self._get_saved_model_path(context, step) + ) + + return step + + def can_load(self, step: 'BaseStep', context: 'ExecutionContext'): + """ + Returns whether or not we can load. + :param step: step to load + :type step: BaseStep + :param context: execution context to load from + :type context: ExecutionContext + :return: loaded step + """ + meta_exists = os.path.exists(os.path.join(context.get_path(), "{0}.ckpt.meta".format(step.get_name()))) + index_exists = os.path.exists(os.path.join(context.get_path(), "{0}.ckpt.index".format(step.get_name()))) + return meta_exists and index_exists + + def _get_saved_model_path(self, context, step): + """ + Returns the saved model path using the given execution context, and step name. + :param step: step to load + :type step: BaseStep + :param context: execution context to load from + :type context: ExecutionContext + :return: loaded step + """ + return os.path.join( + context.get_path(), + "{0}.ckpt".format(step.get_name()) + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..000b858 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +neuraxle>=0.2.1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6af6147 --- /dev/null +++ b/setup.py @@ -0,0 +1,114 @@ +""" +Neuraxle +==================================== +This is the core of Neuraxle. Most pipeline steps derive (inherit) from those classes. They are worth noticing. +.. + Copyright 2019, Neuraxio Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from setuptools import setup, find_packages + +from neuraxle import __version__ as _VERSION + +with open('README.md') as _f: + _README = _f.read() + +setup( + name='neuraxle_tensorflow', + version=_VERSION, + description='TensorFlow steps, savers, and utilities for Neuraxle. Neuraxle is a Machine Learning (ML) library for building neat pipelines, providing the right ' + 'abstractions to both ease research, development, and deployment of your ML applications.', + long_description=_README, + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Financial and Insurance Industry", + "Intended Audience :: Healthcare Industry", + "Intended Audience :: Information Technology", + "Intended Audience :: Manufacturing", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "Intended Audience :: Telecommunications Industry", + 'License :: OSI Approved :: Apache Software License', + "Natural Language :: English", + "Operating System :: OS Independent", + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + "Topic :: Adaptive Technologies", + "Topic :: Office/Business", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Artificial Life", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: Scientific/Engineering :: Image Recognition", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Medical Science Apps.", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Software Development", + "Topic :: Software Development :: Assemblers", + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + # Topic :: Software Development :: Object Brokering, + "Topic :: Software Development :: Pre-processors", + "Topic :: Software Development :: Quality Assurance", + "Topic :: Software Development :: Testing", + "Topic :: System", + # Topic :: System :: Clustering, + # Topic :: System :: Distributed Computing, + # Topic :: System :: Networking, + # Topic :: System :: Systems Administration, + "Topic :: Text Processing", + "Topic :: Text Processing :: Filters", + "Topic :: Text Processing :: Linguistic", + "Topic :: Utilities", + "Typing :: Typed" + ], + url='https://github.com/Neuraxio/Neuraxle', + download_url='https://github.com/Neuraxio/Neuraxle/tarball/{}'.format( + _VERSION), + author='Neuraxio Inc.', + author_email='guillaume.chevalier@neuraxio.com', + packages=find_packages(include=['neuraxle-tensorflow*']), + test_suite="testing", + setup_requires=["pytest-runner"], + install_requires=[ + 'neuraxle>=0.2.1' + ], + extras_requires={ + "tf": ["tensorflow>=1.15"], + "tf_gpu": ["tensorflow-gpu>=1.15"], + }, + tests_require=["pytest", "pytest-cov"], + include_package_data=True, + license='Apache 2.0', + keywords='pipeline pipelines data science machine learning deep learning' +) + +print(""" +____________________________________________________________________ + Thank you for installing neuraxle-tensorflow. + + Learn more: + - https://www.neuraxle.org/stable/index.html + Contribute: + - https://gitter.im/Neuraxle/community + Open issue: + - https://github.com/Neuraxio/Neuraxle + Ask questions: + - https://stackoverflow.com/questions/tagged/neuraxle +____________________________________________________________________ +""") From 6c943dd22050acc8292a9f65fdb1c25f5cf23d1d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 2 Dec 2019 17:47:55 -0500 Subject: [PATCH 02/30] Add TensorflowModelWrapperMixin --- README.md | 69 +++++++++++++++++++++++++++- neuraxle_tensorflow/tensorflow_v1.py | 12 +++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 593a661..55197bf 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,80 @@ neuraxle-tensorflow[tf_gpu] ## Usage example ```python -class YourTensorflowModelWrapper(BaseStep): +class YourTensorflowModelWrapper(TensorflowV1ModelWrapperMixin, BaseStep): def __init__(self): + TensorflowV1ModelWrapperMixin.__init__(self) BaseStep.__init__( self, hyperparams=HYPERPARAMS, savers=[TensorflowV1StepSaver()] ) + + def setup(self) -> BaseStep: + if self.is_initialized: + return self + + self.create_graph() + + with self.graph.as_default(): + # Launch the graph + with tf.variable_scope(LSTM_RNN_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): + pred = tf_model_forward(PRED_NAME, X_NAME, Y_NAME, self.hyperparams) + + # Loss, optimizer and evaluation + # L2 loss prevents this overkill neural network to overfit the data + + l2 = self.hyperparams['lambda_loss_amount'] * sum( + tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables() + ) + + # Softmax loss + self.cost = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + labels=self.get_y_placeholder(), + logits=pred + ) + ) + l2 + + # Adam Optimizer + self.optimizer = tf.train.AdamOptimizer( + learning_rate=self.hyperparams['learning_rate'] + ).minimize(self.cost) + + self.correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(self.get_tensor_by_name(Y_NAME), 1)) + self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32)) + + # To keep track of training's performance + self.test_losses = [] + self.test_accuracies = [] + self.train_losses = [] + self.train_accuracies = [] + + self.create_session() + + self.is_initialized = True + + return self + + def create_graph(self): + self.graph = tf.Graph() + + def create_session(self): + self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) + init = tf.global_variables_initializer() + self.sess.run(init) + + def get_tensor_by_name(self, name): + return self.graph.get_tensor_by_name("{0}/{1}:0".format(LSTM_RNN_VARIABLE_SCOPE, name)) + + def get_graph(self): + return self.graph + + def get_session(self): + return self.sess + + # .... + ``` [See also a complete example](https://github.com/Neuraxio/LSTM-Human-Activity-Recognition/blob/neuraxle-refactor/steps/lstm_rnn_tensorflow_model_wrapper.py) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 7b89ab4..d4f127d 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -16,11 +16,23 @@ """ import os +from abc import abstractmethod + import tensorflow as tf from neuraxle.base import BaseSaver +class TensorflowV1ModelWrapperMixin: + @abstractmethod + def get_session(self): + raise NotImplementedError() + + @abstractmethod + def get_graph(self): + raise NotImplementedError() + + class TensorflowV1StepSaver(BaseSaver): """ Step saver for a tensorflow Session using tf.train.Saver(). From 8c06467aded57983ff8140be842d836fdbbfd940 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 2 Dec 2019 17:49:11 -0500 Subject: [PATCH 03/30] Move Usage Example Link In ReadMe --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 55197bf..1d55548 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ neuraxle-tensorflow[tf_gpu] ## Usage example +[See also a complete example](https://github.com/Neuraxio/LSTM-Human-Activity-Recognition/blob/neuraxle-refactor/steps/lstm_rnn_tensorflow_model_wrapper.py) + ```python class YourTensorflowModelWrapper(TensorflowV1ModelWrapperMixin, BaseStep): def __init__(self): @@ -95,4 +97,3 @@ class YourTensorflowModelWrapper(TensorflowV1ModelWrapperMixin, BaseStep): ``` -[See also a complete example](https://github.com/Neuraxio/LSTM-Human-Activity-Recognition/blob/neuraxle-refactor/steps/lstm_rnn_tensorflow_model_wrapper.py) From fd5b1fac254ae6ef56f8973ce1011b56677d9968 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 2 Dec 2019 18:01:10 -0500 Subject: [PATCH 04/30] Add Docstrings To Tensorflow Model V1 Wrapper Mixin --- neuraxle_tensorflow/tensorflow_v1.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index d4f127d..4cab286 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -24,12 +24,28 @@ class TensorflowV1ModelWrapperMixin: + """ + A class that represents a step that contains a tensorflow v1 model. + + .. seealso:: + `Using the saved model format `_ + """ @abstractmethod def get_session(self): + """ + Get the tensorflow tf.Session() object. + + :return: + """ raise NotImplementedError() @abstractmethod def get_graph(self): + """ + Get the tensorflow tf.Graph() object. + + :return: tf.Graph + """ raise NotImplementedError() From d5ccc5dffebf31a203db7611d8bfd6601cd0f92c Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 30 Dec 2019 14:11:46 -0500 Subject: [PATCH 05/30] Add Tensorflow V2 Step Saver --- .gitignore | 2 + README.md | 73 +++++++++++---------- neuraxle_tensorflow/tensorflow_v1.py | 19 +++--- neuraxle_tensorflow/tensorflow_v2.py | 94 ++++++++++++++++++++++++++ setup.py | 20 +++--- testing/__init__.py | 0 testing/test_tensorflow_v1.py | 90 +++++++++++++++++++++++++ testing/test_tensorflow_v2.py | 98 ++++++++++++++++++++++++++++ 8 files changed, 342 insertions(+), 54 deletions(-) create mode 100644 neuraxle_tensorflow/tensorflow_v2.py create mode 100644 testing/__init__.py create mode 100644 testing/test_tensorflow_v1.py create mode 100644 testing/test_tensorflow_v2.py diff --git a/.gitignore b/.gitignore index 7a93673..c7b7590 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,6 @@ venv.bak/ # mypy .mypy_cache/ + +# custom .idea/* diff --git a/README.md b/README.md index 1d55548..d77e8b3 100644 --- a/README.md +++ b/README.md @@ -34,55 +34,56 @@ class YourTensorflowModelWrapper(TensorflowV1ModelWrapperMixin, BaseStep): if self.is_initialized: return self - self.create_graph() + with self.create_graph().as_default(): + self.initialize_graph() + self.is_initialized = True - with self.graph.as_default(): - # Launch the graph - with tf.variable_scope(LSTM_RNN_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): - pred = tf_model_forward(PRED_NAME, X_NAME, Y_NAME, self.hyperparams) - - # Loss, optimizer and evaluation - # L2 loss prevents this overkill neural network to overfit the data - - l2 = self.hyperparams['lambda_loss_amount'] * sum( - tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables() + return self + + def initialize_graph(self): + with tf.variable_scope(LSTM_RNN_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): + pred = tf_model_forward(PRED_NAME, X_NAME, Y_NAME, self.hyperparams) + + # Loss, optimizer and evaluation + # L2 loss prevents this overkill neural network to overfit the data + + l2 = self.hyperparams['lambda_loss_amount'] * sum( + tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables() + ) + + # Softmax loss + self.cost = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + labels=self.get_y_placeholder(), + logits=pred ) + ) + l2 - # Softmax loss - self.cost = tf.reduce_mean( - tf.nn.softmax_cross_entropy_with_logits( - labels=self.get_y_placeholder(), - logits=pred - ) - ) + l2 - - # Adam Optimizer - self.optimizer = tf.train.AdamOptimizer( - learning_rate=self.hyperparams['learning_rate'] - ).minimize(self.cost) + # Adam Optimizer + self.optimizer = tf.train.AdamOptimizer( + learning_rate=self.hyperparams['learning_rate'] + ).minimize(self.cost) - self.correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(self.get_tensor_by_name(Y_NAME), 1)) - self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32)) + self.correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(self.get_tensor_by_name(Y_NAME), 1)) + self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32)) - # To keep track of training's performance - self.test_losses = [] - self.test_accuracies = [] - self.train_losses = [] - self.train_accuracies = [] + # To keep track of training's performance + self.test_losses = [] + self.test_accuracies = [] + self.train_losses = [] + self.train_accuracies = [] - self.create_session() + self.create_session() - self.is_initialized = True - - return self - - def create_graph(self): + def create_graph(self): self.graph = tf.Graph() + return self.graph def create_session(self): self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) init = tf.global_variables_initializer() self.sess.run(init) + return self.sess def get_tensor_by_name(self, name): return self.graph.get_tensor_by_name("{0}/{1}:0".format(LSTM_RNN_VARIABLE_SCOPE, name)) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 4cab286..db3d08b 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -1,6 +1,6 @@ """ Neuraxle Tensorflow V1 Utility classes -==================================== +========================================= Neuraxle utility classes for tensorflow v1. .. Copyright 2019, Neuraxio Inc. @@ -32,15 +32,14 @@ class TensorflowV1ModelWrapperMixin: """ @abstractmethod def get_session(self): - """ - Get the tensorflow tf.Session() object. - - :return: - """ - raise NotImplementedError() + pass @abstractmethod def get_graph(self): + pass + + @abstractmethod + def strip(self): """ Get the tensorflow tf.Graph() object. @@ -57,7 +56,7 @@ class TensorflowV1StepSaver(BaseSaver): `Using the saved model format `_ """ - def save_step(self, step: 'BaseStep', context: 'ExecutionContext') -> 'BaseStep': + def save_step(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': """ Save a step that is using tf.train.Saver(). :param step: step to save @@ -77,7 +76,7 @@ def save_step(self, step: 'BaseStep', context: 'ExecutionContext') -> 'BaseStep' return step - def load_step(self, step: 'BaseStep', context: 'ExecutionContext') -> 'BaseStep': + def load_step(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': """ Load a step that is using tensorflow using tf.train.Saver(). :param step: step to load @@ -97,7 +96,7 @@ def load_step(self, step: 'BaseStep', context: 'ExecutionContext') -> 'BaseStep' return step - def can_load(self, step: 'BaseStep', context: 'ExecutionContext'): + def can_load(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionContext'): """ Returns whether or not we can load. :param step: step to load diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py new file mode 100644 index 0000000..1ec3564 --- /dev/null +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -0,0 +1,94 @@ +""" +Neuraxle Tensorflow V2 Utility classes +========================================= +Neuraxle utility classes for tensorflow v2. +.. + Copyright 2019, Neuraxio Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +""" +from abc import abstractmethod + +from neuraxle.base import BaseSaver + + +class TensorflowV2ModelWrapperMixin: + """ + A class that represents a step that contains a tensorflow v2 model. + + .. seealso:: + `Using the saved model format `_ + """ + + @abstractmethod + def get_checkpoint(self): + pass + + @abstractmethod + def get_checkpoint_manager(self): + pass + + @abstractmethod + def strip(self): + """ + Get the tensorflow tf.Graph() object. + + :return: tf.Graph + """ + raise NotImplementedError() + + +class TensorflowV2StepSaver(BaseSaver): + """ + Step saver for a tensorflow Session using tf.train.Saver(). + It saves, or restores the tf.Session() checkpoint at the context path using the step name as file name. + .. seealso:: + `Using the saved model format `_ + """ + + def save_step(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': + """ + Save a step that is using tf.train.Saver(). + :param step: step to save + :type step: BaseStep + :param context: execution context to save from + :type context: ExecutionContext + :return: saved step + """ + step.get_checkpoint_manager().save() + step.strip() + return step + + def load_step(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': + """ + Load a step that is using tensorflow using tf.train.Checkpoint(). + :param step: step to load + :type step: BaseStep + :param context: execution context to load from + :type context: ExecutionContext + :return: loaded step + """ + step.is_initialized = False + step.setup() + step.get_checkpoint().restore(step.get_checkpoint_manager().latest_checkpoint) + return step + + def can_load(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionContext') -> bool: + """ + Returns whether or not we can load. + + :param step: step to load + :type step: BaseStep + :param context: execution context to load from + :type context: ExecutionContext + :return: loaded step + """ + return True diff --git a/setup.py b/setup.py index 6af6147..8d79149 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ """ -Neuraxle -==================================== -This is the core of Neuraxle. Most pipeline steps derive (inherit) from those classes. They are worth noticing. +Neuraxle Tensorflow Utility classes +========================================= +Neuraxle utility classes for tensorflow. .. Copyright 2019, Neuraxio Inc. Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,7 @@ 'abstractions to both ease research, development, and deployment of your ML applications.', long_description=_README, classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Intended Audience :: Education", "Intended Audience :: Financial and Insurance Industry", @@ -88,10 +88,6 @@ install_requires=[ 'neuraxle>=0.2.1' ], - extras_requires={ - "tf": ["tensorflow>=1.15"], - "tf_gpu": ["tensorflow-gpu>=1.15"], - }, tests_require=["pytest", "pytest-cov"], include_package_data=True, license='Apache 2.0', @@ -99,6 +95,14 @@ ) print(""" +Thank you for installing + _ _ __ + | \ | | | | + | \| | ___ _ _ _ __ ___ __ __ | | ___ + | . ` |/ _ \| | | || ' _||__ \\\\ \/ / | | / _ \\ + | |\ || __|| |_| | | | / _ | > < | | | __| + |_| \_|\___| \__,_||___| \_,_|/_/\_\ |__|\___| + ==> TensorFlow Edition. ____________________________________________________________________ Thank you for installing neuraxle-tensorflow. diff --git a/testing/__init__.py b/testing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py new file mode 100644 index 0000000..c0ac27f --- /dev/null +++ b/testing/test_tensorflow_v1.py @@ -0,0 +1,90 @@ +import tensorflow as tf +from neuraxle.base import BaseStep, ExecutionContext +from neuraxle.pipeline import Pipeline + +from neuraxle_tensorflow.tensorflow_v1 import TensorflowV1StepSaver, TensorflowV1ModelWrapperMixin + +MATMUL_VARIABLE_SCOPE = "matmul" + + +class TensorflowMatMulModel(TensorflowV1ModelWrapperMixin, BaseStep): + def __init__(self): + BaseStep.__init__(self, savers=[TensorflowV1StepSaver()]) + + self.graph = None + self.sess = None + + def setup(self) -> BaseStep: + if self.is_initialized: + return self + + self.graph = tf.Graph() + + with self.graph.as_default(): + self.in_a = tf.placeholder(dtype=tf.float32, shape=2) + self.in_b = tf.placeholder(dtype=tf.float32, shape=2) + + self.out_a = forward(self.in_a) + self.out_b = forward(self.in_b) + self.reg_loss = tf.losses.get_regularization_loss(scope=MATMUL_VARIABLE_SCOPE) + + with tf.variable_scope(MATMUL_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): + self.create_session() + self.is_initialized = True + + return self + + def strip(self): + self.sess = None + self.graph = None + self.in_a = None + self.in_b = None + self.out_a = None + self.out_b = None + self.reg_loss = None + + def create_session(self): + self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) + init = tf.global_variables_initializer() + self.sess.run(init) + + def get_session(self): + return self.sess + + def get_graph(self): + return self.graph + + def teardown(self): + if self.sess is not None: + self.sess.close() + self.is_initialized = False + + def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': + results = self.sess.run([self.out_a, self.out_b, self.reg_loss], + feed_dict={self.in_a: data_inputs[0], self.in_b: data_inputs[1]}) + loss = results[2] + return self + + def transform(self, data_inputs): + return self.sess.run([self.out_a, self.out_b], + feed_dict={self.in_a: data_inputs[0], self.in_b: data_inputs[1]}) + + +def forward(x): + with tf.variable_scope(MATMUL_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): + W = tf.get_variable("W", initializer=tf.ones(shape=(2, 2)), + regularizer=tf.contrib.layers.l2_regularizer(0.04)) + b = tf.get_variable("b", initializer=tf.zeros(shape=2)) + return W * x + b + + +def test_tensorflowv1_saver(tmpdir): + model = Pipeline([TensorflowMatMulModel()]) + + model = model.fit([[2, 1], [2, 4]]) + model.save(ExecutionContext(root=tmpdir)) + + loaded = Pipeline([TensorflowMatMulModel()]).load(ExecutionContext(root=tmpdir)) + outputs = loaded.transform([[3, 1], [5, 4]]) + + assert outputs diff --git a/testing/test_tensorflow_v2.py b/testing/test_tensorflow_v2.py new file mode 100644 index 0000000..bdd6db6 --- /dev/null +++ b/testing/test_tensorflow_v2.py @@ -0,0 +1,98 @@ +import os + +import tensorflow as tf +from neuraxle.base import BaseStep, ExecutionContext +from neuraxle.pipeline import Pipeline + +from neuraxle_tensorflow.tensorflow_v2 import TensorflowV2ModelWrapperMixin, TensorflowV2StepSaver + + +class LinearModel(tf.keras.Model): + def __init__(self): + super(LinearModel, self).__init__() + self.l1 = tf.keras.layers.Dense(5) + + def call(self, x): + return self.l1(x) + + +class Tensorflow2Model(TensorflowV2ModelWrapperMixin, BaseStep): + def __init__(self, tensorflow_checkpoint_folder=None): + BaseStep.__init__(self, savers=[TensorflowV2StepSaver()]) + if tensorflow_checkpoint_folder is None: + tensorflow_checkpoint_folder = 'tf_chkpts' + self.tensorflow_checkpoint_folder = tensorflow_checkpoint_folder + + def setup(self) -> BaseStep: + if self.is_initialized: + return self + + self.optimizer = tf.keras.optimizers.Adam(0.1) + self.model = LinearModel() + self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) + self.checkpoint_manager = tf.train.CheckpointManager(self.checkpoint, self.tensorflow_checkpoint_folder, + max_to_keep=3) + self.is_initialized = True + + return self + + def get_checkpoint(self): + return self.checkpoint + + def get_checkpoint_manager(self): + return self.checkpoint_manager + + def strip(self): + self.optimizer = None + self.model = None + self.checkpoint = None + self.checkpoint_manager = None + self.loss = None + + def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': + x = tf.convert_to_tensor(data_inputs) + y = tf.convert_to_tensor(expected_outputs) + with tf.GradientTape() as tape: + output = self.model(x) + self.loss = tf.reduce_mean(tf.abs(output - y)) + + variables = self.model.trainable_variables + gradients = tape.gradient(self.loss, variables) + self.optimizer.apply_gradients(zip(gradients, variables)) + + return self + + def transform(self, data_inputs): + x = tf.convert_to_tensor(data_inputs) + return self.model(x) + + +def toy_dataset(): + inputs = tf.range(10.)[:, None] + labels = inputs * 5. + tf.range(5.)[None, :] + return tf.data.Dataset.from_tensor_slices( + dict(x=inputs, y=labels)).repeat(10).batch(2) + + +def test_tensorflowv2_saver(tmpdir): + model = Pipeline([ + Tensorflow2Model(tensorflow_checkpoint_folder=os.path.join(tmpdir, 'tf_checkpoints')) + ]) + dataset = toy_dataset() + loss_first_fit = evaluate_model_on_dataset(model, dataset) + + model.save(ExecutionContext(root=tmpdir)) + + loaded = Pipeline([ + Tensorflow2Model(tensorflow_checkpoint_folder=os.path.join(tmpdir, 'tf_checkpoints')) + ]).load(ExecutionContext(root=tmpdir)) + loss_second_fit = evaluate_model_on_dataset(loaded, dataset) + assert loss_second_fit < (loss_first_fit / 2) + + +def evaluate_model_on_dataset(model, dataset): + loss = [] + for example in dataset: + model, outputs = model.fit_transform(example['x'].numpy(), example['y'].numpy()) + loss.append(model['Tensorflow2Model'].loss.numpy()) + return sum(loss) From d623bdb97515b31cfcf7e9b64f64e332566be3cf Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 30 Dec 2019 14:14:04 -0500 Subject: [PATCH 06/30] Add New Line In file Headers before copyright notice --- neuraxle_tensorflow/tensorflow_v1.py | 1 + neuraxle_tensorflow/tensorflow_v2.py | 1 + setup.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index db3d08b..af278f6 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -2,6 +2,7 @@ Neuraxle Tensorflow V1 Utility classes ========================================= Neuraxle utility classes for tensorflow v1. + .. Copyright 2019, Neuraxio Inc. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 1ec3564..ee9544a 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -2,6 +2,7 @@ Neuraxle Tensorflow V2 Utility classes ========================================= Neuraxle utility classes for tensorflow v2. + .. Copyright 2019, Neuraxio Inc. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/setup.py b/setup.py index 8d79149..a460992 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ Neuraxle Tensorflow Utility classes ========================================= Neuraxle utility classes for tensorflow. + .. Copyright 2019, Neuraxio Inc. Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +14,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + """ from setuptools import setup, find_packages From 8337d090af7bf1c97f9a1bc2e89d8abac00448f4 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 30 Dec 2019 18:05:15 -0500 Subject: [PATCH 07/30] Simplify Tensorflow 1 Model And Saver Wip --- neuraxle_tensorflow/tensorflow_v1.py | 108 ++++++++++------ testing/test_tensorflow_v1.py | 187 ++++++++++++++++++--------- 2 files changed, 197 insertions(+), 98 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index af278f6..9aa65e4 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -20,34 +20,78 @@ from abc import abstractmethod import tensorflow as tf +from neuraxle.base import BaseSaver, BaseStep, ExecutionContext + + +class BaseTensorflowV1ModelStep(BaseStep): + def __init__( + self, + variable_scope=None, + hyperparams=None + ): + BaseStep.__init__( + self, + savers=[TensorflowV1StepSaver()], + hyperparams=hyperparams + ) -from neuraxle.base import BaseSaver + self.variable_scope = variable_scope + self.tensorflow_props = {} + def setup(self) -> BaseStep: + if self.is_initialized: + return self -class TensorflowV1ModelWrapperMixin: - """ - A class that represents a step that contains a tensorflow v1 model. + self.graph = tf.Graph() + with self.graph.as_default(): + with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): + self.session = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - .. seealso:: - `Using the saved model format `_ - """ - @abstractmethod - def get_session(self): - pass + self.tensorflow_props = self.setup_graph() + self.tensorflow_props = self.tensorflow_props if self.tensorflow_props is not None else {} + + init = tf.global_variables_initializer() + self.session.run(init) + self.is_initialized = True + + def teardown(self): + if self.session is not None: + self.session.close() + self.is_initialized = False + + def strip(self): + self.tensorflow_props = {} + self.graph = None + self.session = None @abstractmethod - def get_graph(self): - pass + def setup_graph(self): + raise NotImplementedError() + + def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': + with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): + return self.fit_model(data_inputs, expected_outputs) @abstractmethod - def strip(self): - """ - Get the tensorflow tf.Graph() object. + def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: + raise NotImplementedError() - :return: tf.Graph - """ + def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep': + with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): + return self.transform_model(data_inputs) + + @abstractmethod + def transform_model(self, data_inputs): raise NotImplementedError() + def __getitem__(self, item): + if item in self.tensorflow_props: + return self.tensorflow_props[item] + + return self.graph.get_tensor_by_name( + "{0}/{1}:0".format(self.variable_scope, item) + ) + class TensorflowV1StepSaver(BaseSaver): """ @@ -57,7 +101,7 @@ class TensorflowV1StepSaver(BaseSaver): `Using the saved model format `_ """ - def save_step(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': + def save_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Save a step that is using tf.train.Saver(). :param step: step to save @@ -66,18 +110,14 @@ def save_step(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionCo :type context: ExecutionContext :return: saved step """ - with step.get_graph().as_default(): + with step.graph.as_default(): saver = tf.train.Saver() - saver.save( - step.get_session(), - self._get_saved_model_path(context, step) - ) - + saver.save(step.session, self._get_saved_model_path(context, step)) step.strip() return step - def load_step(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': + def load_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Load a step that is using tensorflow using tf.train.Saver(). :param step: step to load @@ -88,16 +128,14 @@ def load_step(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionCo """ step.is_initialized = False step.setup() - with step.get_graph().as_default(): + + with step.graph.as_default(): saver = tf.train.Saver() - saver.restore( - step.get_session(), - self._get_saved_model_path(context, step) - ) + saver.restore(step.session, self._get_saved_model_path(context, step)) return step - def can_load(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionContext'): + def can_load(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext'): """ Returns whether or not we can load. :param step: step to load @@ -108,9 +146,10 @@ def can_load(self, step: 'TensorflowV1ModelWrapperMixin', context: 'ExecutionCon """ meta_exists = os.path.exists(os.path.join(context.get_path(), "{0}.ckpt.meta".format(step.get_name()))) index_exists = os.path.exists(os.path.join(context.get_path(), "{0}.ckpt.index".format(step.get_name()))) + return meta_exists and index_exists - def _get_saved_model_path(self, context, step): + def _get_saved_model_path(self, context: ExecutionContext, step: BaseStep): """ Returns the saved model path using the given execution context, and step name. :param step: step to load @@ -119,7 +158,4 @@ def _get_saved_model_path(self, context, step): :type context: ExecutionContext :return: loaded step """ - return os.path.join( - context.get_path(), - "{0}.ckpt".format(step.get_name()) - ) + return os.path.join(context.get_path(), "{0}.ckpt".format(step.get_name())) diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index c0ac27f..8a6da62 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -1,90 +1,153 @@ +from typing import Dict + +import numpy as np import tensorflow as tf from neuraxle.base import BaseStep, ExecutionContext +from neuraxle.hyperparams.space import HyperparameterSamples from neuraxle.pipeline import Pipeline -from neuraxle_tensorflow.tensorflow_v1 import TensorflowV1StepSaver, TensorflowV1ModelWrapperMixin +from neuraxle_tensorflow.tensorflow_v1 import BaseTensorflowV1ModelStep MATMUL_VARIABLE_SCOPE = "matmul" -class TensorflowMatMulModel(TensorflowV1ModelWrapperMixin, BaseStep): - def __init__(self): - BaseStep.__init__(self, savers=[TensorflowV1StepSaver()]) +class Tensorflow1Model(BaseTensorflowV1ModelStep): + def __init__(self, variable_scope=None): + if variable_scope is None: + variable_scope = 'Tensorflow1Model' + BaseTensorflowV1ModelStep.__init__( + self, + variable_scope=variable_scope, + hyperparams=HyperparameterSamples({ + 'learning_rate': 0.01 + }) + ) - self.graph = None - self.sess = None + def setup_graph(self) -> Dict: + forward(self.hyperparams) + tf.equal(tf.argmax(self['pred'], 1), tf.argmax(self['y'], 1), name='correct_pred') + tf.reduce_mean(tf.cast(self['correct_pred'], tf.float32), name='accuracy') - def setup(self) -> BaseStep: - if self.is_initialized: - return self + l2 = self.hyperparams['lambda_loss_amount'] * sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) + loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self['y'], logits=self['pred'])) + l2 - self.graph = tf.Graph() + optimizer = tf.train.AdamOptimizer( + learning_rate=self.hyperparams['learning_rate'] + ).minimize(loss) - with self.graph.as_default(): - self.in_a = tf.placeholder(dtype=tf.float32, shape=2) - self.in_b = tf.placeholder(dtype=tf.float32, shape=2) + return { + 'loss': loss, + 'optimizer': optimizer + } - self.out_a = forward(self.in_a) - self.out_b = forward(self.in_b) - self.reg_loss = tf.losses.get_regularization_loss(scope=MATMUL_VARIABLE_SCOPE) + def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: + if not isinstance(data_inputs, np.ndarray): + data_inputs = np.array(data_inputs) - with tf.variable_scope(MATMUL_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): - self.create_session() - self.is_initialized = True + if not isinstance(expected_outputs, np.ndarray): + expected_outputs = np.array(expected_outputs) - return self + if expected_outputs.shape != (len(data_inputs), self.hyperparams['n_classes']): + expected_outputs = np.reshape(expected_outputs, (len(data_inputs), self.hyperparams['n_classes'])) - def strip(self): - self.sess = None - self.graph = None - self.in_a = None - self.in_b = None - self.out_a = None - self.out_b = None - self.reg_loss = None - - def create_session(self): - self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - init = tf.global_variables_initializer() - self.sess.run(init) - - def get_session(self): - return self.sess - - def get_graph(self): - return self.graph - - def teardown(self): - if self.sess is not None: - self.sess.close() - self.is_initialized = False - - def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': - results = self.sess.run([self.out_a, self.out_b, self.reg_loss], - feed_dict={self.in_a: data_inputs[0], self.in_b: data_inputs[1]}) - loss = results[2] - return self + _, loss, acc = self.session.run( + [self['optimizer'], self['loss'], self['accuracy']], + feed_dict={ + self['x']: data_inputs, + self['y']: expected_outputs + } + ) - def transform(self, data_inputs): - return self.sess.run([self.out_a, self.out_b], - feed_dict={self.in_a: data_inputs[0], self.in_b: data_inputs[1]}) + print("Batch Loss = " + "{:.6f}".format(loss) + ", Accuracy = {}".format(acc)) + self.is_invalidated = True + + return self -def forward(x): - with tf.variable_scope(MATMUL_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): - W = tf.get_variable("W", initializer=tf.ones(shape=(2, 2)), - regularizer=tf.contrib.layers.l2_regularizer(0.04)) - b = tf.get_variable("b", initializer=tf.zeros(shape=2)) - return W * x + b + def transform_model(self, data_inputs): + if not isinstance(data_inputs, np.ndarray): + data_inputs = np.array(data_inputs) + + return self.session.run( + [self['pred']], + feed_dict={self['x']: data_inputs} + )[0] + + +def forward(hyperparams): + # Function returns a tensorflow LSTM (RNN) artificial neural network from given parameters. + # Moreover, two LSTM cells are stacked which adds deepness to the neural network. + # Note, some code of this notebook is inspired from an slightly different + # RNN architecture used on another dataset, some of the credits goes to + # "aymericdamien" under the MIT license. + # (NOTE: This step could be greatly optimised by shaping the dataset once + # input shape: (batch_size, n_steps, n_input) + + # Graph input/output + x = tf.placeholder(tf.float32, [None, hyperparams['n_steps'], hyperparams['n_inputs']], name='x') + y = tf.placeholder(tf.float32, [None, hyperparams['n_classes']], name='y') + + # Graph weights + weights = { + 'hidden': tf.Variable( + tf.random_normal([hyperparams['n_inputs'], hyperparams['n_hidden']]) + ), # Hidden layer weights + 'out': tf.Variable( + tf.random_normal([hyperparams['n_hidden'], hyperparams['n_classes']], mean=1.0) + ) + } + + biases = { + 'hidden': tf.Variable( + tf.random_normal([hyperparams['n_hidden']]) + ), + 'out': tf.Variable( + tf.random_normal([hyperparams['n_classes']]) + ) + } + + data_inputs = tf.transpose( + x, + [1, 0, 2]) # permute n_steps and batch_size + + # Reshape to prepare input to hidden activation + data_inputs = tf.reshape(data_inputs, [-1, hyperparams['n_inputs']]) + # new shape: (n_steps*batch_size, n_input) + + # ReLU activation, thanks to Yu Zhao for adding this improvement here: + _X = tf.nn.relu( + tf.matmul(data_inputs, weights['hidden']) + biases['hidden'] + ) + + # Split data because rnn cell needs a list of inputs for the RNN inner loop + _X = tf.split(_X, hyperparams['n_steps'], 0) + # new shape: n_steps * (batch_size, n_hidden) + + # Define two stacked LSTM cells (two recurrent layers deep) with tensorflow + lstm_cell_1 = tf.contrib.rnn.BasicLSTMCell(hyperparams['n_hidden'], forget_bias=1.0, state_is_tuple=True) + lstm_cell_2 = tf.contrib.rnn.BasicLSTMCell(hyperparams['n_hidden'], forget_bias=1.0, state_is_tuple=True) + lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell_1, lstm_cell_2], state_is_tuple=True) + + # Get LSTM cell output + outputs, states = tf.contrib.rnn.static_rnn(lstm_cells, _X, dtype=tf.float32) + + # Get last time step's output feature for a "many-to-one" style classifier, + # as in the image describing RNNs at the top of this page + lstm_last_output = outputs[-1] + + # Linear activation + pred = tf.matmul(lstm_last_output, weights['out']) + biases['out'] + return tf.identity(pred, name='pred') def test_tensorflowv1_saver(tmpdir): - model = Pipeline([TensorflowMatMulModel()]) + model = Pipeline([ + Tensorflow1Model() + ]) - model = model.fit([[2, 1], [2, 4]]) model.save(ExecutionContext(root=tmpdir)) - loaded = Pipeline([TensorflowMatMulModel()]).load(ExecutionContext(root=tmpdir)) + loaded = Pipeline([Tensorflow1Model()]).load(ExecutionContext(root=tmpdir)) outputs = loaded.transform([[3, 1], [5, 4]]) assert outputs From 536b4dd5370d248c41ab06a06335d2e4a0ff4274 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 30 Dec 2019 23:12:43 -0500 Subject: [PATCH 08/30] Simplify Tensorflow 2 Models, And Saver --- neuraxle_tensorflow/tensorflow_v2.py | 70 +++++++++----- testing/test_tensorflow_v1.py | 137 ++++++++------------------- testing/test_tensorflow_v2.py | 41 ++------ 3 files changed, 95 insertions(+), 153 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index ee9544a..70d4217 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -18,32 +18,56 @@ """ from abc import abstractmethod -from neuraxle.base import BaseSaver +from neuraxle.base import BaseSaver, BaseStep, ExecutionContext +import tensorflow as tf -class TensorflowV2ModelWrapperMixin: - """ - A class that represents a step that contains a tensorflow v2 model. +class BaseTensorflowV2ModelStep(BaseStep): + def __init__( + self, + hyperparams=None, + tensorflow_checkpoint_folder=None + ): + BaseStep.__init__( + self, + savers=[TensorflowV2StepSaver()], + hyperparams=hyperparams + ) - .. seealso:: - `Using the saved model format `_ - """ + if tensorflow_checkpoint_folder is None: + tensorflow_checkpoint_folder = 'tensorflow_ckpts' + self.tensorflow_checkpoint_folder = tensorflow_checkpoint_folder - @abstractmethod - def get_checkpoint(self): - pass + def setup(self) -> BaseStep: + if self.is_initialized: + return self - @abstractmethod - def get_checkpoint_manager(self): - pass + self.optimizer = self.create_optimizer() + self.model = self.create_model() + + self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) + self.checkpoint_manager = tf.train.CheckpointManager( + self.checkpoint, + self.tensorflow_checkpoint_folder, + max_to_keep=3 + ) + + self.is_initialized = True + + return self + + def teardown(self): + self.is_initialized = False - @abstractmethod def strip(self): - """ - Get the tensorflow tf.Graph() object. + self.tensorflow_props = {} - :return: tf.Graph - """ + @abstractmethod + def create_optimizer(self): + raise NotImplementedError() + + @abstractmethod + def create_model(self): raise NotImplementedError() @@ -55,7 +79,7 @@ class TensorflowV2StepSaver(BaseSaver): `Using the saved model format `_ """ - def save_step(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': + def save_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Save a step that is using tf.train.Saver(). :param step: step to save @@ -64,11 +88,11 @@ def save_step(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionCo :type context: ExecutionContext :return: saved step """ - step.get_checkpoint_manager().save() + step.checkpoint_manager.save() step.strip() return step - def load_step(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionContext') -> 'BaseStep': + def load_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Load a step that is using tensorflow using tf.train.Checkpoint(). :param step: step to load @@ -79,10 +103,10 @@ def load_step(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionCo """ step.is_initialized = False step.setup() - step.get_checkpoint().restore(step.get_checkpoint_manager().latest_checkpoint) + step.checkpoint.restore(step.checkpoint_manager.latest_checkpoint) return step - def can_load(self, step: 'TensorflowV2ModelWrapperMixin', context: 'ExecutionContext') -> bool: + def can_load(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> bool: """ Returns whether or not we can load. diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index 8a6da62..3dd5594 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -19,135 +19,76 @@ def __init__(self, variable_scope=None): self, variable_scope=variable_scope, hyperparams=HyperparameterSamples({ - 'learning_rate': 0.01 + 'learning_rate': 0.01, + 'lambda_loss_amount': 0.0015 }) ) def setup_graph(self) -> Dict: - forward(self.hyperparams) - tf.equal(tf.argmax(self['pred'], 1), tf.argmax(self['y'], 1), name='correct_pred') - tf.reduce_mean(tf.cast(self['correct_pred'], tf.float32), name='accuracy') + tf.placeholder(tf.float32, [None, None], name='x') + tf.placeholder(tf.float32, [None, None], name='y') l2 = self.hyperparams['lambda_loss_amount'] * sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) - loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self['y'], logits=self['pred'])) + l2 - + pred = tf.keras.layers.Dense(5) + loss = tf.reduce_mean(pred) + l2 optimizer = tf.train.AdamOptimizer( learning_rate=self.hyperparams['learning_rate'] ).minimize(loss) + tf.equal(tf.argmax(self['pred'], 1), tf.argmax(self['y'], 1), name='correct_pred') + tf.reduce_mean(tf.cast(self['correct_pred'], tf.float32), name='accuracy') + return { + 'pred': pred, 'loss': loss, 'optimizer': optimizer } - def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: - if not isinstance(data_inputs, np.ndarray): - data_inputs = np.array(data_inputs) - - if not isinstance(expected_outputs, np.ndarray): - expected_outputs = np.array(expected_outputs) - - if expected_outputs.shape != (len(data_inputs), self.hyperparams['n_classes']): - expected_outputs = np.reshape(expected_outputs, (len(data_inputs), self.hyperparams['n_classes'])) - - _, loss, acc = self.session.run( - [self['optimizer'], self['loss'], self['accuracy']], - feed_dict={ - self['x']: data_inputs, - self['y']: expected_outputs - } - ) - - print("Batch Loss = " + "{:.6f}".format(loss) + ", Accuracy = {}".format(acc)) + def fit_model(self, data_inputs, expected_outputs=None) -> 'BaseStep': + _, loss, acc = self.session.run( + [self['optimizer'], self['loss'], self['accuracy']], + feed_dict={ + self['x']: np.array(data_inputs), + self['y']: np.array(expected_outputs) + } + ) + self.loss = loss self.is_invalidated = True return self def transform_model(self, data_inputs): - if not isinstance(data_inputs, np.ndarray): - data_inputs = np.array(data_inputs) - return self.session.run( [self['pred']], - feed_dict={self['x']: data_inputs} + feed_dict={self['x']: np.array(data_inputs)} )[0] -def forward(hyperparams): - # Function returns a tensorflow LSTM (RNN) artificial neural network from given parameters. - # Moreover, two LSTM cells are stacked which adds deepness to the neural network. - # Note, some code of this notebook is inspired from an slightly different - # RNN architecture used on another dataset, some of the credits goes to - # "aymericdamien" under the MIT license. - # (NOTE: This step could be greatly optimised by shaping the dataset once - # input shape: (batch_size, n_steps, n_input) - - # Graph input/output - x = tf.placeholder(tf.float32, [None, hyperparams['n_steps'], hyperparams['n_inputs']], name='x') - y = tf.placeholder(tf.float32, [None, hyperparams['n_classes']], name='y') - - # Graph weights - weights = { - 'hidden': tf.Variable( - tf.random_normal([hyperparams['n_inputs'], hyperparams['n_hidden']]) - ), # Hidden layer weights - 'out': tf.Variable( - tf.random_normal([hyperparams['n_hidden'], hyperparams['n_classes']], mean=1.0) - ) - } - - biases = { - 'hidden': tf.Variable( - tf.random_normal([hyperparams['n_hidden']]) - ), - 'out': tf.Variable( - tf.random_normal([hyperparams['n_classes']]) - ) - } - - data_inputs = tf.transpose( - x, - [1, 0, 2]) # permute n_steps and batch_size - - # Reshape to prepare input to hidden activation - data_inputs = tf.reshape(data_inputs, [-1, hyperparams['n_inputs']]) - # new shape: (n_steps*batch_size, n_input) - - # ReLU activation, thanks to Yu Zhao for adding this improvement here: - _X = tf.nn.relu( - tf.matmul(data_inputs, weights['hidden']) + biases['hidden'] - ) - - # Split data because rnn cell needs a list of inputs for the RNN inner loop - _X = tf.split(_X, hyperparams['n_steps'], 0) - # new shape: n_steps * (batch_size, n_hidden) - - # Define two stacked LSTM cells (two recurrent layers deep) with tensorflow - lstm_cell_1 = tf.contrib.rnn.BasicLSTMCell(hyperparams['n_hidden'], forget_bias=1.0, state_is_tuple=True) - lstm_cell_2 = tf.contrib.rnn.BasicLSTMCell(hyperparams['n_hidden'], forget_bias=1.0, state_is_tuple=True) - lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell_1, lstm_cell_2], state_is_tuple=True) - - # Get LSTM cell output - outputs, states = tf.contrib.rnn.static_rnn(lstm_cells, _X, dtype=tf.float32) - - # Get last time step's output feature for a "many-to-one" style classifier, - # as in the image describing RNNs at the top of this page - lstm_last_output = outputs[-1] - - # Linear activation - pred = tf.matmul(lstm_last_output, weights['out']) + biases['out'] - return tf.identity(pred, name='pred') +def toy_dataset(): + return [ + ([[0.], [1.]], [[0., 1., 2., 3., 4.], [5., 6., 7., 8., 9.]]), + ([[2.], [3.]], [[10., 11., 12., 13., 14.], [15., 16., 17., 18., 19.]]), + ([[4.], [5.]], [[20., 21., 22., 23., 24.], [25., 26., 27., 28., 29.]]), + ([[6.], [7.]], [[30., 31., 32., 33., 34.], [35., 36., 37., 38., 39.]]) + ] def test_tensorflowv1_saver(tmpdir): - model = Pipeline([ - Tensorflow1Model() - ]) + model = Pipeline([Tensorflow1Model()]) + dataset = toy_dataset() + loss_first_fit = evaluate_model_on_dataset(model, dataset) model.save(ExecutionContext(root=tmpdir)) loaded = Pipeline([Tensorflow1Model()]).load(ExecutionContext(root=tmpdir)) - outputs = loaded.transform([[3, 1], [5, 4]]) + loss_second_fit = evaluate_model_on_dataset(loaded, dataset) + assert loss_second_fit < (loss_first_fit / 2) + - assert outputs +def evaluate_model_on_dataset(model, dataset): + loss = [] + for x, y in dataset: + model, outputs = model.fit_transform(x, y) + loss.append(model['Tensorflow1Model'].loss) + return sum(loss) diff --git a/testing/test_tensorflow_v2.py b/testing/test_tensorflow_v2.py index bdd6db6..cf02968 100644 --- a/testing/test_tensorflow_v2.py +++ b/testing/test_tensorflow_v2.py @@ -4,7 +4,7 @@ from neuraxle.base import BaseStep, ExecutionContext from neuraxle.pipeline import Pipeline -from neuraxle_tensorflow.tensorflow_v2 import TensorflowV2ModelWrapperMixin, TensorflowV2StepSaver +from neuraxle_tensorflow.tensorflow_v2 import BaseTensorflowV2ModelStep class LinearModel(tf.keras.Model): @@ -16,42 +16,20 @@ def call(self, x): return self.l1(x) -class Tensorflow2Model(TensorflowV2ModelWrapperMixin, BaseStep): +class Tensorflow2Model(BaseTensorflowV2ModelStep): def __init__(self, tensorflow_checkpoint_folder=None): - BaseStep.__init__(self, savers=[TensorflowV2StepSaver()]) - if tensorflow_checkpoint_folder is None: - tensorflow_checkpoint_folder = 'tf_chkpts' - self.tensorflow_checkpoint_folder = tensorflow_checkpoint_folder - - def setup(self) -> BaseStep: - if self.is_initialized: - return self - - self.optimizer = tf.keras.optimizers.Adam(0.1) - self.model = LinearModel() - self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) - self.checkpoint_manager = tf.train.CheckpointManager(self.checkpoint, self.tensorflow_checkpoint_folder, - max_to_keep=3) - self.is_initialized = True + BaseTensorflowV2ModelStep.__init__(self, tensorflow_checkpoint_folder) - return self - - def get_checkpoint(self): - return self.checkpoint - - def get_checkpoint_manager(self): - return self.checkpoint_manager + def create_optimizer(self): + return tf.keras.optimizers.Adam(0.1) - def strip(self): - self.optimizer = None - self.model = None - self.checkpoint = None - self.checkpoint_manager = None - self.loss = None + def create_model(self): + return LinearModel() def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': x = tf.convert_to_tensor(data_inputs) y = tf.convert_to_tensor(expected_outputs) + with tf.GradientTape() as tape: output = self.model(x) self.loss = tf.reduce_mean(tf.abs(output - y)) @@ -63,8 +41,7 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': return self def transform(self, data_inputs): - x = tf.convert_to_tensor(data_inputs) - return self.model(x) + return self.model(tf.convert_to_tensor(data_inputs)) def toy_dataset(): From 84680d62a3246df4754e8ead67597f21bfc5758d Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 30 Dec 2019 23:24:48 -0500 Subject: [PATCH 09/30] Simplify Tensorflow 2 Models, And Saver --- neuraxle_tensorflow/tensorflow_v1.py | 4 +++- neuraxle_tensorflow/tensorflow_v2.py | 25 ++++++++++++++----------- testing/test_tensorflow_v2.py | 8 ++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 9aa65e4..fc6e2bc 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -97,8 +97,10 @@ class TensorflowV1StepSaver(BaseSaver): """ Step saver for a tensorflow Session using tf.train.Saver(). It saves, or restores the tf.Session() checkpoint at the context path using the step name as file name. + .. seealso:: - `Using the saved model format `_ + `Using the saved model format `_, + :class:`~neuraxle.base.BaseSaver` """ def save_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 70d4217..d3b3175 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -23,20 +23,16 @@ class BaseTensorflowV2ModelStep(BaseStep): - def __init__( - self, - hyperparams=None, - tensorflow_checkpoint_folder=None - ): + def __init__(self, checkpoint_folder=None, hyperparams=None): BaseStep.__init__( self, savers=[TensorflowV2StepSaver()], hyperparams=hyperparams ) - if tensorflow_checkpoint_folder is None: - tensorflow_checkpoint_folder = 'tensorflow_ckpts' - self.tensorflow_checkpoint_folder = tensorflow_checkpoint_folder + if checkpoint_folder is None: + checkpoint_folder = 'tensorflow_ckpts' + self.checkpoint_folder = checkpoint_folder def setup(self) -> BaseStep: if self.is_initialized: @@ -48,7 +44,7 @@ def setup(self) -> BaseStep: self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) self.checkpoint_manager = tf.train.CheckpointManager( self.checkpoint, - self.tensorflow_checkpoint_folder, + self.checkpoint_folder, max_to_keep=3 ) @@ -60,7 +56,10 @@ def teardown(self): self.is_initialized = False def strip(self): - self.tensorflow_props = {} + self.optimizer = None + self.model = None + self.checkpoint = None + self.checkpoint_manager = None @abstractmethod def create_optimizer(self): @@ -73,15 +72,18 @@ def create_model(self): class TensorflowV2StepSaver(BaseSaver): """ - Step saver for a tensorflow Session using tf.train.Saver(). + Step saver for a tensorflow Session using tf.train.Checkpoint(). It saves, or restores the tf.Session() checkpoint at the context path using the step name as file name. + .. seealso:: `Using the saved model format `_ + :class:`~neuraxle.base.BaseSaver` """ def save_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Save a step that is using tf.train.Saver(). + :param step: step to save :type step: BaseStep :param context: execution context to save from @@ -95,6 +97,7 @@ def save_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContex def load_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Load a step that is using tensorflow using tf.train.Checkpoint(). + :param step: step to load :type step: BaseStep :param context: execution context to load from diff --git a/testing/test_tensorflow_v2.py b/testing/test_tensorflow_v2.py index cf02968..148c776 100644 --- a/testing/test_tensorflow_v2.py +++ b/testing/test_tensorflow_v2.py @@ -17,8 +17,8 @@ def call(self, x): class Tensorflow2Model(BaseTensorflowV2ModelStep): - def __init__(self, tensorflow_checkpoint_folder=None): - BaseTensorflowV2ModelStep.__init__(self, tensorflow_checkpoint_folder) + def __init__(self=None, checkpoint_folder=None, hyperparams=None): + BaseTensorflowV2ModelStep.__init__(self, checkpoint_folder=checkpoint_folder, hyperparams=hyperparams) def create_optimizer(self): return tf.keras.optimizers.Adam(0.1) @@ -53,7 +53,7 @@ def toy_dataset(): def test_tensorflowv2_saver(tmpdir): model = Pipeline([ - Tensorflow2Model(tensorflow_checkpoint_folder=os.path.join(tmpdir, 'tf_checkpoints')) + Tensorflow2Model(os.path.join(tmpdir, 'tf_checkpoints')) ]) dataset = toy_dataset() loss_first_fit = evaluate_model_on_dataset(model, dataset) @@ -61,7 +61,7 @@ def test_tensorflowv2_saver(tmpdir): model.save(ExecutionContext(root=tmpdir)) loaded = Pipeline([ - Tensorflow2Model(tensorflow_checkpoint_folder=os.path.join(tmpdir, 'tf_checkpoints')) + Tensorflow2Model(os.path.join(tmpdir, 'tf_checkpoints')) ]).load(ExecutionContext(root=tmpdir)) loss_second_fit = evaluate_model_on_dataset(loaded, dataset) assert loss_second_fit < (loss_first_fit / 2) From 0efda6badf7775fb6e9ddc4cdd0de8d234f2d0ab Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 30 Dec 2019 23:50:34 -0500 Subject: [PATCH 10/30] Add Docstrings To Base Tensorflow Steps --- neuraxle_tensorflow/tensorflow_v1.py | 54 +++++++++++++++++++++++++++- neuraxle_tensorflow/tensorflow_v2.py | 41 +++++++++++++++++---- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index fc6e2bc..2062440 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -24,6 +24,14 @@ class BaseTensorflowV1ModelStep(BaseStep): + """ + Base class for tensorflow 1 steps. + It uses :class:`TensorflowV1StepSaver` for saving the model. + + .. seealso:: + `Using the saved model format `_, + :class:`~neuraxle.base.BaseStep` + """ def __init__( self, variable_scope=None, @@ -39,6 +47,12 @@ def __init__( self.tensorflow_props = {} def setup(self) -> BaseStep: + """ + Setup tensorflow 1 graph, and session using a variable scope. + + :return: self + :rtype: BaseStep + """ if self.is_initialized: return self @@ -54,16 +68,31 @@ def setup(self) -> BaseStep: self.session.run(init) self.is_initialized = True - def teardown(self): + def teardown(self) -> BaseStep: + """ + Close session on teardown. + + :return: + """ if self.session is not None: self.session.close() self.is_initialized = False + return self + def strip(self): + """ + Strip tensorflow 1 properties from to step to make the step serializable. + + :return: stripped step + :rtype: BaseStep + """ self.tensorflow_props = {} self.graph = None self.session = None + return self + @abstractmethod def setup_graph(self): raise NotImplementedError() @@ -74,6 +103,14 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': @abstractmethod def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: + """ + Fit tensorflow model using the variable scope. + + :param data_inputs: data inputs + :param expected_outputs: expected outputs to fit on + :return: fitted self + :rtype: BaseStep + """ raise NotImplementedError() def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep': @@ -82,9 +119,24 @@ def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep': @abstractmethod def transform_model(self, data_inputs): + """ + Transform tensorflow model using the variable scope. + + :param data_inputs: + :return: + """ raise NotImplementedError() def __getitem__(self, item): + """ + Get a graph tensor by name using get item. + + :param item: tensor name + :type item: str + + :return: tensor + :rtype: tf.Tensor + """ if item in self.tensorflow_props: return self.tensorflow_props[item] diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index d3b3175..28167ea 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -20,9 +20,19 @@ from neuraxle.base import BaseSaver, BaseStep, ExecutionContext import tensorflow as tf +from tensorflow_core.python.keras.optimizer_v2 import optimizer_v2 class BaseTensorflowV2ModelStep(BaseStep): + """ + Base class for tensorflow 2 steps. + It uses :class:`TensorflowV2StepSaver` for saving the model. + + .. seealso:: + `Using the checkpoint model format `_, + :class:`~neuraxle.base.BaseStep` + """ + def __init__(self, checkpoint_folder=None, hyperparams=None): BaseStep.__init__( self, @@ -35,6 +45,12 @@ def __init__(self, checkpoint_folder=None, hyperparams=None): self.checkpoint_folder = checkpoint_folder def setup(self) -> BaseStep: + """ + Setup optimizer, model, and checkpoints for saving. + + :return: step + :rtype: BaseStep + """ if self.is_initialized: return self @@ -52,21 +68,34 @@ def setup(self) -> BaseStep: return self - def teardown(self): - self.is_initialized = False - def strip(self): + """ + Strip tensorflow 2 properties from to step to make it serializable. + + :return: + """ self.optimizer = None self.model = None self.checkpoint = None self.checkpoint_manager = None @abstractmethod - def create_optimizer(self): + def create_optimizer(self) -> optimizer_v2.OptimizerV2: + """ + Create the tensorflow 2 optimizer to apply gradients. + + :return: tensorflow optimizer v2 + :rtype: optimizer_v2.OptimizerV2 + """ raise NotImplementedError() @abstractmethod - def create_model(self): + def create_model(self) -> tf.keras.Model: + """ + Create the Tensorflow 2 Model to apply gradients. + + :return: + """ raise NotImplementedError() @@ -76,7 +105,7 @@ class TensorflowV2StepSaver(BaseSaver): It saves, or restores the tf.Session() checkpoint at the context path using the step name as file name. .. seealso:: - `Using the saved model format `_ + `Using the checkpoint model format `_ :class:`~neuraxle.base.BaseSaver` """ From bb41beee0c778d29b0abea86cef25fe12bdb619f Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Tue, 31 Dec 2019 11:47:16 -0500 Subject: [PATCH 11/30] Fix Tensorflow 1 And Tensorflow 2 Test / Examples --- testing/test_tensorflow_v1.py | 70 ++++++++++++----------------------- testing/test_tensorflow_v2.py | 7 ++-- 2 files changed, 27 insertions(+), 50 deletions(-) diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index 3dd5594..b612f93 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -8,6 +8,8 @@ from neuraxle_tensorflow.tensorflow_v1 import BaseTensorflowV1ModelStep +N_SAMPLES = 17 + MATMUL_VARIABLE_SCOPE = "matmul" @@ -19,76 +21,50 @@ def __init__(self, variable_scope=None): self, variable_scope=variable_scope, hyperparams=HyperparameterSamples({ - 'learning_rate': 0.01, - 'lambda_loss_amount': 0.0015 + 'learning_rate': 0.01 }) ) def setup_graph(self) -> Dict: - tf.placeholder(tf.float32, [None, None], name='x') - tf.placeholder(tf.float32, [None, None], name='y') + tf.placeholder('float', name='x') + tf.placeholder('float', name='y') + + tf.Variable(np.random.rand(), name='weight') + tf.Variable(np.random.rand(), name='bias') - l2 = self.hyperparams['lambda_loss_amount'] * sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()) - pred = tf.keras.layers.Dense(5) - loss = tf.reduce_mean(pred) + l2 - optimizer = tf.train.AdamOptimizer( - learning_rate=self.hyperparams['learning_rate'] - ).minimize(loss) + tf.add(tf.multiply(self['x'], self['weight']), self['bias'], name='pred') - tf.equal(tf.argmax(self['pred'], 1), tf.argmax(self['y'], 1), name='correct_pred') - tf.reduce_mean(tf.cast(self['correct_pred'], tf.float32), name='accuracy') + loss = tf.reduce_sum(tf.pow(self['pred'] - self['y'], 2)) / (2 * N_SAMPLES) + optimizer = tf.train.GradientDescentOptimizer(self.hyperparams['learning_rate']).minimize(loss) return { - 'pred': pred, 'loss': loss, 'optimizer': optimizer } def fit_model(self, data_inputs, expected_outputs=None) -> 'BaseStep': - _, loss, acc = self.session.run( - [self['optimizer'], self['loss'], self['accuracy']], - feed_dict={ - self['x']: np.array(data_inputs), - self['y']: np.array(expected_outputs) - } - ) - self.loss = loss + for (x, y) in zip(data_inputs, expected_outputs): + self.session.run(self['optimizer'], feed_dict={self['x']: x, self['y']: y}) self.is_invalidated = True return self def transform_model(self, data_inputs): - return self.session.run( - [self['pred']], - feed_dict={self['x']: np.array(data_inputs)} - )[0] - - -def toy_dataset(): - return [ - ([[0.], [1.]], [[0., 1., 2., 3., 4.], [5., 6., 7., 8., 9.]]), - ([[2.], [3.]], [[10., 11., 12., 13., 14.], [15., 16., 17., 18., 19.]]), - ([[4.], [5.]], [[20., 21., 22., 23., 24.], [25., 26., 27., 28., 29.]]), - ([[6.], [7.]], [[30., 31., 32., 33., 34.], [35., 36., 37., 38., 39.]]) - ] + return self.session.run(self['weight']) * data_inputs + self.session.run(self['bias']) def test_tensorflowv1_saver(tmpdir): + data_inputs = np.array([3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167, + 7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1]) + expected_ouptuts = np.array([1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221, + 2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3]) model = Pipeline([Tensorflow1Model()]) - dataset = toy_dataset() - loss_first_fit = evaluate_model_on_dataset(model, dataset) + for i in range(50): + model, outputs = model.fit_transform(data_inputs, expected_ouptuts) model.save(ExecutionContext(root=tmpdir)) - loaded = Pipeline([Tensorflow1Model()]).load(ExecutionContext(root=tmpdir)) - loss_second_fit = evaluate_model_on_dataset(loaded, dataset) - assert loss_second_fit < (loss_first_fit / 2) - - -def evaluate_model_on_dataset(model, dataset): - loss = [] - for x, y in dataset: - model, outputs = model.fit_transform(x, y) - loss.append(model['Tensorflow1Model'].loss) - return sum(loss) + model = Pipeline([Tensorflow1Model()]).load(ExecutionContext(root=tmpdir)) + model, outputs = model.fit_transform(data_inputs, expected_ouptuts) + assert ((outputs - expected_ouptuts) ** 2).mean() < 0.25 diff --git a/testing/test_tensorflow_v2.py b/testing/test_tensorflow_v2.py index 148c776..5029b84 100644 --- a/testing/test_tensorflow_v2.py +++ b/testing/test_tensorflow_v2.py @@ -34,9 +34,10 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': output = self.model(x) self.loss = tf.reduce_mean(tf.abs(output - y)) - variables = self.model.trainable_variables - gradients = tape.gradient(self.loss, variables) - self.optimizer.apply_gradients(zip(gradients, variables)) + self.optimizer.apply_gradients(zip( + tape.gradient(self.loss, self.model.trainable_variables), + self.model.trainable_variables + )) return self From a98b150d1227ade7be66aca02192d204e25fa888 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 6 Jan 2020 14:20:33 -0500 Subject: [PATCH 12/30] Apply Review Comments #5 --- neuraxle_tensorflow/tensorflow.py | 8 ++++++++ neuraxle_tensorflow/tensorflow_v1.py | 16 +++++++++++++++- neuraxle_tensorflow/tensorflow_v2.py | 10 +++++----- requirements.txt | 2 +- testing/test_tensorflow_v2.py | 4 ++-- 5 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 neuraxle_tensorflow/tensorflow.py diff --git a/neuraxle_tensorflow/tensorflow.py b/neuraxle_tensorflow/tensorflow.py new file mode 100644 index 0000000..a05f9da --- /dev/null +++ b/neuraxle_tensorflow/tensorflow.py @@ -0,0 +1,8 @@ +class BaseTensorflowModelStep(BaseStep): + def __init__(self, create_graph, create_loss, create_optimizer): + self.create_graph = create_graph + self.create_loss = create_loss + self.create_optimizer = create_optimizer + + self.set_hyperparams(self.__class__.HYPERPARAMS) + self.set_hyperparams_space(self.__class__.HYPERPARAMS_SPACE) \ No newline at end of file diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 2062440..6701223 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -22,6 +22,8 @@ import tensorflow as tf from neuraxle.base import BaseSaver, BaseStep, ExecutionContext +from neuraxle_tensorflow.tensorflow import BaseTensorflowModelStep + class BaseTensorflowV1ModelStep(BaseStep): """ @@ -137,13 +139,25 @@ def __getitem__(self, item): :return: tensor :rtype: tf.Tensor """ + if ":" in item: + split = item.split(":") + tensor_name = split[0] + device = split[1] + else: + tensor_name = item + device = "0" + if item in self.tensorflow_props: return self.tensorflow_props[item] return self.graph.get_tensor_by_name( - "{0}/{1}:0".format(self.variable_scope, item) + "{0}/{1}:{2}".format(self.variable_scope, tensor_name, device) ) +class TensorflowV1ModelStep(BaseTensorflowModelStep): + def __init__(self): + pass + class TensorflowV1StepSaver(BaseSaver): """ diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 28167ea..d07fb2c 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -33,16 +33,16 @@ class BaseTensorflowV2ModelStep(BaseStep): :class:`~neuraxle.base.BaseStep` """ - def __init__(self, checkpoint_folder=None, hyperparams=None): + def __init__(self, tf_model_checkpoint_folder=None, hyperparams=None): BaseStep.__init__( self, savers=[TensorflowV2StepSaver()], hyperparams=hyperparams ) - if checkpoint_folder is None: - checkpoint_folder = 'tensorflow_ckpts' - self.checkpoint_folder = checkpoint_folder + if tf_model_checkpoint_folder is None: + tf_model_checkpoint_folder = 'tensorflow_ckpts' + self.tf_model_checkpoint_folder = tf_model_checkpoint_folder def setup(self) -> BaseStep: """ @@ -60,7 +60,7 @@ def setup(self) -> BaseStep: self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) self.checkpoint_manager = tf.train.CheckpointManager( self.checkpoint, - self.checkpoint_folder, + self.tf_model_checkpoint_folder, max_to_keep=3 ) diff --git a/requirements.txt b/requirements.txt index 000b858..7a84c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -neuraxle>=0.2.1 \ No newline at end of file +Neuraxle \ No newline at end of file diff --git a/testing/test_tensorflow_v2.py b/testing/test_tensorflow_v2.py index 5029b84..c41ec45 100644 --- a/testing/test_tensorflow_v2.py +++ b/testing/test_tensorflow_v2.py @@ -17,8 +17,8 @@ def call(self, x): class Tensorflow2Model(BaseTensorflowV2ModelStep): - def __init__(self=None, checkpoint_folder=None, hyperparams=None): - BaseTensorflowV2ModelStep.__init__(self, checkpoint_folder=checkpoint_folder, hyperparams=hyperparams) + def __init__(self, tf_model_checkpoint_folder=None, hyperparams=None): + BaseTensorflowV2ModelStep.__init__(self, tf_model_checkpoint_folder=tf_model_checkpoint_folder, hyperparams=hyperparams) def create_optimizer(self): return tf.keras.optimizers.Adam(0.1) From c6baca8f255b2fef28f184001f6a8632d2a72339 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 6 Jan 2020 15:55:51 -0500 Subject: [PATCH 13/30] Apply Review Comments Pr #5 --- neuraxle_tensorflow/tensorflow.py | 13 ++++- neuraxle_tensorflow/tensorflow_v1.py | 72 ++++++++++++++++------------ neuraxle_tensorflow/tensorflow_v2.py | 3 +- testing/test_tensorflow_v1.py | 70 ++++++++++++--------------- 4 files changed, 84 insertions(+), 74 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow.py b/neuraxle_tensorflow/tensorflow.py index a05f9da..20293b5 100644 --- a/neuraxle_tensorflow/tensorflow.py +++ b/neuraxle_tensorflow/tensorflow.py @@ -1,8 +1,17 @@ +from neuraxle.base import BaseStep + + class BaseTensorflowModelStep(BaseStep): - def __init__(self, create_graph, create_loss, create_optimizer): + def __init__(self, create_graph, create_loss, create_optimizer, step_saver): self.create_graph = create_graph self.create_loss = create_loss self.create_optimizer = create_optimizer self.set_hyperparams(self.__class__.HYPERPARAMS) - self.set_hyperparams_space(self.__class__.HYPERPARAMS_SPACE) \ No newline at end of file + self.set_hyperparams_space(self.__class__.HYPERPARAMS_SPACE) + + BaseStep.__init__( + self, + savers=[step_saver], + hyperparams=self.HYPERPARAMS + ) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 6701223..ac21d0c 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -17,15 +17,15 @@ """ import os -from abc import abstractmethod import tensorflow as tf from neuraxle.base import BaseSaver, BaseStep, ExecutionContext +from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace from neuraxle_tensorflow.tensorflow import BaseTensorflowModelStep -class BaseTensorflowV1ModelStep(BaseStep): +class TensorflowV1ModelStep(BaseTensorflowModelStep): """ Base class for tensorflow 1 steps. It uses :class:`TensorflowV1StepSaver` for saving the model. @@ -34,19 +34,29 @@ class BaseTensorflowV1ModelStep(BaseStep): `Using the saved model format `_, :class:`~neuraxle.base.BaseStep` """ + HYPERPARAMS = HyperparameterSamples({}) + HYPERPARAMS_SPACE = HyperparameterSpace({}) + def __init__( self, + create_graph, + create_loss, + create_optimizer, variable_scope=None, - hyperparams=None + has_expected_outputs=False, ): - BaseStep.__init__( + BaseTensorflowModelStep.__init__( self, - savers=[TensorflowV1StepSaver()], - hyperparams=hyperparams + create_graph=create_graph, + create_loss=create_loss, + create_optimizer=create_optimizer, + step_saver=TensorflowV1StepSaver() ) + if variable_scope is None: + variable_scope = self.name self.variable_scope = variable_scope - self.tensorflow_props = {} + self.has_expected_outputs = has_expected_outputs def setup(self) -> BaseStep: """ @@ -63,8 +73,9 @@ def setup(self) -> BaseStep: with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): self.session = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - self.tensorflow_props = self.setup_graph() - self.tensorflow_props = self.tensorflow_props if self.tensorflow_props is not None else {} + self.create_graph(self) + tf.identity(self.create_loss(self), name='loss') + self.create_optimizer(self).minimize(self['loss'], name='optimizer') init = tf.global_variables_initializer() self.session.run(init) @@ -89,21 +100,15 @@ def strip(self): :return: stripped step :rtype: BaseStep """ - self.tensorflow_props = {} self.graph = None self.session = None return self - @abstractmethod - def setup_graph(self): - raise NotImplementedError() - def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): return self.fit_model(data_inputs, expected_outputs) - @abstractmethod def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: """ Fit tensorflow model using the variable scope. @@ -113,13 +118,19 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: :return: fitted self :rtype: BaseStep """ - raise NotImplementedError() + self.session.run( + self['optimizer'], + feed_dict={ + self['data_inputs']: data_inputs, + self['expected_outputs']: expected_outputs + } + ) + return self def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep': with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): return self.transform_model(data_inputs) - @abstractmethod def transform_model(self, data_inputs): """ Transform tensorflow model using the variable scope. @@ -127,7 +138,12 @@ def transform_model(self, data_inputs): :param data_inputs: :return: """ - raise NotImplementedError() + return self.session.run( + self['output'], + feed_dict={ + self['data_inputs']: data_inputs + } + ) def __getitem__(self, item): """ @@ -147,16 +163,12 @@ def __getitem__(self, item): tensor_name = item device = "0" - if item in self.tensorflow_props: - return self.tensorflow_props[item] - - return self.graph.get_tensor_by_name( - "{0}/{1}:{2}".format(self.variable_scope, tensor_name, device) - ) + try: + result = self.graph.get_tensor_by_name("{0}/{1}:{2}".format(self.variable_scope, tensor_name, device)) + except KeyError: + result = self.graph.get_operation_by_name("{0}/{1}".format(self.variable_scope, tensor_name)) -class TensorflowV1ModelStep(BaseTensorflowModelStep): - def __init__(self): - pass + return result class TensorflowV1StepSaver(BaseSaver): @@ -169,7 +181,7 @@ class TensorflowV1StepSaver(BaseSaver): :class:`~neuraxle.base.BaseSaver` """ - def save_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': + def save_step(self, step: 'TensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Save a step that is using tf.train.Saver(). :param step: step to save @@ -185,7 +197,7 @@ def save_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContex return step - def load_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': + def load_step(self, step: 'TensorflowV1ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Load a step that is using tensorflow using tf.train.Saver(). :param step: step to load @@ -203,7 +215,7 @@ def load_step(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContex return step - def can_load(self, step: 'BaseTensorflowV1ModelStep', context: 'ExecutionContext'): + def can_load(self, step: 'TensorflowV1ModelStep', context: 'ExecutionContext'): """ Returns whether or not we can load. :param step: step to load diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index d07fb2c..8ee5932 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -20,7 +20,6 @@ from neuraxle.base import BaseSaver, BaseStep, ExecutionContext import tensorflow as tf -from tensorflow_core.python.keras.optimizer_v2 import optimizer_v2 class BaseTensorflowV2ModelStep(BaseStep): @@ -80,7 +79,7 @@ def strip(self): self.checkpoint_manager = None @abstractmethod - def create_optimizer(self) -> optimizer_v2.OptimizerV2: + def create_optimizer(self): """ Create the tensorflow 2 optimizer to apply gradients. diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index b612f93..c5fb028 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -1,57 +1,33 @@ -from typing import Dict - import numpy as np import tensorflow as tf -from neuraxle.base import BaseStep, ExecutionContext -from neuraxle.hyperparams.space import HyperparameterSamples +from neuraxle.base import ExecutionContext +from neuraxle.hyperparams.distributions import LogUniform +from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace from neuraxle.pipeline import Pipeline -from neuraxle_tensorflow.tensorflow_v1 import BaseTensorflowV1ModelStep +from neuraxle_tensorflow.tensorflow_v1 import TensorflowV1ModelStep N_SAMPLES = 17 MATMUL_VARIABLE_SCOPE = "matmul" -class Tensorflow1Model(BaseTensorflowV1ModelStep): - def __init__(self, variable_scope=None): - if variable_scope is None: - variable_scope = 'Tensorflow1Model' - BaseTensorflowV1ModelStep.__init__( - self, - variable_scope=variable_scope, - hyperparams=HyperparameterSamples({ - 'learning_rate': 0.01 - }) - ) - - def setup_graph(self) -> Dict: - tf.placeholder('float', name='x') - tf.placeholder('float', name='y') - - tf.Variable(np.random.rand(), name='weight') - tf.Variable(np.random.rand(), name='bias') - - tf.add(tf.multiply(self['x'], self['weight']), self['bias'], name='pred') +def create_graph(step: TensorflowV1ModelStep): + tf.placeholder('float', name='data_inputs') + tf.placeholder('float', name='expected_outputs') - loss = tf.reduce_sum(tf.pow(self['pred'] - self['y'], 2)) / (2 * N_SAMPLES) - optimizer = tf.train.GradientDescentOptimizer(self.hyperparams['learning_rate']).minimize(loss) + tf.Variable(np.random.rand(), name='weight') + tf.Variable(np.random.rand(), name='bias') - return { - 'loss': loss, - 'optimizer': optimizer - } + tf.add(tf.multiply(step['data_inputs'], step['weight']), step['bias'], name='output') - def fit_model(self, data_inputs, expected_outputs=None) -> 'BaseStep': - for (x, y) in zip(data_inputs, expected_outputs): - self.session.run(self['optimizer'], feed_dict={self['x']: x, self['y']: y}) - self.is_invalidated = True +def create_loss(step: TensorflowV1ModelStep): + return tf.reduce_sum(tf.pow(step['output'] - step['expected_outputs'], 2)) / (2 * N_SAMPLES) - return self - def transform_model(self, data_inputs): - return self.session.run(self['weight']) * data_inputs + self.session.run(self['bias']) +def create_optimizer(step: TensorflowV1ModelStep): + return tf.train.GradientDescentOptimizer(step.hyperparams['learning_rate']) def test_tensorflowv1_saver(tmpdir): @@ -59,12 +35,26 @@ def test_tensorflowv1_saver(tmpdir): 7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1]) expected_ouptuts = np.array([1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221, 2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3]) - model = Pipeline([Tensorflow1Model()]) + model = Pipeline([create_model_step()]) + for i in range(50): model, outputs = model.fit_transform(data_inputs, expected_ouptuts) model.save(ExecutionContext(root=tmpdir)) - model = Pipeline([Tensorflow1Model()]).load(ExecutionContext(root=tmpdir)) + model = Pipeline([create_model_step()]).load(ExecutionContext(root=tmpdir)) model, outputs = model.fit_transform(data_inputs, expected_ouptuts) assert ((outputs - expected_ouptuts) ** 2).mean() < 0.25 + + +def create_model_step(): + return TensorflowV1ModelStep( + create_graph=create_graph, + create_loss=create_loss, + create_optimizer=create_optimizer, + has_expected_outputs=False + ).set_hyperparams(HyperparameterSamples({ + 'learning_rate': 0.01 + })).set_hyperparams_space(HyperparameterSpace({ + 'learning_rate': LogUniform(0.0001, 0.01) + })) From 333945d3aae3685235403c171b7e4ddc726329fe Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 6 Jan 2020 16:33:38 -0500 Subject: [PATCH 14/30] Apply Review Comments #5 --- neuraxle_tensorflow/tensorflow.py | 4 +- neuraxle_tensorflow/tensorflow_v1.py | 21 +++++---- neuraxle_tensorflow/tensorflow_v2.py | 65 ++++++++++++++++------------ testing/test_tensorflow_v1.py | 2 +- testing/test_tensorflow_v2.py | 65 +++++++++++++--------------- 5 files changed, 83 insertions(+), 74 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow.py b/neuraxle_tensorflow/tensorflow.py index 20293b5..00e2e9d 100644 --- a/neuraxle_tensorflow/tensorflow.py +++ b/neuraxle_tensorflow/tensorflow.py @@ -2,8 +2,8 @@ class BaseTensorflowModelStep(BaseStep): - def __init__(self, create_graph, create_loss, create_optimizer, step_saver): - self.create_graph = create_graph + def __init__(self, create_model, create_loss, create_optimizer, step_saver): + self.create_model = create_model self.create_loss = create_loss self.create_optimizer = create_optimizer diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index ac21d0c..935134e 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -39,7 +39,7 @@ class TensorflowV1ModelStep(BaseTensorflowModelStep): def __init__( self, - create_graph, + create_grah, create_loss, create_optimizer, variable_scope=None, @@ -47,7 +47,7 @@ def __init__( ): BaseTensorflowModelStep.__init__( self, - create_graph=create_graph, + create_model=create_grah, create_loss=create_loss, create_optimizer=create_optimizer, step_saver=TensorflowV1StepSaver() @@ -73,7 +73,7 @@ def setup(self) -> BaseStep: with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): self.session = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - self.create_graph(self) + self.create_model(self) tf.identity(self.create_loss(self), name='loss') self.create_optimizer(self).minimize(self['loss'], name='optimizer') @@ -118,13 +118,16 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: :return: fitted self :rtype: BaseStep """ - self.session.run( - self['optimizer'], - feed_dict={ - self['data_inputs']: data_inputs, + feed_dict = { + self['data_inputs']: data_inputs + } + + if self.has_expected_outputs: + feed_dict.update({ self['expected_outputs']: expected_outputs - } - ) + }) + + self.session.run(self['optimizer'], feed_dict=feed_dict) return self def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep': diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 8ee5932..ee75aef 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -16,13 +16,15 @@ limitations under the License. """ -from abc import abstractmethod -from neuraxle.base import BaseSaver, BaseStep, ExecutionContext import tensorflow as tf +from neuraxle.base import BaseSaver, BaseStep, ExecutionContext +from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace + +from neuraxle_tensorflow.tensorflow import BaseTensorflowModelStep -class BaseTensorflowV2ModelStep(BaseStep): +class Tensorflow2ModelStep(BaseTensorflowModelStep): """ Base class for tensorflow 2 steps. It uses :class:`TensorflowV2StepSaver` for saving the model. @@ -31,12 +33,22 @@ class BaseTensorflowV2ModelStep(BaseStep): `Using the checkpoint model format `_, :class:`~neuraxle.base.BaseStep` """ + HYPERPARAMS = HyperparameterSamples({}) + HYPERPARAMS_SPACE = HyperparameterSpace({}) - def __init__(self, tf_model_checkpoint_folder=None, hyperparams=None): - BaseStep.__init__( + def __init__( self, - savers=[TensorflowV2StepSaver()], - hyperparams=hyperparams + create_model, + create_loss, + create_optimizer, + tf_model_checkpoint_folder=None + ): + BaseTensorflowModelStep.__init__( + self, + create_model=create_model, + create_loss=create_loss, + create_optimizer=create_optimizer, + step_saver=TensorflowV2StepSaver() ) if tf_model_checkpoint_folder is None: @@ -53,8 +65,8 @@ def setup(self) -> BaseStep: if self.is_initialized: return self - self.optimizer = self.create_optimizer() - self.model = self.create_model() + self.optimizer = self.create_optimizer(self) + self.model = self.create_model(self) self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) self.checkpoint_manager = tf.train.CheckpointManager( @@ -78,24 +90,23 @@ def strip(self): self.checkpoint = None self.checkpoint_manager = None - @abstractmethod - def create_optimizer(self): - """ - Create the tensorflow 2 optimizer to apply gradients. + def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': + x = tf.convert_to_tensor(data_inputs) + y = tf.convert_to_tensor(expected_outputs) - :return: tensorflow optimizer v2 - :rtype: optimizer_v2.OptimizerV2 - """ - raise NotImplementedError() + with tf.GradientTape() as tape: + output = self.model(x) + self.loss = self.create_loss(self, y, output) - @abstractmethod - def create_model(self) -> tf.keras.Model: - """ - Create the Tensorflow 2 Model to apply gradients. + self.optimizer.apply_gradients(zip( + tape.gradient(self.loss, self.model.trainable_variables), + self.model.trainable_variables + )) - :return: - """ - raise NotImplementedError() + return self + + def transform(self, data_inputs): + return self.model(tf.convert_to_tensor(data_inputs)).numpy() class TensorflowV2StepSaver(BaseSaver): @@ -108,7 +119,7 @@ class TensorflowV2StepSaver(BaseSaver): :class:`~neuraxle.base.BaseSaver` """ - def save_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> 'BaseStep': + def save_step(self, step: 'Tensorflow2ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Save a step that is using tf.train.Saver(). @@ -122,7 +133,7 @@ def save_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContex step.strip() return step - def load_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> 'BaseStep': + def load_step(self, step: 'Tensorflow2ModelStep', context: 'ExecutionContext') -> 'BaseStep': """ Load a step that is using tensorflow using tf.train.Checkpoint(). @@ -137,7 +148,7 @@ def load_step(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContex step.checkpoint.restore(step.checkpoint_manager.latest_checkpoint) return step - def can_load(self, step: 'BaseTensorflowV2ModelStep', context: 'ExecutionContext') -> bool: + def can_load(self, step: 'Tensorflow2ModelStep', context: 'ExecutionContext') -> bool: """ Returns whether or not we can load. diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index c5fb028..deae49a 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -49,7 +49,7 @@ def test_tensorflowv1_saver(tmpdir): def create_model_step(): return TensorflowV1ModelStep( - create_graph=create_graph, + create_grah=create_graph, create_loss=create_loss, create_optimizer=create_optimizer, has_expected_outputs=False diff --git a/testing/test_tensorflow_v2.py b/testing/test_tensorflow_v2.py index c41ec45..2a5c21e 100644 --- a/testing/test_tensorflow_v2.py +++ b/testing/test_tensorflow_v2.py @@ -1,10 +1,10 @@ import os import tensorflow as tf -from neuraxle.base import BaseStep, ExecutionContext +from neuraxle.base import ExecutionContext from neuraxle.pipeline import Pipeline -from neuraxle_tensorflow.tensorflow_v2 import BaseTensorflowV2ModelStep +from neuraxle_tensorflow.tensorflow_v2 import Tensorflow2ModelStep class LinearModel(tf.keras.Model): @@ -16,61 +16,56 @@ def call(self, x): return self.l1(x) -class Tensorflow2Model(BaseTensorflowV2ModelStep): - def __init__(self, tf_model_checkpoint_folder=None, hyperparams=None): - BaseTensorflowV2ModelStep.__init__(self, tf_model_checkpoint_folder=tf_model_checkpoint_folder, hyperparams=hyperparams) - - def create_optimizer(self): - return tf.keras.optimizers.Adam(0.1) +def toy_dataset(): + inputs = tf.range(10.)[:, None] + labels = inputs * 5. + tf.range(5.)[None, :] + return tf.data.Dataset.from_tensor_slices( + dict(x=inputs, y=labels)).repeat(10).batch(2) - def create_model(self): - return LinearModel() - def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': - x = tf.convert_to_tensor(data_inputs) - y = tf.convert_to_tensor(expected_outputs) +def create_model(step: Tensorflow2ModelStep): + return LinearModel() - with tf.GradientTape() as tape: - output = self.model(x) - self.loss = tf.reduce_mean(tf.abs(output - y)) - self.optimizer.apply_gradients(zip( - tape.gradient(self.loss, self.model.trainable_variables), - self.model.trainable_variables - )) +def create_optimizer(step: Tensorflow2ModelStep): + return tf.keras.optimizers.Adam(0.1) - return self - def transform(self, data_inputs): - return self.model(tf.convert_to_tensor(data_inputs)) - - -def toy_dataset(): - inputs = tf.range(10.)[:, None] - labels = inputs * 5. + tf.range(5.)[None, :] - return tf.data.Dataset.from_tensor_slices( - dict(x=inputs, y=labels)).repeat(10).batch(2) +def create_loss(step: Tensorflow2ModelStep, expected_outputs, actual_outputs): + return tf.reduce_mean(tf.abs(actual_outputs - expected_outputs)) def test_tensorflowv2_saver(tmpdir): + dataset = toy_dataset() model = Pipeline([ - Tensorflow2Model(os.path.join(tmpdir, 'tf_checkpoints')) + create_model_step(tmpdir) ]) - dataset = toy_dataset() loss_first_fit = evaluate_model_on_dataset(model, dataset) model.save(ExecutionContext(root=tmpdir)) loaded = Pipeline([ - Tensorflow2Model(os.path.join(tmpdir, 'tf_checkpoints')) + create_model_step(tmpdir) ]).load(ExecutionContext(root=tmpdir)) loss_second_fit = evaluate_model_on_dataset(loaded, dataset) + assert loss_second_fit < (loss_first_fit / 2) +def create_model_step(tmpdir): + return Tensorflow2ModelStep( + create_model=create_model, + create_optimizer=create_optimizer, + create_loss=create_loss, + tf_model_checkpoint_folder=os.path.join(tmpdir, 'tf_checkpoints') + ) + + def evaluate_model_on_dataset(model, dataset): loss = [] for example in dataset: - model, outputs = model.fit_transform(example['x'].numpy(), example['y'].numpy()) - loss.append(model['Tensorflow2Model'].loss.numpy()) + expected_outputs = example['y'].numpy() + model, outputs = model.fit_transform(example['x'].numpy(), expected_outputs) + loss.append(((outputs - expected_outputs) ** 2).mean()) + return sum(loss) From d68b876e39a5158f87ec7b84d98619625332d8ce Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 6 Jan 2020 16:42:53 -0500 Subject: [PATCH 15/30] Fix README --- README.md | 133 ++++++++++++++++++++---------------------------------- 1 file changed, 48 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index d77e8b3..f34c727 100644 --- a/README.md +++ b/README.md @@ -4,97 +4,60 @@ TensorFlow steps, savers, and utilities for [Neuraxle](https://github.com/Neurax Neuraxle is a Machine Learning (ML) library for building neat pipelines, providing the right abstractions to both ease research, development, and deployment of your ML applications. -## Installation for tensorflow>=1.15 +## Usage example -``` -neuraxle-tensorflow[tf] -``` +[See also a complete example](https://github.com/Neuraxio/LSTM-Human-Activity-Recognition/blob/neuraxle-refactor/steps/lstm_rnn_tensorflow_model_wrapper.py) -## Installation for tensorflow-gpu>=1.15 +### Tensorflow 1 -``` -neuraxle-tensorflow[tf_gpu] +Create a tensorflow 1 model step by giving it a graph, an optimizer, and a loss function. + +```python +def create_graph(step: TensorflowV1ModelStep): + tf.placeholder('float', name='data_inputs') + tf.placeholder('float', name='expected_outputs') + + tf.Variable(np.random.rand(), name='weight') + tf.Variable(np.random.rand(), name='bias') + + tf.add(tf.multiply(step['data_inputs'], step['weight']), step['bias'], name='output') + +def create_loss(step: TensorflowV1ModelStep): + return tf.reduce_sum(tf.pow(step['output'] - step['expected_outputs'], 2)) / (2 * N_SAMPLES) + +def create_optimizer(step: TensorflowV1ModelStep): + return tf.train.GradientDescentOptimizer(step.hyperparams['learning_rate']) + +model_step = TensorflowV1ModelStep( + create_grah=create_graph, + create_loss=create_loss, + create_optimizer=create_optimizer, + has_expected_outputs=False +).set_hyperparams(HyperparameterSamples({ + 'learning_rate': 0.01 +})).set_hyperparams_space(HyperparameterSpace({ + 'learning_rate': LogUniform(0.0001, 0.01) +})) ``` -## Usage example +### Tensorflow 2 -[See also a complete example](https://github.com/Neuraxio/LSTM-Human-Activity-Recognition/blob/neuraxle-refactor/steps/lstm_rnn_tensorflow_model_wrapper.py) +Create a tensorflow 2 model step by giving it a model, an optimizer, and a loss function. ```python -class YourTensorflowModelWrapper(TensorflowV1ModelWrapperMixin, BaseStep): - def __init__(self): - TensorflowV1ModelWrapperMixin.__init__(self) - BaseStep.__init__( - self, - hyperparams=HYPERPARAMS, - savers=[TensorflowV1StepSaver()] - ) - - def setup(self) -> BaseStep: - if self.is_initialized: - return self - - with self.create_graph().as_default(): - self.initialize_graph() - self.is_initialized = True - - return self - - def initialize_graph(self): - with tf.variable_scope(LSTM_RNN_VARIABLE_SCOPE, reuse=tf.AUTO_REUSE): - pred = tf_model_forward(PRED_NAME, X_NAME, Y_NAME, self.hyperparams) - - # Loss, optimizer and evaluation - # L2 loss prevents this overkill neural network to overfit the data - - l2 = self.hyperparams['lambda_loss_amount'] * sum( - tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables() - ) - - # Softmax loss - self.cost = tf.reduce_mean( - tf.nn.softmax_cross_entropy_with_logits( - labels=self.get_y_placeholder(), - logits=pred - ) - ) + l2 - - # Adam Optimizer - self.optimizer = tf.train.AdamOptimizer( - learning_rate=self.hyperparams['learning_rate'] - ).minimize(self.cost) - - self.correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(self.get_tensor_by_name(Y_NAME), 1)) - self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32)) - - # To keep track of training's performance - self.test_losses = [] - self.test_accuracies = [] - self.train_losses = [] - self.train_accuracies = [] - - self.create_session() - - def create_graph(self): - self.graph = tf.Graph() - return self.graph - - def create_session(self): - self.sess = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - init = tf.global_variables_initializer() - self.sess.run(init) - return self.sess - - def get_tensor_by_name(self, name): - return self.graph.get_tensor_by_name("{0}/{1}:0".format(LSTM_RNN_VARIABLE_SCOPE, name)) - - def get_graph(self): - return self.graph - - def get_session(self): - return self.sess - - # .... - -``` +def create_model(step: Tensorflow2ModelStep): + return LinearModel() +def create_optimizer(step: Tensorflow2ModelStep): + return tf.keras.optimizers.Adam(0.1) + +def create_loss(step: Tensorflow2ModelStep, expected_outputs, actual_outputs): + return tf.reduce_mean(tf.abs(actual_outputs - expected_outputs)) + +model_step = Tensorflow2ModelStep( + create_model=create_model, + create_optimizer=create_optimizer, + create_loss=create_loss, + tf_model_checkpoint_folder=os.path.join(tmpdir, 'tf_checkpoints') +) +``` From 1d72294c9e00408a0a9da55c2647182f326a8668 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 6 Jan 2020 19:10:50 -0500 Subject: [PATCH 16/30] Fix Tensorflow V1 Model Graph Argument And Loss --- neuraxle_tensorflow/tensorflow_v1.py | 12 +++++++----- testing/test_tensorflow_v1.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 935134e..8cc7801 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -39,15 +39,15 @@ class TensorflowV1ModelStep(BaseTensorflowModelStep): def __init__( self, - create_grah, + create_graph, create_loss, create_optimizer, variable_scope=None, - has_expected_outputs=False, + has_expected_outputs=True, ): BaseTensorflowModelStep.__init__( self, - create_model=create_grah, + create_model=create_graph, create_loss=create_loss, create_optimizer=create_optimizer, step_saver=TensorflowV1StepSaver() @@ -73,7 +73,7 @@ def setup(self) -> BaseStep: with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): self.session = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - self.create_model(self) + tf.identity(self.create_model(self), name='output') tf.identity(self.create_loss(self), name='loss') self.create_optimizer(self).minimize(self['loss'], name='optimizer') @@ -127,7 +127,9 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: self['expected_outputs']: expected_outputs }) - self.session.run(self['optimizer'], feed_dict=feed_dict) + results = self.session.run([self['optimizer'], self['loss']], feed_dict=feed_dict) + self.loss = results[1] + return self def transform(self, data_inputs, expected_outputs=None) -> 'BaseStep': diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index deae49a..9ede8a8 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -49,10 +49,10 @@ def test_tensorflowv1_saver(tmpdir): def create_model_step(): return TensorflowV1ModelStep( - create_grah=create_graph, + create_graph=create_graph, create_loss=create_loss, create_optimizer=create_optimizer, - has_expected_outputs=False + has_expected_outputs=True ).set_hyperparams(HyperparameterSamples({ 'learning_rate': 0.01 })).set_hyperparams_space(HyperparameterSpace({ From 16846bf2f9f379d2216fa15e9aae487ed1a88b2b Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 8 Jan 2020 14:14:07 -0500 Subject: [PATCH 17/30] Add the possibility to return 2 outputs on the create graph function : training output(fit), inference output(transform) --- neuraxle_tensorflow/tensorflow_v1.py | 39 ++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 8cc7801..4d261ee 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -42,6 +42,7 @@ def __init__( create_graph, create_loss, create_optimizer, + create_feed_dict=None, variable_scope=None, has_expected_outputs=True, ): @@ -57,6 +58,7 @@ def __init__( variable_scope = self.name self.variable_scope = variable_scope self.has_expected_outputs = has_expected_outputs + self.create_feed_dict = create_feed_dict def setup(self) -> BaseStep: """ @@ -73,7 +75,13 @@ def setup(self) -> BaseStep: with tf.variable_scope(self.variable_scope, reuse=tf.AUTO_REUSE): self.session = tf.Session(config=tf.ConfigProto(log_device_placement=True), graph=self.graph) - tf.identity(self.create_model(self), name='output') + model = self.create_model(self) + if not isinstance(model, tuple): + tf.identity(model, name='output') + else: + tf.identity(model[0], name='output') + tf.identity(model[1], name='inference_output') + tf.identity(self.create_loss(self), name='loss') self.create_optimizer(self).minimize(self['loss'], name='optimizer') @@ -127,6 +135,10 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: self['expected_outputs']: expected_outputs }) + if self.create_feed_dict is not None: + additional_feed_dict_arguments = self.create_feed_dict(self, data_inputs, expected_outputs) + feed_dict.update(additional_feed_dict_arguments) + results = self.session.run([self['optimizer'], self['loss']], feed_dict=feed_dict) self.loss = results[1] @@ -143,12 +155,17 @@ def transform_model(self, data_inputs): :param data_inputs: :return: """ - return self.session.run( - self['output'], - feed_dict={ - self['data_inputs']: data_inputs - } - ) + inference_output_name = 'output' + if len(self['inference_output'].get_shape().as_list()) > 0: + inference_output_name = 'inference_output' + + feed_dict = { + self['data_inputs']: data_inputs + } + + results = self.session.run(self[inference_output_name], feed_dict=feed_dict) + + return results def __getitem__(self, item): """ @@ -171,7 +188,13 @@ def __getitem__(self, item): try: result = self.graph.get_tensor_by_name("{0}/{1}:{2}".format(self.variable_scope, tensor_name, device)) except KeyError: - result = self.graph.get_operation_by_name("{0}/{1}".format(self.variable_scope, tensor_name)) + result = None + + if result is None: + try: + result = self.graph.get_operation_by_name("{0}/{1}".format(self.variable_scope, tensor_name)) + except KeyError: + result = tf.get_variable(tensor_name, []) return result From d562cc9aebb7df12a3bd4722635f77acb281a922 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sat, 11 Jan 2020 16:04:52 -0500 Subject: [PATCH 18/30] Save Loss array in Tensorflow v1 Model step, And Add Print Loss func --- neuraxle_tensorflow/tensorflow_v1.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 4d261ee..397578f 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -45,6 +45,8 @@ def __init__( create_feed_dict=None, variable_scope=None, has_expected_outputs=True, + print_loss=False, + print_func=None ): BaseTensorflowModelStep.__init__( self, @@ -59,6 +61,11 @@ def __init__( self.variable_scope = variable_scope self.has_expected_outputs = has_expected_outputs self.create_feed_dict = create_feed_dict + self.loss = [] + self.print_loss = print_loss + if print_func is None: + print_func = print + self.print_func = print_func def setup(self) -> BaseStep: """ @@ -140,7 +147,9 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: feed_dict.update(additional_feed_dict_arguments) results = self.session.run([self['optimizer'], self['loss']], feed_dict=feed_dict) - self.loss = results[1] + self.loss.append(results[1]) + if self.print_loss: + self.print_func(self.loss[-1]) return self From 2333f12a1bcb9704d3fec21e8e35354c6e0e1519 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 12 Jan 2020 20:17:27 -0500 Subject: [PATCH 19/30] Apply Review Comments #5 --- README.md | 19 +++++++++++++++---- neuraxle_tensorflow/tensorflow_v1.py | 23 +++++++++++++++++------ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f34c727..01d51c7 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,19 @@ def create_graph(step: TensorflowV1ModelStep): tf.Variable(np.random.rand(), name='weight') tf.Variable(np.random.rand(), name='bias') + + return tf.add(tf.multiply(step['data_inputs'], step['weight']), step['bias']) + +""" +# Note: you can also return a tuple containing two elements : tensor for training (fit), tensor for inference (transform) +def create_graph(step: TensorflowV1ModelStep) + # ... + decoder_outputs_training = create_training_decoder(step, encoder_state, decoder_cell) + decoder_outputs_inference = create_inference_decoder(step, encoder_state, decoder_cell) + + return decoder_outputs_training, decoder_outputs_inference +""" - tf.add(tf.multiply(step['data_inputs'], step['weight']), step['bias'], name='output') def create_loss(step: TensorflowV1ModelStep): return tf.reduce_sum(tf.pow(step['output'] - step['expected_outputs'], 2)) / (2 * N_SAMPLES) @@ -32,7 +43,7 @@ model_step = TensorflowV1ModelStep( create_grah=create_graph, create_loss=create_loss, create_optimizer=create_optimizer, - has_expected_outputs=False + has_expected_outputs=True ).set_hyperparams(HyperparameterSamples({ 'learning_rate': 0.01 })).set_hyperparams_space(HyperparameterSpace({ @@ -51,8 +62,8 @@ def create_model(step: Tensorflow2ModelStep): def create_optimizer(step: Tensorflow2ModelStep): return tf.keras.optimizers.Adam(0.1) -def create_loss(step: Tensorflow2ModelStep, expected_outputs, actual_outputs): - return tf.reduce_mean(tf.abs(actual_outputs - expected_outputs)) +def create_loss(step: Tensorflow2ModelStep, expected_outputs, predicted_outputs): + return tf.reduce_mean(tf.abs(predicted_outputs - expected_outputs)) model_step = Tensorflow2ModelStep( create_model=create_model, diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 397578f..2181cf8 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -61,7 +61,7 @@ def __init__( self.variable_scope = variable_scope self.has_expected_outputs = has_expected_outputs self.create_feed_dict = create_feed_dict - self.loss = [] + self.losses = [] self.print_loss = print_loss if print_func is None: print_func = print @@ -147,9 +147,9 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: feed_dict.update(additional_feed_dict_arguments) results = self.session.run([self['optimizer'], self['loss']], feed_dict=feed_dict) - self.loss.append(results[1]) + self.losses.append(results[1]) if self.print_loss: - self.print_func(self.loss[-1]) + self.print_func(self.losses[-1]) return self @@ -164,9 +164,7 @@ def transform_model(self, data_inputs): :param data_inputs: :return: """ - inference_output_name = 'output' - if len(self['inference_output'].get_shape().as_list()) > 0: - inference_output_name = 'inference_output' + inference_output_name = self._get_inference_output_name() feed_dict = { self['data_inputs']: data_inputs @@ -176,6 +174,19 @@ def transform_model(self, data_inputs): return results + def _get_inference_output_name(self): + """ + Return the output tensor name for inference (transform). + In create_graph, the user can return a tuple of two elements : the output tensor for training, and the output tensor for inference. + + :return: + """ + inference_output_name = 'output' + if len(self['inference_output'].get_shape().as_list()) > 0: + inference_output_name = 'inference_output' + + return inference_output_name + def __getitem__(self, item): """ Get a graph tensor by name using get item. From 6d4f34ef9fb684eab44885f58858b0e9ff0d514c Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 12 Jan 2020 20:24:03 -0500 Subject: [PATCH 20/30] Apply Review Comments #5 --- requirements.txt | 2 +- setup.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7a84c38..d0cb604 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Neuraxle \ No newline at end of file +neuraxle>=0.3.0 \ No newline at end of file diff --git a/setup.py b/setup.py index a460992..7a47461 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ from setuptools import setup, find_packages -from neuraxle import __version__ as _VERSION +from neuraxle_tensorflow import __version__ as _VERSION with open('README.md') as _f: _README = _f.read() @@ -29,6 +29,7 @@ version=_VERSION, description='TensorFlow steps, savers, and utilities for Neuraxle. Neuraxle is a Machine Learning (ML) library for building neat pipelines, providing the right ' 'abstractions to both ease research, development, and deployment of your ML applications.', + long_description_content_type='text/markdown', long_description=_README, classifiers=[ "Development Status :: 3 - Alpha", @@ -79,16 +80,16 @@ "Topic :: Utilities", "Typing :: Typed" ], - url='https://github.com/Neuraxio/Neuraxle', - download_url='https://github.com/Neuraxio/Neuraxle/tarball/{}'.format( + url='https://github.com/Neuraxio/Neuraxle-Tensorflow', + download_url='https://github.com/Neuraxio/Neuraxle-Tensorflow/tarball/{}'.format( _VERSION), author='Neuraxio Inc.', author_email='guillaume.chevalier@neuraxio.com', - packages=find_packages(include=['neuraxle-tensorflow*']), + packages=find_packages(include=['neuraxle_tensorflow*']), test_suite="testing", setup_requires=["pytest-runner"], install_requires=[ - 'neuraxle>=0.2.1' + 'neuraxle>=0.3.0' ], tests_require=["pytest", "pytest-cov"], include_package_data=True, From 50bcd7d385049abefb291b78deae3c52d10c6efd Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Sun, 12 Jan 2020 20:27:45 -0500 Subject: [PATCH 21/30] Fix tensorflow v1 create graph return in test --- testing/test_tensorflow_v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_tensorflow_v1.py b/testing/test_tensorflow_v1.py index 9ede8a8..503100b 100644 --- a/testing/test_tensorflow_v1.py +++ b/testing/test_tensorflow_v1.py @@ -19,7 +19,7 @@ def create_graph(step: TensorflowV1ModelStep): tf.Variable(np.random.rand(), name='weight') tf.Variable(np.random.rand(), name='bias') - tf.add(tf.multiply(step['data_inputs'], step['weight']), step['bias'], name='output') + return tf.add(tf.multiply(step['data_inputs'], step['weight']), step['bias']) def create_loss(step: TensorflowV1ModelStep): From 40f156d1e858186f41472c810736b7b3170fc7cf Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:00:12 -0500 Subject: [PATCH 22/30] Add create_inputs to tensorflow models, and add data_inputs, expected_outputs tensor dtype --- neuraxle_tensorflow/tensorflow.py | 15 ++++++++++++- neuraxle_tensorflow/tensorflow_v1.py | 12 ++++++++--- neuraxle_tensorflow/tensorflow_v2.py | 32 ++++++++++++++++++++++------ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow.py b/neuraxle_tensorflow/tensorflow.py index 00e2e9d..beb2fee 100644 --- a/neuraxle_tensorflow/tensorflow.py +++ b/neuraxle_tensorflow/tensorflow.py @@ -2,11 +2,24 @@ class BaseTensorflowModelStep(BaseStep): - def __init__(self, create_model, create_loss, create_optimizer, step_saver): + def __init__( + self, + create_model, + create_loss, + create_optimizer, + step_saver, + create_inputs=None, + data_inputs_dtype=None, + expected_outputs_dtype=None + ): + self.create_inputs = create_inputs self.create_model = create_model self.create_loss = create_loss self.create_optimizer = create_optimizer + self.expected_outputs_dtype = expected_outputs_dtype + self.data_inputs_dtype = data_inputs_dtype + self.set_hyperparams(self.__class__.HYPERPARAMS) self.set_hyperparams_space(self.__class__.HYPERPARAMS_SPACE) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 2181cf8..626502d 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -43,6 +43,8 @@ def __init__( create_loss, create_optimizer, create_feed_dict=None, + data_inputs_dtype=None, + expected_outputs_dtype=None, variable_scope=None, has_expected_outputs=True, print_loss=False, @@ -53,6 +55,9 @@ def __init__( create_model=create_graph, create_loss=create_loss, create_optimizer=create_optimizer, + create_inputs=create_feed_dict, + data_inputs_dtype=data_inputs_dtype, + expected_outputs_dtype=expected_outputs_dtype, step_saver=TensorflowV1StepSaver() ) @@ -61,6 +66,7 @@ def __init__( self.variable_scope = variable_scope self.has_expected_outputs = has_expected_outputs self.create_feed_dict = create_feed_dict + self.losses = [] self.print_loss = print_loss if print_func is None: @@ -142,14 +148,14 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: self['expected_outputs']: expected_outputs }) - if self.create_feed_dict is not None: - additional_feed_dict_arguments = self.create_feed_dict(self, data_inputs, expected_outputs) + if self.create_inputs is not None: + additional_feed_dict_arguments = self.create_inputs(self, data_inputs, expected_outputs) feed_dict.update(additional_feed_dict_arguments) results = self.session.run([self['optimizer'], self['loss']], feed_dict=feed_dict) self.losses.append(results[1]) if self.print_loss: - self.print_func(self.losses[-1]) + self.print_func('Loss: {}'.format(self.losses[-1])) return self diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index ee75aef..588d539 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -16,7 +16,6 @@ limitations under the License. """ - import tensorflow as tf from neuraxle.base import BaseSaver, BaseStep, ExecutionContext from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace @@ -41,6 +40,9 @@ def __init__( create_model, create_loss, create_optimizer, + create_inputs=None, + data_inputs_dtype=None, + expected_outputs_dtype=None, tf_model_checkpoint_folder=None ): BaseTensorflowModelStep.__init__( @@ -48,12 +50,16 @@ def __init__( create_model=create_model, create_loss=create_loss, create_optimizer=create_optimizer, + create_inputs=create_inputs, + data_inputs_dtype=data_inputs_dtype, + expected_outputs_dtype=expected_outputs_dtype, step_saver=TensorflowV2StepSaver() ) if tf_model_checkpoint_folder is None: tf_model_checkpoint_folder = 'tensorflow_ckpts' self.tf_model_checkpoint_folder = tf_model_checkpoint_folder + self.losses = [] def setup(self) -> BaseStep: """ @@ -91,22 +97,34 @@ def strip(self): self.checkpoint_manager = None def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': - x = tf.convert_to_tensor(data_inputs) - y = tf.convert_to_tensor(expected_outputs) + inputs = self._create_inputs(data_inputs, expected_outputs) with tf.GradientTape() as tape: - output = self.model(x) - self.loss = self.create_loss(self, y, output) + output = self.model(inputs, training=True) + loss = self.create_loss( + self, + expected_outputs=tf.convert_to_tensor(expected_outputs, dtype=self.expected_outputs_dtype), + predicted_outputs=output + ) + self.losses.append(loss) + self.model.losses.append(loss) self.optimizer.apply_gradients(zip( - tape.gradient(self.loss, self.model.trainable_variables), + tape.gradient(loss, self.model.trainable_variables), self.model.trainable_variables )) return self def transform(self, data_inputs): - return self.model(tf.convert_to_tensor(data_inputs)).numpy() + return self.model(self._create_inputs(data_inputs), training=False).numpy() + + def _create_inputs(self, data_inputs, expected_outputs=None): + if self.create_inputs is not None: + inputs = self.create_inputs(self, data_inputs, expected_outputs) + else: + inputs = tf.convert_to_tensor(data_inputs, self.data_inputs_dtype) + return inputs class TensorflowV2StepSaver(BaseSaver): From ffbf7496b91d0a9146414571b006e44f4ffb2954 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:11:08 -0500 Subject: [PATCH 23/30] Add Train Loss, And Test Loss To BaseTensorflow Model --- neuraxle_tensorflow/tensorflow.py | 20 +++++++++++++++++++- neuraxle_tensorflow/tensorflow_v1.py | 16 ++++++---------- neuraxle_tensorflow/tensorflow_v2.py | 12 ++++++++---- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow.py b/neuraxle_tensorflow/tensorflow.py index beb2fee..3dd73ef 100644 --- a/neuraxle_tensorflow/tensorflow.py +++ b/neuraxle_tensorflow/tensorflow.py @@ -10,7 +10,9 @@ def __init__( step_saver, create_inputs=None, data_inputs_dtype=None, - expected_outputs_dtype=None + expected_outputs_dtype=None, + print_loss=False, + print_func=None ): self.create_inputs = create_inputs self.create_model = create_model @@ -23,8 +25,24 @@ def __init__( self.set_hyperparams(self.__class__.HYPERPARAMS) self.set_hyperparams_space(self.__class__.HYPERPARAMS_SPACE) + self.train_losses = [] + self.test_losses = [] + self.print_loss = print_loss + if print_func is None: + print_func = print + self.print_func = print_func + BaseStep.__init__( self, savers=[step_saver], hyperparams=self.HYPERPARAMS ) + + def add_new_loss(self, loss): + if self.is_train: + self.train_losses.append(loss) + else: + self.test_losses.append(loss) + + if self.print_loss: + self.print_func('{} Loss: {}'.format('Train' if self.is_train else 'Test', loss)) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 626502d..66672c8 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -58,7 +58,9 @@ def __init__( create_inputs=create_feed_dict, data_inputs_dtype=data_inputs_dtype, expected_outputs_dtype=expected_outputs_dtype, - step_saver=TensorflowV1StepSaver() + step_saver=TensorflowV1StepSaver(), + print_loss=print_loss, + print_func=print_func ) if variable_scope is None: @@ -67,12 +69,6 @@ def __init__( self.has_expected_outputs = has_expected_outputs self.create_feed_dict = create_feed_dict - self.losses = [] - self.print_loss = print_loss - if print_func is None: - print_func = print - self.print_func = print_func - def setup(self) -> BaseStep: """ Setup tensorflow 1 graph, and session using a variable scope. @@ -153,9 +149,9 @@ def fit_model(self, data_inputs, expected_outputs=None) -> BaseStep: feed_dict.update(additional_feed_dict_arguments) results = self.session.run([self['optimizer'], self['loss']], feed_dict=feed_dict) - self.losses.append(results[1]) - if self.print_loss: - self.print_func('Loss: {}'.format(self.losses[-1])) + + loss = results[1] + self.add_new_loss(loss) return self diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 588d539..af8419a 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -43,7 +43,9 @@ def __init__( create_inputs=None, data_inputs_dtype=None, expected_outputs_dtype=None, - tf_model_checkpoint_folder=None + tf_model_checkpoint_folder=None, + print_loss=False, + print_func=None ): BaseTensorflowModelStep.__init__( self, @@ -53,13 +55,14 @@ def __init__( create_inputs=create_inputs, data_inputs_dtype=data_inputs_dtype, expected_outputs_dtype=expected_outputs_dtype, - step_saver=TensorflowV2StepSaver() + step_saver=TensorflowV2StepSaver(), + print_loss=print_loss, + print_func=print_func ) if tf_model_checkpoint_folder is None: tf_model_checkpoint_folder = 'tensorflow_ckpts' self.tf_model_checkpoint_folder = tf_model_checkpoint_folder - self.losses = [] def setup(self) -> BaseStep: """ @@ -106,7 +109,8 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': expected_outputs=tf.convert_to_tensor(expected_outputs, dtype=self.expected_outputs_dtype), predicted_outputs=output ) - self.losses.append(loss) + + self.add_new_loss(loss) self.model.losses.append(loss) self.optimizer.apply_gradients(zip( From 1c68f89f680b607182b6768f4e6f9e5445eeb595 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:15:50 -0500 Subject: [PATCH 24/30] Remove Dead Code --- neuraxle_tensorflow/tensorflow_v2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index af8419a..dcb4e25 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -111,7 +111,6 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': ) self.add_new_loss(loss) - self.model.losses.append(loss) self.optimizer.apply_gradients(zip( tape.gradient(loss, self.model.trainable_variables), From 7824c5d4759d62beeb337c74d7681d9d00b45fcf Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:33:17 -0500 Subject: [PATCH 25/30] Fix Test Losses --- neuraxle_tensorflow/tensorflow.py | 10 +++++++--- neuraxle_tensorflow/tensorflow_v1.py | 5 +++-- neuraxle_tensorflow/tensorflow_v2.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow.py b/neuraxle_tensorflow/tensorflow.py index 3dd73ef..9c4cbee 100644 --- a/neuraxle_tensorflow/tensorflow.py +++ b/neuraxle_tensorflow/tensorflow.py @@ -38,11 +38,15 @@ def __init__( hyperparams=self.HYPERPARAMS ) - def add_new_loss(self, loss): + def add_new_loss(self, loss, test_only=False): + if test_only: + if not self.is_train: + self.test_losses.append(loss) + else: + return + if self.is_train: self.train_losses.append(loss) - else: - self.test_losses.append(loss) if self.print_loss: self.print_func('{} Loss: {}'.format('Train' if self.is_train else 'Test', loss)) diff --git a/neuraxle_tensorflow/tensorflow_v1.py b/neuraxle_tensorflow/tensorflow_v1.py index 66672c8..ee23ae2 100644 --- a/neuraxle_tensorflow/tensorflow_v1.py +++ b/neuraxle_tensorflow/tensorflow_v1.py @@ -172,9 +172,10 @@ def transform_model(self, data_inputs): self['data_inputs']: data_inputs } - results = self.session.run(self[inference_output_name], feed_dict=feed_dict) + results = self.session.run([self[inference_output_name], self['loss']], feed_dict=feed_dict) + self.add_new_loss(results[1], test_only=True) - return results + return results[0] def _get_inference_output_name(self): """ diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index dcb4e25..4920d89 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -18,6 +18,7 @@ """ import tensorflow as tf from neuraxle.base import BaseSaver, BaseStep, ExecutionContext +from neuraxle.data_container import DataContainer from neuraxle.hyperparams.space import HyperparameterSamples, HyperparameterSpace from neuraxle_tensorflow.tensorflow import BaseTensorflowModelStep @@ -111,6 +112,7 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': ) self.add_new_loss(loss) + self.model.losses.append(loss) self.optimizer.apply_gradients(zip( tape.gradient(loss, self.model.trainable_variables), @@ -119,6 +121,17 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': return self + def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: + output = self.model(data_container.data_inputs, training=False) + loss = self.create_loss( + self, + expected_outputs=tf.convert_to_tensor(data_container.expected_outputs, dtype=self.expected_outputs_dtype), + predicted_outputs=output + ) + self.add_new_loss(loss, test_only=True) + + return output.numpy() + def transform(self, data_inputs): return self.model(self._create_inputs(data_inputs), training=False).numpy() From 7d26a911828097fbd771fb9a49501007596c4065 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:36:23 -0500 Subject: [PATCH 26/30] Fix Transform Data Container For Tensorflow 2 --- neuraxle_tensorflow/tensorflow_v2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 4920d89..f3797ff 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -122,7 +122,7 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': return self def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: - output = self.model(data_container.data_inputs, training=False) + output = self.model(self._create_inputs(data_container.data_inputs), training=False) loss = self.create_loss( self, expected_outputs=tf.convert_to_tensor(data_container.expected_outputs, dtype=self.expected_outputs_dtype), @@ -132,9 +132,6 @@ def _transform_data_container(self, data_container: DataContainer, context: Exec return output.numpy() - def transform(self, data_inputs): - return self.model(self._create_inputs(data_inputs), training=False).numpy() - def _create_inputs(self, data_inputs, expected_outputs=None): if self.create_inputs is not None: inputs = self.create_inputs(self, data_inputs, expected_outputs) From 7db07716b9d394a6ab6545bddf5b1b7efecd7eda Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:39:04 -0500 Subject: [PATCH 27/30] Fix Tensorflow 2 Transform --- neuraxle_tensorflow/tensorflow_v2.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index f3797ff..a7e3927 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -123,13 +123,19 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: output = self.model(self._create_inputs(data_container.data_inputs), training=False) - loss = self.create_loss( - self, - expected_outputs=tf.convert_to_tensor(data_container.expected_outputs, dtype=self.expected_outputs_dtype), - predicted_outputs=output - ) - self.add_new_loss(loss, test_only=True) + if data_container.expected_outputs is not None: + loss = self.create_loss( + self, + expected_outputs=tf.convert_to_tensor(data_container.expected_outputs, dtype=self.expected_outputs_dtype), + predicted_outputs=output + ) + self.add_new_loss(loss, test_only=True) + + return output.numpy() + + def transform(self, data_inputs): + output = self.model(self._create_inputs(data_inputs), training=False) return output.numpy() def _create_inputs(self, data_inputs, expected_outputs=None): From 4864bb99d29b21fc7b06aa03f7f765ad79747dd5 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:41:19 -0500 Subject: [PATCH 28/30] Fix Transform Data Container In Tensorflow 2 --- neuraxle_tensorflow/tensorflow_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index a7e3927..3dd79eb 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -132,7 +132,7 @@ def _transform_data_container(self, data_container: DataContainer, context: Exec ) self.add_new_loss(loss, test_only=True) - return output.numpy() + return data_container.set_data_inputs(output.numpy()) def transform(self, data_inputs): output = self.model(self._create_inputs(data_inputs), training=False) From d41e46ef8bd1d63b6abff96ba7a147800a8a1d03 Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Mon, 13 Jan 2020 19:42:42 -0500 Subject: [PATCH 29/30] Fit Transform Data Container In Tensorflow 2 --- neuraxle_tensorflow/tensorflow_v2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index 3dd79eb..e8fce53 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -132,7 +132,8 @@ def _transform_data_container(self, data_container: DataContainer, context: Exec ) self.add_new_loss(loss, test_only=True) - return data_container.set_data_inputs(output.numpy()) + data_container.set_data_inputs(output.numpy()) + return data_container def transform(self, data_inputs): output = self.model(self._create_inputs(data_inputs), training=False) From 1cbbdd1262e1413e10789bcc4f9613fdec0b4d6f Mon Sep 17 00:00:00 2001 From: alexbrillant Date: Wed, 15 Jan 2020 10:38:32 -0500 Subject: [PATCH 30/30] Add device support in tensorflow 2 (GPU, CPU, etc.) --- neuraxle_tensorflow/tensorflow_v2.py | 57 ++++++++++++++++++---------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/neuraxle_tensorflow/tensorflow_v2.py b/neuraxle_tensorflow/tensorflow_v2.py index e8fce53..8df4e7f 100644 --- a/neuraxle_tensorflow/tensorflow_v2.py +++ b/neuraxle_tensorflow/tensorflow_v2.py @@ -46,7 +46,8 @@ def __init__( expected_outputs_dtype=None, tf_model_checkpoint_folder=None, print_loss=False, - print_func=None + print_func=None, + device_name=None ): BaseTensorflowModelStep.__init__( self, @@ -61,6 +62,10 @@ def __init__( print_func=print_func ) + if device_name is None: + device_name = '/CPU:0' + self.device_name = device_name + if tf_model_checkpoint_folder is None: tf_model_checkpoint_folder = 'tensorflow_ckpts' self.tf_model_checkpoint_folder = tf_model_checkpoint_folder @@ -75,15 +80,16 @@ def setup(self) -> BaseStep: if self.is_initialized: return self - self.optimizer = self.create_optimizer(self) - self.model = self.create_model(self) + with tf.device(self.device_name): + self.optimizer = self.create_optimizer(self) + self.model = self.create_model(self) - self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) - self.checkpoint_manager = tf.train.CheckpointManager( - self.checkpoint, - self.tf_model_checkpoint_folder, - max_to_keep=3 - ) + self.checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self.model) + self.checkpoint_manager = tf.train.CheckpointManager( + self.checkpoint, + self.tf_model_checkpoint_folder, + max_to_keep=3 + ) self.is_initialized = True @@ -101,8 +107,13 @@ def strip(self): self.checkpoint_manager = None def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': - inputs = self._create_inputs(data_inputs, expected_outputs) + with tf.device(self.device_name): + self._fit_model(data_inputs, expected_outputs) + + return self + def _fit_model(self, data_inputs, expected_outputs): + inputs = self._create_inputs(data_inputs, expected_outputs) with tf.GradientTape() as tape: output = self.model(inputs, training=True) loss = self.create_loss( @@ -110,7 +121,6 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': expected_outputs=tf.convert_to_tensor(expected_outputs, dtype=self.expected_outputs_dtype), predicted_outputs=output ) - self.add_new_loss(loss) self.model.losses.append(loss) @@ -119,24 +129,31 @@ def fit(self, data_inputs, expected_outputs=None) -> 'BaseStep': self.model.trainable_variables )) - return self - def _transform_data_container(self, data_container: DataContainer, context: ExecutionContext) -> DataContainer: - output = self.model(self._create_inputs(data_container.data_inputs), training=False) + data_inputs = data_container.data_inputs + expected_outputs = data_container.data_inputs + + with tf.device(self.device_name): + output = self._transform_model(data_inputs, expected_outputs) - if data_container.expected_outputs is not None: + data_container.set_data_inputs(output.numpy()) + return data_container + + def _transform_model(self, data_inputs, expected_outputs): + output = self.model(self._create_inputs(data_inputs), training=False) + + if expected_outputs is not None: loss = self.create_loss( self, - expected_outputs=tf.convert_to_tensor(data_container.expected_outputs, dtype=self.expected_outputs_dtype), + expected_outputs=tf.convert_to_tensor(expected_outputs, dtype=self.expected_outputs_dtype), predicted_outputs=output ) self.add_new_loss(loss, test_only=True) - - data_container.set_data_inputs(output.numpy()) - return data_container + return output def transform(self, data_inputs): - output = self.model(self._create_inputs(data_inputs), training=False) + with tf.device(self.device_name): + output = self.model(self._create_inputs(data_inputs), training=False) return output.numpy() def _create_inputs(self, data_inputs, expected_outputs=None):