Implement Simple RNN Layer and Demo (#42) (#43)

* Implement Simple RNN Layer and Demo (#42)

* implement dropout layer and fix pbm inspector

* start demo rnn

* complete linear rnn demo

* update eigen

* add nonlinear rnn demo

* fix concat

* add concat grader

* fix nonlin rnn_demo not converging

* enhance matmul with contract

* fix and complete contract implementation

* use contraction to speedup input and output rnn

* update both rnn demo to use the best weights

* improve rnn loss

* remove sigmoid grad

* fix convolution operator

* fix session

* update rnn demo

* use eteq derivatives instead of backward connect in rnn

* start using layr dense

* make funcarg pairvecs readable

* make dense contract dimensions parameterized at layr level

* implement recurrent layers and update rnn_demo to use new layers

* clean up rbm demos

* update-todo

* cleanup

* generate coverage in the first travis test

* increase bazel jobs to 3

* split eteq and remove useless dependencies

* move session to teq and slowly remove eteq dependencies

* reduce warnings

* organize tests

.bazelrc

@@ -1,10 +1,10 @@
 
 # ------ ALL SETTINGS ------
 
-run --incompatible_depset_is_not_iterable=false --jobs=2
-build --incompatible_depset_is_not_iterable=false --jobs=2
-test --incompatible_depset_is_not_iterable=false --jobs=2
-coverage --incompatible_depset_is_not_iterable=false --jobs=2
+run --incompatible_depset_is_not_iterable=false --jobs=3
+build --incompatible_depset_is_not_iterable=false --jobs=3
+test --incompatible_depset_is_not_iterable=false --jobs=3
+coverage --incompatible_depset_is_not_iterable=false --jobs=3
 
 # ------ OPTIMIZATION ------
 

Dockerfile

@@ -10,4 +10,4 @@ WORKDIR $APP_DIR
 COPY . $APP_DIR
 RUN pip3 install -r requirements.txt
 
-CMD [ "./tests.sh" ]
+ENTRYPOINT [ "./tests.sh" ]

Doxyfile

@@ -791,7 +791,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = README.md ccur dbg eteq layr opt pbm tag teq
+INPUT                  = README.md ccur dbg eigen eteq layr opt pbm pyutils tag teq
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

Makefile

@@ -2,7 +2,7 @@ COVERAGE_INFO_FILE := bazel-out/_coverage/_coverage_report.dat
 
 COVERAGE_IGNORE := 'external/*' '**/test/*' 'testutil/*' '**/genfiles/*' 'dbg/*'
 
-CCOVER := bazel coverage --config asan --action_env="ASAN_OPTIONS=detect_leaks=0" --config gtest --config cc_coverage --define ETEQ_CFG=MIN
+CCOVER := bazel coverage --config asan --action_env="ASAN_OPTIONS=detect_leaks=0" --config gtest --config cc_coverage
 
 CCUR_TEST := //ccur:test
 
@@ -18,7 +18,7 @@ TAG_TEST := //tag:test
 
 TEQ_TEST := //teq:test
 
-CC := gcc
+CC := clang
 
 COVERAGE_PIPE := ./bazel-bin/external/com_github_mingkaic_cppkg/merge_cov $(COVERAGE_INFO_FILE)
 
@@ -35,25 +35,48 @@ rocnnet_py_export: bazel-bin/rocnnet/rocnnet.so bazel-bin/eteq/tenncor.so bazel-
 	cp -f bazel-bin/eteq/*.so rocnnet/notebooks/eteq
 
 
+model_jsonupdate: model_jsongd model_jsondqn model_jsonrbm model_jsondbn
+
+model_jsongd: models/gdmodel.pbx
+	bazel run //pbm:inspector -- --read ${CURDIR}/models/gdmodel.pbx --write /tmp/gdmodel.json
+	mv /tmp/gdmodel.json models
+
+model_jsondqn: models/dqnmodel.pbx
+	bazel run //pbm:inspector -- --read ${CURDIR}/models/dqnmodel.pbx --write /tmp/dqnmodel.json
+	mv /tmp/dqnmodel.json models
+
+model_jsonrbm: models/rbmmodel.pbx
+	bazel run //pbm:inspector -- --read ${CURDIR}/models/rbmmodel.pbx --write /tmp/rbmmodel.json
+	mv /tmp/rbmmodel.json models
+
+model_jsondbn: models/dbnmodel.pbx
+	bazel run //pbm:inspector -- --read ${CURDIR}/models/dbnmodel.pbx --write /tmp/dbnmodel.json
+	mv /tmp/dbnmodel.json models
+
+model_jsonrnn: models/rnnmodel.pbx
+	bazel run //pbm:inspector -- --read ${CURDIR}/models/rnnmodel.pbx --write /tmp/rnnmodel.json
+	mv /tmp/rnnmodel.json models
+
+
 coverage:
 	$(CCOVER) $(TEQ_TEST) $(TAG_TEST) $(PBM_TEST) $(OPT_TEST) $(ETEQ_CTEST) $(CCUR_TEST) $(LAYR_TEST)
 	lcov --remove $(COVERAGE_INFO_FILE) -o coverage.info
 
 cover_ccur:
 	$(CCOVER) $(CCUR_TEST)
-	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'opt/*' 'eteq/*' -o coverage.info
+	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'opt/*' 'eigen/*' 'eteq/*' -o coverage.info
 
 cover_eteq:
 	$(CCOVER) $(ETEQ_CTEST)
 	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'opt/*' -o coverage.info
 
 cover_layr:
 	$(CCOVER) $(LAYR_TEST)
-	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'opt/*' 'eteq/*' -o coverage.info
+	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'opt/*' 'eigen/*' 'eteq/*' -o coverage.info
 
 cover_opt:
 	$(CCOVER) $(OPT_TEST)
-	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'eteq/*' -o coverage.info
+	lcov --remove $(COVERAGE_INFO_FILE) 'teq/*' 'tag/*' 'eigen/*' 'eteq/*' -o coverage.info
 
 cover_pbm:
 	$(CCOVER) $(PBM_TEST)
@@ -70,13 +93,13 @@ cover_teq:
 
 # optimized comparisons
 compare_matmul:
-	bazel run $(EIGEN_OPT) //rocnnet:comparison_matmul
+	bazel run --config $(CC)_eigen_optimal //rocnnet:comparison_matmul
 
 compare_mlp:
-	bazel run $(EIGEN_OPT) //rocnnet:comparison_mlp
+	bazel run --config $(CC)_eigen_optimal //rocnnet:comparison_mlp
 
 compare_mlp_grad:
-	bazel run $(EIGEN_OPT) //rocnnet:comparison_mlp_grad
+	bazel run --config $(CC)_eigen_optimal //rocnnet:comparison_mlp_grad
 
 
 cov_clean: coverage.info
@@ -107,3 +130,4 @@ lcov_teq: cover_teq cov_clean
 .PHONY: coverage cover_ccur cover_eteq cover_layr cover_opt cover_pbm cover_tag cover_teq
 .PHONY: compare_matmul compare_mlp compare_mlp_grad cov_clean cov_genhtml
 .PHONY: lcov lcov_ccur lcov_eteq lcov_layr lcov_opt lcov_pbm lcov_tag lcov_teq
+.PHONY: model_jsonupdate model_jsongd model_jsondqn model_jsonrbm model_jsondbn model_jsonrnn

README.md

@@ -20,6 +20,10 @@ Constraints to the equation is limited to each tensor's shape.
 
 This module is contains debug libraries for TEQ Graphs.
 
+- [EIGEN (EIGEN wrapper)](eigen/README_EIGEN.md)
+
+This module wraps eigen operators using TEQ shape and coordinate arguments.
+
 - [ETEQ (Eigen TEQ)](eteq/README_ETEQ.md)
 
 This module is implements basic operations for Tenncor's TEQ Tensor objects generated through pybinder.

ccur/BUILD.bazel

@@ -51,15 +51,18 @@ cc_library(
     srcs = glob(["src/*.cpp"]),
     copts = ["-std=c++17"],
     deps = [
-        "//eteq:eteq",
+        "//teq:teq",
         "@boost//:asio",
     ],
 )
 
 pybind_library(
     name = "ccur_py",
     cc_srcs = ["//ccur:python/ccur.cpp"],
-    cc_deps = ["//ccur:ccur"],
+    cc_deps = [
+        "//ccur:ccur",
+        "//opt:opt",
+    ],
     py_deps = ["//eteq:eteq_py"],
     visibility = ["//visibility:public"],
 )
@@ -96,6 +99,7 @@ cc_test(
     deps = [
         "//dbg:stream_out",
         "//ccur:ccur",
+        "//eteq:eteq",
         "@gtest//:gtest",
         "@com_github_mingkaic_cppkg//exam:exam",
     ],

ccur/python/ccur.cpp

@@ -2,8 +2,7 @@
 #include "pybind11/numpy.h"
 #include "pybind11/stl.h"
 
-#include "eteq/generated/pyapi.hpp"
-#include "eteq/parse.hpp"
+#include "opt/optimize.hpp"
 
 #include "ccur/session.hpp"
 
@@ -14,21 +13,19 @@ PYBIND11_MODULE(ccur, m)
 	m.doc() = "ccur session";
 
 	// ==== session ====
-	auto isess = (py::class_<eteq::iSession>)
+	auto isess = (py::class_<teq::iSession>)
 		py::module::import("eteq.eteq").attr("iSession");
 	py::class_<ccur::Session> session(m, "Session", isess);
 
-	py::implicitly_convertible<eteq::iSession,ccur::Session>();
+	py::implicitly_convertible<teq::iSession,ccur::Session>();
 	session
 		.def(py::init<int,ccur::OpWeightT>(),
 			py::arg("nthread") = 2,
 			py::arg("weights") = ccur::OpWeightT())
 		.def("optimize",
-			[](py::object self, std::string filename)
+			[](ccur::Session* self, opt::OptCtx rules)
 			{
-				auto sess = self.cast<ccur::Session*>();
-				opt::OptCtx rules = eteq::parse_file<PybindT>(filename);
-				sess->optimize(rules);
+				opt::optimize(*self, rules);
 			},
 			py::arg("filename") = "cfg/optimizations.rules",
 			"Optimize using rules for specified filename");

ccur/rtscale.cpp

@@ -61,7 +61,6 @@ int main (int argc, const char** argv)
 			case egen::SQRT:
 			case egen::ROUND:
 			case egen::SIGMOID:
-			case egen::SIGMOID_GRAD:
 			case egen::TANH:
 			case egen::SQUARE:
 			case egen::CUBE:

ccur/session.hpp

@@ -11,7 +11,7 @@
 #include <boost/asio/thread_pool.hpp>
 #include <boost/asio/post.hpp>
 
-#include "eteq/session.hpp"
+#include "teq/session.hpp"
 
 #include "ccur/partition.hpp"
 
@@ -36,7 +36,7 @@ using AtomicFulfilMapT = std::unordered_map<
 
 /// Session that updates operable functors concurrently
 /// across specified a number of jobs
-struct Session final : public eteq::iSession
+struct Session final : public teq::iSession
 {
 	Session (size_t nthreads = 2, OpWeightT weights = OpWeightT()) :
 		nthreads_(nthreads), weights_(weights) {}
@@ -270,13 +270,19 @@ struct Session final : public eteq::iSession
 		pool.join();
 	}
 
-	/// Apply input optimization rules using opt module, then re-track
-	void optimize (const opt::OptCtx& rules)
+	/// Implementation of iSession
+	void clear (void) override
 	{
-		teq::TensptrsT tracked(tracked_.begin(), tracked_.end());
-		opt::optimize(tracked, rules);
+		ops_.clear();
+		tracked_.clear();
 		parents_.clear();
-		track(tracked);
+		requirements_.clear();
+	}
+
+	/// Implementation of iSession
+	teq::TensptrSetT get_tracked (void) const override
+	{
+		return tracked_;
 	}
 
 	/// Set of all tensors input through tracked function