Merge Cortenn and Rocnnet to Tenncor with optimizations (#28)

* Merge Cortenn and Rocnnet to Tenncor with optimizations (#27)

* merge cortenn repo

* add ead without derivative

* fix permute and extend

* implement gradient for ead

* add py tests for ead

* add sigmoid,tanh,square, and cube in ead

* add ead serialize

* add rocnnet

* add optimizers

* add opt tests for llo and fix

* add opt test for ead and fix

* fix opt and add to rocnnet

* add ead to doxygen

* add ead benchmarks

* initial implementation of ead compiler

* remove redundant tensmaps

* prevent building ead compiler

* add matrices to ead operator

* add more tools to evaluate performance

* add transformation code to ead coordmap

* evaluate peak performance, and implement more optimizations

* benchmark and test optimized fast sigmoid mlp

* fix ead operators for benchmark and prototype optimization minilanguage

* initial implementation of subgraph match

* use cppkg to simplify string parsing

* resolve warnings

* fix and mostly complete subgraph matching

* move matcher in ead

* centralize pybind type as PybindT and reuse in rocnnet then set as float

* add tf benchmark for gd_demo

* add some more tf comparisons

* fix pruning and matmul gradient

* session refactor

* implement interactive debugger

* improve interactive debugger

* clean up, fix reduce_sum_1d

* update matcher with abstract rep and create experiment binary

* fix experiment

* implement bottom-up gradient tree building

* remove derive_bu (not working)

* separate chain rule and local derivative lookup

* provide more options for graph representation

* add representation support

* implement rep tree to tensor graph conversion

* add rule conversion

* fix rule conversion

* thoroughly tested optimizer

* fix optimization api to update edge

* add optimization flags in makefile

* revert dqn_trainer labelling

* add cortenn coverage make commands

* add documentation and fix docker tests

* increase coverage

* update some ade docs

.bazelrc

@@ -1,4 +1,19 @@
 
+# ------ ALL SETTINGS ------
+
+run --incompatible_disallow_dict_plus=false
+build --incompatible_disallow_dict_plus=false
+test --incompatible_disallow_dict_plus=false
+coverage --incompatible_disallow_dict_plus=false
+
+# ------ OPTIMIZATION ------
+
+run:clang_eigen_optimal --copt="-march=native" --copt="-openmp" --copt="-O3"
+build:clang_eigen_optimal --copt="-march=native" --copt="-openmp" --copt="-O3"
+
+run:gcc_eigen_optimal --copt="-march=native" --copt="-fopenmp" --copt="-O3"
+build:gcc_eigen_optimal --copt="-march=native" --copt="-fopenmp" --copt="-O3"
+
 # ------ TEST SETTINGS ------
 
 # === ALL TESTS ===

.gitignore

@@ -75,3 +75,4 @@ CMakeFiles
 !.gitmodules
 !.astylerc
 !.bazelrc
+!rocnnet/pretrained/*.pbx

BUILD.bazel

@@ -13,6 +13,11 @@ filegroup(
         "//age:srcs",
         "//bwd:srcs",
         "//dbg:srcs",
+        "//ead:srcs",
+        "//llo:srcs",
+        "//opt:srcs",
+        "//pbm:srcs",
+        "//rocnnet:srcs",
         "BUILD.bazel",
     ],
 )

Dockerfile

@@ -6,5 +6,6 @@ RUN mkdir -p $APP_DIR
 WORKDIR $APP_DIR
 
 COPY . $APP_DIR
+RUN pip install -r requirements.txt
 
 CMD [ "./tests.sh" ]

Doxyfile

@@ -791,7 +791,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = README.md ade bwd
+INPUT                  = README.md ade bwd ead llo opt pbm age/README_AGE.md pybinder/README_PY.md
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

LICENSE.boost

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

Makefile

@@ -6,21 +6,64 @@ ADE_TEST := //ade:test
 
 BWD_TEST := //bwd:test
 
+LLO_CTEST := //llo:ctest
+
+EAD_CTEST := //ead:ctest
+
+OPT_TEST := //opt:test
+
+PBM_TEST := //pbm:test
+
 COVERAGE_IGNORE := 'external/*' '**/test/*' 'testutil/*' '**/genfiles/*' 'dbg/*'
 
 COVERAGE_PIPE := ./bazel-bin/external/com_github_mingkaic_cppkg/merge_cov $(COVERAGE_INFO_FILE)
 
 TMP_LOGFILE := /tmp/tenncor-test.log
 
+CC := gcc
+
+print_vars:
+	@echo "CC: " $(CC)
+
+rocnnet_py_build:
+	bazel build --config $(CC)_eigen_optimal //rocnnet:rocnnet_py
+
+rocnnet_py_export: rocnnet_py_build
+	cp -f bazel-bin/rocnnet/*.so rocnnet/notebooks/rocnnet
+	cp -f bazel-bin/ead/*.so rocnnet/notebooks/ead
 
-coverage: cover_ade cover_bwd
+
+coverage: cover_ade cover_bwd cover_ead cover_opt cover_pbm
 
 cover_ade:
 	$(COVER) $(ADE_TEST)
 
 cover_bwd:
 	$(COVER) $(BWD_TEST)
 
+cover_llo:
+	$(COVER) $(LLO_CTEST)
+
+cover_ead:
+	$(COVER) $(EAD_CTEST)
+
+cover_opt:
+	$(COVER) $(OPT_TEST)
+
+cover_pbm:
+	$(COVER) $(PBM_TEST)
+
+
+# optimized comparisons
+compare_matmul:
+	bazel run $(EIGEN_OPT) //rocnnet:comparison_matmul
+
+compare_mlp:
+	bazel run $(EIGEN_OPT) //rocnnet:comparison_mlp
+
+compare_mlp_grad:
+	bazel run $(EIGEN_OPT) //rocnnet:comparison_mlp_grad
+
 
 merge_cov:
 	bazel build @com_github_mingkaic_cppkg//:merge_cov
@@ -29,6 +72,9 @@ lcov: merge_cov coverage
 	rm -f $(TMP_LOGFILE)
 	cat bazel-testlogs/ade/test/test.log >> $(TMP_LOGFILE)
 	cat bazel-testlogs/bwd/test/test.log >> $(TMP_LOGFILE)
+	cat bazel-testlogs/opt/test/test.log >> $(TMP_LOGFILE)
+	cat bazel-testlogs/ead/ctest/test.log >> $(TMP_LOGFILE)
+	cat bazel-testlogs/pbm/test/test.log >> $(TMP_LOGFILE)
 	cat $(TMP_LOGFILE) | $(COVERAGE_PIPE)
 	lcov --remove $(COVERAGE_INFO_FILE) $(COVERAGE_IGNORE) -o $(COVERAGE_INFO_FILE)
 	rm -f $(TMP_LOGFILE)
@@ -45,3 +91,25 @@ lcov_bwd: merge_cov over_bwd
 	lcov --remove $(COVERAGE_INFO_FILE) $(COVERAGE_IGNORE) 'ade/*' -o $(COVERAGE_INFO_FILE)
 	rm -f $(TMP_LOGFILE)
 	lcov --list $(COVERAGE_INFO_FILE)
+
+lcov_opt: cover_opt
+	rm -f $(TMP_LOGFILE)
+	cat bazel-testlogs/opt/test/test.log | $(COVERAGE_PIPE)
+	lcov --remove $(COVERAGE_INFO_FILE) $(COVERAGE_IGNORE) -o $(COVERAGE_INFO_FILE)
+	rm -f $(TMP_LOGFILE)
+	lcov --list $(COVERAGE_INFO_FILE)
+
+lcov_llo: cover_llo
+	cat bazel-testlogs/llo/ctest/test.log | $(COVERAGE_PIPE)
+	lcov --remove $(COVERAGE_INFO_FILE) $(COVERAGE_IGNORE) 'opt/*' -o $(COVERAGE_INFO_FILE)
+	lcov --list $(COVERAGE_INFO_FILE)
+
+lcov_ead: cover_ead
+	cat bazel-testlogs/ead/ctest/test.log | $(COVERAGE_PIPE)
+	lcov --remove $(COVERAGE_INFO_FILE) $(COVERAGE_IGNORE) 'opt/*' -o $(COVERAGE_INFO_FILE)
+	lcov --list $(COVERAGE_INFO_FILE)
+
+lcov_pbm: cover_pbm
+	cat bazel-testlogs/pbm/test/test.log | $(COVERAGE_PIPE)
+	lcov --remove $(COVERAGE_INFO_FILE) $(COVERAGE_IGNORE) -o $(COVERAGE_INFO_FILE)
+	lcov --list $(COVERAGE_INFO_FILE)

README.md

@@ -20,10 +20,32 @@ Constraints to the equation is limited to each tensor's shape.
 
 This generator creates glue layer between ADE and data manipulation libraries as well as map operational codes to its respective chain rule.
 
-- [BWD (Backward operations)](bwd/README_BWD.md)
+- [BWD (Backward Operations)](bwd/README_BWD.md)
 
 This library provides traveler for generating partial derivative equations using some set of chain rules.
 
+- [LLO (Low Level Operators)](llo/README_LLO.md)
+
+This module is implements basic operations for Tenncor's ADE Tensor objects generated through pybinder.
+Additionally, llo also defines data format and (de)serialization methods required by PBM.
+
+- [EAD (Eigen ADE Operators)](ead/README_EAD.md)
+
+This module is implements basic operations for Tenncor's ADE Tensor objects generated through pybinder.
+Additionally, llo also defines data format and (de)serialization methods required by PBM.
+
+- [OPT (Optimizer)](opt/README_OPT.md)
+
+This module specifies graph optimization through ADE's visitor pattern.
+
+- [PBM (Protobuf Marshaller)](pbm/README_PBM.md)
+
+This module marshals any ADE graph, but requires data serialization functors when saving and loading.
+
+- [Pybinder](pybinder/README_PY.md)
+
+This generator extends Tenncor's AGE generator. In this instance, on top of generating the ADE operators specified in LLO, pybinder generates pybind11 binding code.
+
 ## Tools and utility
 
 - DBG (Debugger)

WORKSPACE

@@ -1,17 +1,31 @@
 workspace(name = "com_github_mingkaic_tenncor")
 
-# local dependencies
+# === local dependencies ===
 
-load("//:tenncor.bzl", "dependencies")
+load("//:third_party/all.bzl", "dependencies")
 dependencies()
 
-# test dependencies
+load("@protobuf_rules//cpp:deps.bzl", "cpp_proto_library")
+cpp_proto_library()
+
+# === test dependencies ===
 
 load("@com_github_mingkaic_cppkg//:gtest.bzl", "gtest_repository")
 gtest_repository(name = "gtest")
 
-git_repository(
-    name = "com_github_google_benchmark",
-    remote = "https://github.com/google/benchmark",
-    commit = "e776aa0275e293707b6a0901e0e8d8a8a3679508",
-)
+load("//third_party/repos:benchmark.bzl", "benchmark_repository")
+benchmark_repository()
+
+load("@com_github_mingkaic_cppkg//:cppkg.bzl", "dependencies")
+dependencies()
+
+load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
+boost_deps()
+
+# === dev dependencies ===
+
+load("@protobuf_rules//cpp:deps.bzl", "cpp_grpc_library")
+cpp_grpc_library()
+
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+grpc_deps()

ade/BUILD.bazel

@@ -31,7 +31,7 @@ cc_library(
     name = "ade",
     hdrs = glob(["*.hpp"]),
     srcs = glob(["src/*.cpp"]),
-    copts = ["-std=c++14"],
+    copts = ["-std=c++17"],
     deps = ["@com_github_mingkaic_cppkg//logs:logs"],
     visibility = ["//visibility:public"],
 )
@@ -46,7 +46,7 @@ cc_test(
         "//ade:ade",
         "//testutil:testutil",
     ],
-    copts = ["-std=c++14"],
+    copts = ["-std=c++17"],
 )
 
 ######### BENCHMARK #########
@@ -58,5 +58,5 @@ cc_binary(
         "//ade:ade",
         "@com_github_google_benchmark//:benchmark",
     ],
-    copts = ["-std=c++14"],
+    copts = ["-std=c++17"],
 )

ade/ade.hpp

@@ -7,13 +7,6 @@
 ///
 
 #include "ade/functor.hpp"
-#include "ade/ileaf.hpp"
 #include "ade/traveler.hpp"
-
-namespace ade
-{
-
-/// Vector representation of ade tensor pointers
-using TensT = std::vector<TensptrT>;
-
-}
+#include "ade/opfunc.hpp"
+#include "ade/edge.hpp"

ade/coord.hpp

@@ -108,6 +108,9 @@ using CoordptrT = std::shared_ptr<iCoordMap>;
 /// Identity matrix instance
 extern CoordptrT identity;
 
+/// Checks if the coord mapper is an identity mapper
+bool is_identity (iCoordMap* coorder);
+
 /// Return coordinate mapper dividing dimensions after rank
 /// by values in red vector
 /// For example, given coordinate [2, 2, 6, 6], rank=2, and red=[3, 3],

ade/edge.hpp

@@ -0,0 +1,65 @@
+///
+/// ead.hpp
+/// ade
+///
+/// Purpose:
+/// Define non-operating edges between tensor nodes
+///
+
+#include "ade/ifunctor.hpp"
+
+#ifndef ADE_EDGE_HPP
+#define ADE_EDGE_HPP
+
+namespace ade
+{
+
+/// Edge between parent and child tensor references and labelled with edge code
+struct Edge final
+{
+	/// Return true if either parent or child are expired, otherwise false
+	bool expired (void) const
+	{
+		return parent_.expired() || child_.expired();
+	}
+
+	/// Parent reference
+	TensrefT parent_;
+
+	/// Child refence
+	TensrefT child_;
+
+	/// Edge label and enumeration
+	Opcode edge_code_;
+};
+
+/// Vector of edges
+using EdgesT = std::vector<Edge>;
+
+/// Edge hasher
+struct EdgeHash final
+{
+	size_t operator() (const Edge& edge) const
+	{
+		if (edge.expired())
+		{
+			return 0;
+		}
+		std::stringstream ss;
+		ss << edge.parent_.lock().get() <<
+			edge.child_.lock().get() <<
+			edge.edge_code_.code_;
+		return std::hash<std::string>()(ss.str());
+	}
+};
+
+/// Edge equality comparator
+inline bool operator == (const Edge& lhs, const Edge& rhs)
+{
+	EdgeHash hasher;
+    return hasher(lhs) == hasher(rhs);
+}
+
+}
+
+#endif // ADE_EDGE_HPP