[Build] Simplify the integration of external backends

.gitignore

@@ -17,6 +17,7 @@ compile_commands.json
 build/
 external/googletest/
 external/llvm/
+externalbackends/*/
 .vscode/
 .vim/
 .idea/

CMakeLists.txt

@@ -30,6 +30,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 include(GlowDefaults)
 include(GlowTestSupport)
+include(GlowExternalBackends)
 include(SanitizerSupport)
 include(CoverageSupport)
 include(DoxygenSupport)
@@ -51,6 +52,7 @@ include_directories(BEFORE
 
 include_directories(${GLOW_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(${GLOW_SOURCE_DIR})
 
 file(GLOB_RECURSE header_files include/*.h tools/*.h lib/*.h)
 add_custom_target(CollectHeaders SOURCES ${header_files})
@@ -83,6 +85,9 @@ if (GLOW_WITH_HABANA)
   add_definitions(-DGLOW_WITH_HABANA=1)
 endif ()
 
+# Top level setup for external backends
+ExternalBackendsInit()
+
 # Prefer LLVM 7.
 find_package(LLVM 7 CONFIG)
 

cmake/modules/GlowDefaults.cmake

@@ -76,3 +76,15 @@ function(make_whole_archive DST SRC)
     target_link_libraries(${DST} INTERFACE ${SRC})
   endif()
 endfunction()
+
+# Return in result the name of curdir sub-directories
+MACRO(getSubDirList result curdir)
+  file(GLOB children RELATIVE ${curdir} ${curdir}/[^\.]*)
+  SET(dirlist "")
+  FOREACH(child ${children})
+    IF(IS_DIRECTORY ${curdir}/${child})
+      LIST(APPEND dirlist ${child})
+    ENDIF()
+  ENDFOREACH()
+  SET(${result} ${dirlist})
+ENDMACRO()

cmake/modules/GlowExternalBackends.cmake

@@ -0,0 +1,118 @@
+# Copyright (c) 2019-present, Facebook, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A function to add a test to be driven through the 'check' target.
+# Unlike the 'test' target, the 'check' target rebuilds the executables
+# before invoking the tests.
+
+# Returns the name of the backend enabling.
+MACRO(getBackendEnableVariable result backend_dir_name)
+  string(TOUPPER "GLOW_WITH_${backend_dir_name}" ${result})
+ENDMACRO()
+
+
+# Macro to be called at top level.
+MACRO(ExternalBackendsInit)
+# External backends
+set(EXTERNAL_BACKENDS_DIR ${GLOW_SOURCE_DIR}/externalbackends)
+include_directories(${EXTERNAL_BACKENDS_DIR})
+getSubDirList(SUBDIRS ${EXTERNAL_BACKENDS_DIR})
+
+FOREACH(child ${SUBDIRS})
+  # Add an option for the backend. The backend is enabled by default.
+  # The user can disable it by setting the right variable to OFF.
+  getBackendEnableVariable(backend_enable_variable ${child})
+  option(${backend_enable_variable} "Build the ${child} backend" ON)
+  # define BACKEND_ROOT_<upper case backend name>
+  string(TOUPPER "BACKEND_ROOT_${child}" BACKEND_ROOT_NAME)
+  set(${BACKEND_ROOT_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/externalbackends/${child})
+
+  # Verbosing
+  message(STATUS "Detected external backend '${child}'")
+  message(STATUS " -> Backend '${child}' can be disabled by setting ${backend_enable_variable}=OFF")
+  message(STATUS " -> Backend '${child}' specific variables:")
+  message(STATUS "    - ${backend_enable_variable} = ${${backend_enable_variable}}")
+  message(STATUS "    - ${BACKEND_ROOT_NAME} = ${${BACKEND_ROOT_NAME}}")
+  if (${backend_enable_variable})
+      message(STATUS " -> Backend '${child}' ENABLED")
+      add_definitions(-D${backend_enable_variable}=1)
+  else()
+      message(STATUS " -> Backend '${child}' DISABLED")
+  endif()
+
+  # Handle the backend only when activated
+  if (${backend_enable_variable})
+      # If the backend has a global CMakefile, include it.
+     if(EXISTS "${EXTERNAL_BACKENDS_DIR}/${child}/CMakeLists.txt")
+        include("${EXTERNAL_BACKENDS_DIR}/${child}/CMakeLists.txt")
+     else()
+        message(STATUS "External backend '${child}' has no global CMakeLists.txt")
+     endif()
+  endif()
+ENDFOREACH()
+ENDMACRO()
+
+# Macro to register external backends.
+MACRO(ExternalBackendsRegister)
+getSubDirList(SUBDIRS ${GLOW_SOURCE_DIR}/externalbackends)
+FOREACH(child ${SUBDIRS})
+  getBackendEnableVariable(backend_enable_variable ${child})
+  # Handle the backend only when activated
+  if (${backend_enable_variable})
+    # If the backend has a 'Backends' sub-directory, add it.
+    if(EXISTS ${EXTERNAL_BACKENDS_DIR}/${child}/Backends)
+      message("Adding external ${child} backend.")
+      set(EXTERNAL_BACKEND_NAME ${child}Backend)
+      add_subdirectory(${EXTERNAL_BACKENDS_DIR}/${child}/Backends EXT_${EXTERNAL_BACKEND_NAME})
+    else()
+      message(FATAL_ERROR "External backend '${child}' has no 'Backends' sub-directory (${EXTERNAL_BACKENDS_DIR}/${child}/Backends)")
+    endif()
+  endif()
+ENDFOREACH()
+ENDMACRO()
+
+# Macro to register backend specific nodes and instructions.
+MACRO(ExternalBackendsClassGen)
+getSubDirList(SUBDIRS ${GLOW_SOURCE_DIR}/externalbackends)
+FOREACH(child ${SUBDIRS})
+  getBackendEnableVariable(backend_enable_variable ${child})
+  # Handle the backend only when activated
+  if (${backend_enable_variable})
+    # If the backend has a 'ClassGen' sub-directory, add it.
+    if(EXISTS ${EXTERNAL_BACKENDS_DIR}/${child}/ClassGen)
+       add_subdirectory(${EXTERNAL_BACKENDS_DIR}/${child}/ClassGen EXT_${child})
+    else()
+      message(STATUS "External backend '${child}' has no 'ClassGen' sub-directory")
+    endif()
+  endif()
+ENDFOREACH()
+ENDMACRO()
+
+# Macro to add external backends tests.
+MACRO(ExternalBackendsTest)
+getSubDirList(SUBDIRS ${GLOW_SOURCE_DIR}/externalbackends)
+FOREACH(child ${SUBDIRS})
+  getBackendEnableVariable(backend_enable_variable ${child})
+  # Handle the backend only when activated
+  if (${backend_enable_variable})
+    # If the backend has a 'Tests' sub-directory, add it.
+    if(EXISTS "${EXTERNAL_BACKENDS_DIR}/${child}/Tests")
+      add_subdirectory("${EXTERNAL_BACKENDS_DIR}/${child}/Tests" EXT_${child})
+    else()
+      message(STATUS "External backend '${child}' has no 'Tests' sub-directory")
+    endif()
+  endif()
+ENDFOREACH()
+ENDMACRO()
+

externalbackends/README.txt

@@ -0,0 +1 @@
+This directory is intended to contain external backends.

lib/Backends/CMakeLists.txt

@@ -16,6 +16,9 @@ foreach(object ${subdirs})
   endif()
 endforeach()
 
+# External backends
+ExternalBackendsRegister()
+
 add_library(ExecutionContext
               TraceEvents.cpp)
 target_link_libraries(ExecutionContext
@@ -38,6 +41,7 @@ target_link_libraries(Backend
                         Optimizer)
 add_library(Backends
               Backends.cpp)
+
 target_link_libraries(Backends
                       PRIVATE
                         Base

tests/CMakeLists.txt

@@ -5,6 +5,9 @@ add_subdirectory(images)
 add_subdirectory(benchmark)
 add_subdirectory(models)
 
+# External backends
+ExternalBackendsTest()
+
 # Function which creates a new OutputCheck test that runs test_script.
 function(add_output_check_test)
   set(oneValueArgs NAME)

tools/ClassGen/CMakeLists.txt

@@ -30,3 +30,6 @@ endif()
 if(GLOW_WITH_HABANA)
   add_subdirectory(Backends/Habana)
 endif()
+
+# External backends
+ExternalBackendsClassGen()

tools/ClassGen/InstrGen.cpp

@@ -625,6 +625,11 @@ int main(int argc, char **argv) {
 #include "Backends/CPU/CPUSpecificInstrs.h"
 #include "Backends/Habana/HabanaSpecificInstrs.h"
 #include "Backends/OpenCL/OpenCLSpecificInstrs.h"
+  // Add here external backend specific instructions headers.
+  // Example:
+  // #ifdef GLOW_WITH_<NAME>
+  // #include "<Name>/ClassGen/<Name>SpecificInstrs.h"
+  // #endif
 
   return 0;
 }

tools/ClassGen/NodeGen.cpp

@@ -716,6 +716,11 @@ int main(int argc, char **argv) {
 #include "Backends/CPU/CPUSpecificNodes.h"
 #include "Backends/OpenCL/OpenCLSpecificNodes.h"
 #include "Backends/Habana/HabanaSpecificNodes.h"
+  // Add here external backend specific node headers.
+  // Example:
+  // #ifdef GLOW_WITH_<NAME>
+  // #include "<Name>/ClassGen/<Name>SpecificNodes.h"
+  // #endif
 
   return 0;
 }