Skip to content

Fetch TORCH_SRCS from build_variables.bzl #36737

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Fetch TORCH_SRCS from build_variables.bzl #36737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 3 additions & 159 deletions caffe2/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,165 +345,9 @@ if(NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
DEPENDS ${TORCH_GENERATED_CODE}
)


set(TORCH_SRCS
${GENERATED_CXX_TORCH}
${GENERATED_H_TORCH}
${TORCH_SRC_DIR}/csrc/autograd/anomaly_mode.cpp
${TORCH_SRC_DIR}/csrc/autograd/autograd.cpp
${TORCH_SRC_DIR}/csrc/autograd/custom_function.cpp
${TORCH_SRC_DIR}/csrc/autograd/cpp_hook.cpp
${TORCH_SRC_DIR}/csrc/autograd/engine.cpp
${TORCH_SRC_DIR}/csrc/autograd/function.cpp
${TORCH_SRC_DIR}/csrc/autograd/function_hook.cpp
${TORCH_SRC_DIR}/csrc/autograd/functions/accumulate_grad.cpp
${TORCH_SRC_DIR}/csrc/autograd/functions/basic_ops.cpp
${TORCH_SRC_DIR}/csrc/autograd/functions/tensor.cpp
${TORCH_SRC_DIR}/csrc/autograd/functions/utils.cpp
${TORCH_SRC_DIR}/csrc/autograd/input_buffer.cpp
${TORCH_SRC_DIR}/csrc/autograd/profiler.cpp
${TORCH_SRC_DIR}/csrc/autograd/record_function.cpp
${TORCH_SRC_DIR}/csrc/autograd/record_function_ops.cpp
${TORCH_SRC_DIR}/csrc/autograd/saved_variable.cpp
${TORCH_SRC_DIR}/csrc/autograd/variable.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/autodiff.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/attributes.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/argument_spec.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/pass_manager.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/pickler.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/unpickler.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/graph_executor.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/import_source.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/import.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/pickle.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/import_export_helpers.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/instruction.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/interpreter.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/constants.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/node_hashing.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/type_hashing.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/ir.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/irparser.cpp
${TORCH_SRC_DIR}/csrc/jit/jit_log.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/operator.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_c10_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/subgraph_matcher.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/symbolic_script.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/profiling_record.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/profiling_graph_executor_impl.cpp
${TORCH_SRC_DIR}/csrc/jit/python/update_graph_executor_opt.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/alias_analysis.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/batch_mm.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/bailout_graph.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/canonicalize.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/clear_undefinedness.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/clear_profiling.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/constant_propagation.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/constant_pooling.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/common_subexpression_elimination.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/create_autodiff_subgraphs.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/create_functional_graphs.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/inline_autodiff_subgraphs.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/insert_guards.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/inliner.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/lift_closures.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/inline_forked_closures.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/dead_code_elimination.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/decompose_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/canonicalize_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/erase_number_types.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/fixup_trace_scope_blocks.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/inline_fork_wait.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/graph_fuser.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/graph_rewrite_helper.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/guard_elimination.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/inplace_check.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/liveness.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/loop_unrolling.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/lower_grad_of.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/lower_graph.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/lower_tuples.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/peephole.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/remove_expands.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/remove_inplace_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/shape_analysis.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/requires_grad_analysis.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/specialize_autogradzero.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/subgraph_rewrite.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/tensorexpr_fuser.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/python_print.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/utils/subgraph_utils.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/utils/check_alias_annotation.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/utils/memory_dag.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/prepack_folding.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/quantization.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/xnnpack_rewrite.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/fuse_linear.cpp
${TORCH_SRC_DIR}/csrc/jit/passes/freeze_module.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/print_handler.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/interface.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/cuda/interface.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_ops_utils.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_prim_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_prim_ops_fulljit.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_prim_ops_c10.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_string_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/register_special_ops.cpp
${TORCH_SRC_DIR}/csrc/jit/ir/scope.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/ir_emitter.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/name_mangler.cpp
${TORCH_SRC_DIR}/csrc/jit/testing/file_check.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/convert_to_ssa.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/exit_transforms.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/inline_loop_condition.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/schema_matching.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/script_type_parser.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/sugared_value.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/parser.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/builtin_functions.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/versioned_symbols.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/canonicalize_modified_loop.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/edit_distance.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/logging.cpp
${TORCH_SRC_DIR}/csrc/jit/api/module.cpp
${TORCH_SRC_DIR}/csrc/jit/api/object.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/jit_exception.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/string_to_type.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/source_range_serialization.cpp
${TORCH_SRC_DIR}/csrc/jit/serialization/type_name_uniquer.cpp
${TORCH_SRC_DIR}/csrc/jit/frontend/tracer.cpp
${TORCH_SRC_DIR}/csrc/jit/testing/hooks_for_testing.cpp
${TORCH_SRC_DIR}/csrc/utils/tensor_flatten.cpp
${TORCH_SRC_DIR}/csrc/utils/variadic.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/kernel_cache.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/compiler.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/executor.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/codegen.cpp
${TORCH_SRC_DIR}/csrc/jit/codegen/fuser/fallback.cpp
${TORCH_SRC_DIR}/csrc/jit/api/function_impl.cpp
${TORCH_SRC_DIR}/csrc/jit/runtime/vararg_functions.cpp

${TORCH_SRC_DIR}/csrc/jit/tensorexpr/bounds_inference.cpp
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@malfet do you have plan to read other lists (those gated by USE_DISTRIBUTED / USE_CUDA / etc) from build_variables.bzl as well?
I'm asking because I plan to remove 'tensorexpr/' from mobile build to reduce binary size. My plan is to create a separate list in build_variables.bzl and move tensorexpr/ related stuff from "libtorch_cmake_sources" into it. I'd like to make sure it's aligned with your long term plan before making the change.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that aligns with the longer term plan.
Please go ahead and create a list in build_variables.bzl and read it in caffe2/CmakeLists.txt using get_filelist() macro. And feel free to expand it to append_filelist, if it'll make more sense.

${TORCH_SRC_DIR}/csrc/jit/tensorexpr/codegen.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/eval.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/expr.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/function.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/hash_provider.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/ir.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/ir_mutator.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/ir_printer.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/ir_simplifier.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/ir_visitor.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/kernel.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/llvm_codegen.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/llvm_jit.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/loopnest.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/mem_arena.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/tensor.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/types.cpp
${TORCH_SRC_DIR}/csrc/jit/tensorexpr/unique_name_manager.cpp
${TORCH_SRC_DIR}/csrc/jit/mobile/type_parser.cpp
)
get_filelist("libtorch_cmake_sources" TORCH_SRCS)
list(APPEND TORCH_SRCS ${GENERATED_CXX_TORCH})
list(APPEND TORCH_SRCS ${GENERATED_H_TORCH})

# Required workaround for LLVM 9 includes.
if(NOT MSVC)
Expand Down
12 changes: 12 additions & 0 deletions cmake/Codegen.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# - Configures caffe2/core/macros.h
# - Creates an ATen target for its generated C++ files and adds it
# as a dependency
# - Reads build lists defined in build_variables.bzl

################################################################################
# Helper functions
Expand Down Expand Up @@ -204,3 +205,14 @@ if(INTERN_BUILD_ATEN_OPS)
add_dependencies(ATEN_CPU_FILES_GEN_LIB ATEN_CPU_FILES_GEN_TARGET)
add_dependencies(ATEN_CUDA_FILES_GEN_LIB ATEN_CUDA_FILES_GEN_TARGET)
endif()

function(get_filelist name outputvar)
set(_rootdir "${CMAKE_CURRENT_LIST_DIR}/../")
execute_process(
COMMAND "${PYTHON_EXECUTABLE}" -c
"exec(open('../tools/build_variables.bzl').read());print(';'.join(['${_rootdir}' + x for x in ${name}]))"
WORKING_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}"
OUTPUT_VARIABLE _tempvar)
string(REPLACE "\n" "" _tempvar "${_tempvar}")
set(${outputvar} ${_tempvar} PARENT_SCOPE)
endfunction()
29 changes: 19 additions & 10 deletions tools/build_variables.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,17 +203,8 @@ libtorch_distributed_sources = [
"torch/csrc/distributed/rpc/utils.cpp",
]

libtorch_extra_sources = [
"torch/csrc/autograd/VariableTypeManual.cpp",
"torch/csrc/jit/api/module_save.cpp",
libtorch_core_jit_sources = [
"torch/csrc/jit/codegen/cuda/interface.cpp",
"torch/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp",
"torch/csrc/jit/mobile/function.cpp",
"torch/csrc/jit/mobile/import.cpp",
"torch/csrc/jit/mobile/interpreter.cpp",
"torch/csrc/jit/mobile/module.cpp",
"torch/csrc/jit/mobile/register_mobile_autograd.cpp",
"torch/csrc/jit/mobile/register_mobile_ops.cpp",
"torch/csrc/jit/passes/create_functional_graphs.cpp",
"torch/csrc/jit/passes/lower_graph.cpp",
"torch/csrc/jit/runtime/register_c10_ops.cpp",
Expand All @@ -222,6 +213,24 @@ libtorch_extra_sources = [
"torch/csrc/jit/runtime/register_prim_ops_fulljit.cpp",
"torch/csrc/jit/runtime/register_special_ops.cpp",
"torch/csrc/jit/runtime/register_string_ops.cpp",
]

libtorch_cmake_sources = libtorch_core_sources + libtorch_core_jit_sources + [
"torch/csrc/jit/passes/inline_fork_wait.cpp",
"torch/csrc/jit/passes/remove_inplace_ops.cpp",
"torch/csrc/jit/passes/utils/check_alias_annotation.cpp",
]

libtorch_extra_sources = libtorch_core_jit_sources + [
"torch/csrc/autograd/VariableTypeManual.cpp",
"torch/csrc/jit/api/module_save.cpp",
"torch/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp",
"torch/csrc/jit/mobile/function.cpp",
"torch/csrc/jit/mobile/import.cpp",
"torch/csrc/jit/mobile/interpreter.cpp",
"torch/csrc/jit/mobile/module.cpp",
"torch/csrc/jit/mobile/register_mobile_autograd.cpp",
"torch/csrc/jit/mobile/register_mobile_ops.cpp",
"torch/csrc/jit/serialization/export.cpp",
"torch/csrc/jit/serialization/export_module.cpp",
"torch/csrc/jit/serialization/import_legacy.cpp",
Expand Down