From 2e51b6561faa7163f2efbacf95e8fbe2e23f7048 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 20 Aug 2021 22:25:42 -0700 Subject: [PATCH 01/51] Added generator.fbs --- buildcc/schema/CMakeLists.txt | 2 ++ buildcc/schema/generator.fbs | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 buildcc/schema/generator.fbs diff --git a/buildcc/schema/CMakeLists.txt b/buildcc/schema/CMakeLists.txt index c1a6b604..cbbe4b23 100644 --- a/buildcc/schema/CMakeLists.txt +++ b/buildcc/schema/CMakeLists.txt @@ -3,10 +3,12 @@ set(SCHEMA_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated CACHE PATH "Generate set(FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/path.fbs + ${CMAKE_CURRENT_SOURCE_DIR}/generator.fbs ${CMAKE_CURRENT_SOURCE_DIR}/target.fbs ) set(FBS_GEN_FILES ${SCHEMA_BUILD_DIR}/path_generated.h + ${SCHEMA_BUILD_DIR}/generator_generated.h ${SCHEMA_BUILD_DIR}/target_generated.h ) set(FBS_GEN_OPTIONS diff --git a/buildcc/schema/generator.fbs b/buildcc/schema/generator.fbs new file mode 100644 index 00000000..b796c509 --- /dev/null +++ b/buildcc/schema/generator.fbs @@ -0,0 +1,29 @@ +// Copyright 2021 Niket Naidu. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +include "path.fbs"; + +namespace schema.internal; + +table Generator { + name:string (key); + + // Input and output files are both tracked via timestamp + input:[Path]; + output:[Path]; + + command:[string]; +} + +root_type Generator; From df47716a6ec76ed6c8dba1b9846349cac0edb5d8 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 20 Aug 2021 22:58:11 -0700 Subject: [PATCH 02/51] Update buildcc.h --- buildcc/buildcc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buildcc/buildcc.h b/buildcc/buildcc.h index b4262684..39420747 100644 --- a/buildcc/buildcc.h +++ b/buildcc/buildcc.h @@ -24,8 +24,12 @@ #include "env/assert_fatal.h" #include "env/logging.h" #include "env/host_os.h" +#include "env/host_compiler.h" #include "env/util.h" +// +#include "command/command.h" + // Base #include "toolchain/toolchain.h" #include "target/target.h" From 9f2f7b34fe869b2e3e387222b92690e509d48fff Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 23 Aug 2021 23:06:27 -0700 Subject: [PATCH 03/51] Renamed fbs_loader.h to target_loader.h - Updated ifdef macros - Updated include paths - Updated test include paths --- buildcc/lib/target/cmake/target.cmake | 15 ++++++++------- buildcc/lib/target/include/target/target.h | 2 +- .../target/{fbs_loader.h => target_loader.h} | 4 ++-- buildcc/lib/target/src/fbs/fbs_loader.cpp | 2 +- .../test/target/test_target_c_compile_flags.cpp | 2 +- .../test/target/test_target_cpp_compile_flags.cpp | 2 +- .../test/target/test_target_external_lib.cpp | 2 +- .../target/test/target/test_target_lib_dep.cpp | 2 +- .../target/test/target/test_target_link_flags.cpp | 2 +- .../target/test_target_preprocessor_flags.cpp | 2 +- 10 files changed, 18 insertions(+), 17 deletions(-) rename buildcc/lib/target/include/target/{fbs_loader.h => target_loader.h} (98%) diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 2603ce76..246b951b 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -1,4 +1,10 @@ set(TARGET_SRCS + + src/util/util.cpp + + src/fbs/fbs_loader.cpp + src/fbs/fbs_storer.cpp + src/target/target.cpp src/target/source.cpp src/target/include_dir.cpp @@ -9,15 +15,10 @@ set(TARGET_SRCS src/target/recheck_states.cpp src/target/tasks.cpp - src/fbs/fbs_loader.cpp - src/fbs/fbs_storer.cpp - - src/util/util.cpp - - include/target/target.h - include/target/fbs_loader.h include/target/path.h include/target/util.h + include/target/target_loader.h + include/target/target.h ) if(${BUILDCC_BUILD_AS_SINGLE_LIB}) diff --git a/buildcc/lib/target/include/target/target.h b/buildcc/lib/target/include/target/target.h index 7e8bc3de..6c539f30 100644 --- a/buildcc/lib/target/include/target/target.h +++ b/buildcc/lib/target/include/target/target.h @@ -27,8 +27,8 @@ #include // Internal -#include "target/fbs_loader.h" #include "target/path.h" +#include "target/target_loader.h" #include "command/command.h" #include "env/env.h" diff --git a/buildcc/lib/target/include/target/fbs_loader.h b/buildcc/lib/target/include/target/target_loader.h similarity index 98% rename from buildcc/lib/target/include/target/fbs_loader.h rename to buildcc/lib/target/include/target/target_loader.h index 9e2e1753..8eb8998c 100644 --- a/buildcc/lib/target/include/target/fbs_loader.h +++ b/buildcc/lib/target/include/target/target_loader.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef TARGET_FBS_LOADER_H_ -#define TARGET_FBS_LOADER_H_ +#ifndef TARGET_TARGET_LOADER_H_ +#define TARGET_TARGET_LOADER_H_ #include #include diff --git a/buildcc/lib/target/src/fbs/fbs_loader.cpp b/buildcc/lib/target/src/fbs/fbs_loader.cpp index de743c7a..16b7f119 100644 --- a/buildcc/lib/target/src/fbs/fbs_loader.cpp +++ b/buildcc/lib/target/src/fbs/fbs_loader.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "target/fbs_loader.h" +#include "target/target_loader.h" #include "target_generated.h" diff --git a/buildcc/lib/target/test/target/test_target_c_compile_flags.cpp b/buildcc/lib/target/test/target/test_target_c_compile_flags.cpp index b9299ac6..7d6c0090 100644 --- a/buildcc/lib/target/test/target/test_target_c_compile_flags.cpp +++ b/buildcc/lib/target/test/target/test_target_c_compile_flags.cpp @@ -8,7 +8,7 @@ #include "env/env.h" // -#include "target/fbs_loader.h" +#include "target/target_loader.h" // Third Party diff --git a/buildcc/lib/target/test/target/test_target_cpp_compile_flags.cpp b/buildcc/lib/target/test/target/test_target_cpp_compile_flags.cpp index 27df7c16..6d67a46c 100644 --- a/buildcc/lib/target/test/target/test_target_cpp_compile_flags.cpp +++ b/buildcc/lib/target/test/target/test_target_cpp_compile_flags.cpp @@ -8,7 +8,7 @@ #include "env/env.h" // -#include "target/fbs_loader.h" +#include "target/target_loader.h" // Third Party diff --git a/buildcc/lib/target/test/target/test_target_external_lib.cpp b/buildcc/lib/target/test/target/test_target_external_lib.cpp index 6ffd0e90..2c558d99 100644 --- a/buildcc/lib/target/test/target/test_target_external_lib.cpp +++ b/buildcc/lib/target/test/target/test_target_external_lib.cpp @@ -7,7 +7,7 @@ #include "target/target.h" -#include "target/fbs_loader.h" +#include "target/target_loader.h" #include #include diff --git a/buildcc/lib/target/test/target/test_target_lib_dep.cpp b/buildcc/lib/target/test/target/test_target_lib_dep.cpp index 7e3ebe0c..eff66c1d 100644 --- a/buildcc/lib/target/test/target/test_target_lib_dep.cpp +++ b/buildcc/lib/target/test/target/test_target_lib_dep.cpp @@ -9,7 +9,7 @@ #include "target/target.h" // -#include "target/fbs_loader.h" +#include "target/target_loader.h" #include #include diff --git a/buildcc/lib/target/test/target/test_target_link_flags.cpp b/buildcc/lib/target/test/target/test_target_link_flags.cpp index d76f290d..16631d00 100644 --- a/buildcc/lib/target/test/target/test_target_link_flags.cpp +++ b/buildcc/lib/target/test/target/test_target_link_flags.cpp @@ -8,7 +8,7 @@ #include "env/env.h" // -#include "target/fbs_loader.h" +#include "target/target_loader.h" // Third Party diff --git a/buildcc/lib/target/test/target/test_target_preprocessor_flags.cpp b/buildcc/lib/target/test/target/test_target_preprocessor_flags.cpp index 578614a0..a0462376 100644 --- a/buildcc/lib/target/test/target/test_target_preprocessor_flags.cpp +++ b/buildcc/lib/target/test/target/test_target_preprocessor_flags.cpp @@ -8,7 +8,7 @@ #include "env/env.h" // -#include "target/fbs_loader.h" +#include "target/target_loader.h" // Third Party From 8e2e0fe2c0c88f5c73ed8b0288ab3af740bbf83e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 23 Aug 2021 23:24:43 -0700 Subject: [PATCH 04/51] Renamed fbs_loader and fbs_storer to target_loader and target_storer - Updated cmake files for target and mock_target --- buildcc/lib/target/cmake/mock_target.cmake | 4 ++-- buildcc/lib/target/cmake/target.cmake | 4 ++-- .../src/{fbs/fbs_loader.cpp => target/target_loader.cpp} | 0 .../src/{fbs/fbs_storer.cpp => target/target_storer.cpp} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename buildcc/lib/target/src/{fbs/fbs_loader.cpp => target/target_loader.cpp} (100%) rename buildcc/lib/target/src/{fbs/fbs_storer.cpp => target/target_storer.cpp} (100%) diff --git a/buildcc/lib/target/cmake/mock_target.cmake b/buildcc/lib/target/cmake/mock_target.cmake index e75b4ed7..8d1a2286 100644 --- a/buildcc/lib/target/cmake/mock_target.cmake +++ b/buildcc/lib/target/cmake/mock_target.cmake @@ -8,8 +8,8 @@ add_library(mock_target STATIC mock/target/recheck_states.cpp mock/target/tasks.cpp - src/fbs/fbs_loader.cpp - src/fbs/fbs_storer.cpp + src/target/target_loader.cpp + src/target/target_storer.cpp src/util/util.cpp ) diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 246b951b..c77a5f6b 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -2,8 +2,8 @@ set(TARGET_SRCS src/util/util.cpp - src/fbs/fbs_loader.cpp - src/fbs/fbs_storer.cpp + src/target/target_loader.cpp + src/target/target_storer.cpp src/target/target.cpp src/target/source.cpp diff --git a/buildcc/lib/target/src/fbs/fbs_loader.cpp b/buildcc/lib/target/src/target/target_loader.cpp similarity index 100% rename from buildcc/lib/target/src/fbs/fbs_loader.cpp rename to buildcc/lib/target/src/target/target_loader.cpp diff --git a/buildcc/lib/target/src/fbs/fbs_storer.cpp b/buildcc/lib/target/src/target/target_storer.cpp similarity index 100% rename from buildcc/lib/target/src/fbs/fbs_storer.cpp rename to buildcc/lib/target/src/target/target_storer.cpp From 5455adcb28ac1487b4230ff6a230ea848b0f05ae Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Tue, 24 Aug 2021 01:54:11 -0700 Subject: [PATCH 05/51] Added loader_interface - Multiple loaders present which requires the interface pattern --- buildcc/lib/target/cmake/target.cmake | 2 + .../target/include/target/loader_interface.h | 39 +++++++++++++++++++ .../lib/target/include/target/target_loader.h | 16 ++++---- 3 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 buildcc/lib/target/include/target/loader_interface.h diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index c77a5f6b..c5743228 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -17,6 +17,8 @@ set(TARGET_SRCS include/target/path.h include/target/util.h + include/target/loader_interface.h + include/target/target_loader.h include/target/target.h ) diff --git a/buildcc/lib/target/include/target/loader_interface.h b/buildcc/lib/target/include/target/loader_interface.h new file mode 100644 index 00000000..1123262f --- /dev/null +++ b/buildcc/lib/target/include/target/loader_interface.h @@ -0,0 +1,39 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TARGET_LOADER_INTERFACE_H_ +#define TARGET_LOADER_INTERFACE_H_ + +#include + +namespace fs = std::filesystem; + +namespace buildcc::internal { + +class LoaderInterface { +public: + virtual bool Load() = 0; + virtual fs::path GetBinaryPath() const = 0; + + bool IsLoaded() const { return loaded_; }; + +protected: + bool loaded_{false}; +}; + +} // namespace buildcc::internal + +#endif diff --git a/buildcc/lib/target/include/target/target_loader.h b/buildcc/lib/target/include/target/target_loader.h index 8eb8998c..0bdc3c0e 100644 --- a/buildcc/lib/target/include/target/target_loader.h +++ b/buildcc/lib/target/include/target/target_loader.h @@ -17,17 +17,17 @@ #ifndef TARGET_TARGET_LOADER_H_ #define TARGET_TARGET_LOADER_H_ -#include +#include "target/loader_interface.h" + #include #include #include "target/path.h" -namespace fs = std::filesystem; - namespace buildcc::internal { -class FbsLoader { +// TODO, Change name to TargetLoader +class FbsLoader : public LoaderInterface { public: explicit FbsLoader(const std::string &name, const fs::path &relative_path) : name_(name), relative_path_(relative_path) { @@ -37,11 +37,12 @@ class FbsLoader { FbsLoader(const FbsLoader &loader) = delete; public: - bool Load(); + bool Load() override; // Getters - bool IsLoaded() const { return loaded_; } - fs::path GetBinaryPath() const { return relative_path_ / (name_ + ".bin"); } + fs::path GetBinaryPath() const override { + return relative_path_ / (name_ + ".bin"); + } const path_unordered_set &GetLoadedSources() const { return loaded_sources_; } const path_unordered_set &GetLoadedHeaders() const { return loaded_headers_; } @@ -85,7 +86,6 @@ class FbsLoader { private: std::string name_; fs::path relative_path_; - bool loaded_ = false; path_unordered_set loaded_sources_; path_unordered_set loaded_headers_; From 041b671c95604100b2349b74a6968f2c61d2bc5d Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Tue, 24 Aug 2021 01:57:40 -0700 Subject: [PATCH 06/51] Update target_loader.h --- buildcc/lib/target/include/target/target_loader.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/include/target/target_loader.h b/buildcc/lib/target/include/target/target_loader.h index 0bdc3c0e..7a9bb5c8 100644 --- a/buildcc/lib/target/include/target/target_loader.h +++ b/buildcc/lib/target/include/target/target_loader.h @@ -22,6 +22,8 @@ #include #include +#include "fmt/format.h" + #include "target/path.h" namespace buildcc::internal { @@ -41,7 +43,7 @@ class FbsLoader : public LoaderInterface { // Getters fs::path GetBinaryPath() const override { - return relative_path_ / (name_ + ".bin"); + return relative_path_ / fmt::format("{}.bin", name_); } const path_unordered_set &GetLoadedSources() const { return loaded_sources_; } From c14e3e407977058ab2996655826ae7dba9dfcdf3 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Tue, 24 Aug 2021 02:22:47 -0700 Subject: [PATCH 07/51] Added generator_loader.h and generator_loader.cpp --- buildcc/lib/target/cmake/target.cmake | 3 ++ .../target/include/target/generator_loader.h | 47 +++++++++++++++++++ .../target/src/generator/generator_loader.cpp | 26 ++++++++++ 3 files changed, 76 insertions(+) create mode 100644 buildcc/lib/target/include/target/generator_loader.h create mode 100644 buildcc/lib/target/src/generator/generator_loader.cpp diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index c5743228..9dd48f6d 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -1,4 +1,7 @@ set(TARGET_SRCS + src/generator/generator_loader.cpp + + include/target/generator_loader.h src/util/util.cpp diff --git a/buildcc/lib/target/include/target/generator_loader.h b/buildcc/lib/target/include/target/generator_loader.h new file mode 100644 index 00000000..124e7e41 --- /dev/null +++ b/buildcc/lib/target/include/target/generator_loader.h @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TARGET_GENERATOR_LOADER_H_ +#define TARGET_GENERATOR_LOADER_H_ + +#include "loader_interface.h" + +#include "fmt/format.h" + +namespace buildcc::internal { + +class GeneratorLoader : public LoaderInterface { +public: + GeneratorLoader(const std::string &name, const fs::path &relative_path) + : name_(name), relative_path_(relative_path) {} + + GeneratorLoader(const GeneratorLoader &loader) = delete; + + bool Load() override; + + // Getters + fs::path GetBinaryPath() const override { + return relative_path_ / fmt::format("{}.bin", name_); + } + +private: + std::string name_; + fs::path relative_path_; +}; + +} // namespace buildcc::internal + +#endif diff --git a/buildcc/lib/target/src/generator/generator_loader.cpp b/buildcc/lib/target/src/generator/generator_loader.cpp new file mode 100644 index 00000000..636db61b --- /dev/null +++ b/buildcc/lib/target/src/generator/generator_loader.cpp @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/generator_loader.h" + +namespace buildcc::internal { + +bool GeneratorLoader::Load() { + // TODO, + return true; +} + +} // namespace buildcc::internal From 385845c61f7bb8e21a2856799be470957d50b9e6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Tue, 24 Aug 2021 19:17:14 -0700 Subject: [PATCH 08/51] Update generator_loader.h --- buildcc/lib/target/include/target/generator_loader.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildcc/lib/target/include/target/generator_loader.h b/buildcc/lib/target/include/target/generator_loader.h index 124e7e41..195147cd 100644 --- a/buildcc/lib/target/include/target/generator_loader.h +++ b/buildcc/lib/target/include/target/generator_loader.h @@ -25,8 +25,8 @@ namespace buildcc::internal { class GeneratorLoader : public LoaderInterface { public: - GeneratorLoader(const std::string &name, const fs::path &relative_path) - : name_(name), relative_path_(relative_path) {} + GeneratorLoader(const std::string &name, const fs::path &path) + : name_(name), path_(path) {} GeneratorLoader(const GeneratorLoader &loader) = delete; @@ -34,12 +34,12 @@ class GeneratorLoader : public LoaderInterface { // Getters fs::path GetBinaryPath() const override { - return relative_path_ / fmt::format("{}.bin", name_); + return path_ / fmt::format("{}.bin", name_); } private: std::string name_; - fs::path relative_path_; + fs::path path_; }; } // namespace buildcc::internal From b0d9027629e3296bf9ad8de6c1c4aac096c61a3f Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Tue, 24 Aug 2021 23:07:15 -0700 Subject: [PATCH 09/51] Added private schema_extract file --- .../include/target/private/schema_extract.h | 58 +++++++++++++++++++ .../lib/target/src/target/target_loader.cpp | 36 +----------- 2 files changed, 61 insertions(+), 33 deletions(-) create mode 100644 buildcc/lib/target/include/target/private/schema_extract.h diff --git a/buildcc/lib/target/include/target/private/schema_extract.h b/buildcc/lib/target/include/target/private/schema_extract.h new file mode 100644 index 00000000..e1b066f9 --- /dev/null +++ b/buildcc/lib/target/include/target/private/schema_extract.h @@ -0,0 +1,58 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TARGET_PRIVATE_SCHEMA_EXTRACT_H_ +#define TARGET_PRIVATE_SCHEMA_EXTRACT_H_ + +#include "path_generated.h" + +#include "target/path.h" + +namespace fbs = schema::internal; + +namespace buildcc::internal { + +inline void ExtractPath( + const flatbuffers::Vector> + *fbs_paths, + buildcc::internal::path_unordered_set &out) { + if (fbs_paths == nullptr) { + return; + } + + for (auto iter = fbs_paths->begin(); iter != fbs_paths->end(); iter++) { + out.insert(buildcc::internal::Path::CreateNewPath( + iter->pathname()->c_str(), iter->last_write_timestamp())); + } +} + +template +inline void +Extract(const flatbuffers::Vector> + *fbs_paths, + T &out) { + if (fbs_paths == nullptr) { + return; + } + + for (auto iter = fbs_paths->begin(); iter != fbs_paths->end(); iter++) { + out.insert(iter->str()); + } +} + +} // namespace buildcc::internal + +#endif diff --git a/buildcc/lib/target/src/target/target_loader.cpp b/buildcc/lib/target/src/target/target_loader.cpp index 16b7f119..018378d6 100644 --- a/buildcc/lib/target/src/target/target_loader.cpp +++ b/buildcc/lib/target/src/target/target_loader.cpp @@ -16,42 +16,12 @@ #include "target/target_loader.h" -#include "target_generated.h" - #include "env/logging.h" #include "env/util.h" -namespace fbs = schema::internal; - -namespace { -void ExtractPath( - const flatbuffers::Vector> - *fbs_paths, - buildcc::internal::path_unordered_set &out) { - if (fbs_paths == nullptr) { - return; - } - - for (auto iter = fbs_paths->begin(); iter != fbs_paths->end(); iter++) { - out.insert(buildcc::internal::Path::CreateNewPath( - iter->pathname()->c_str(), iter->last_write_timestamp())); - } -} - -template -void Extract(const flatbuffers::Vector> - *fbs_paths, - T &out) { - if (fbs_paths == nullptr) { - return; - } - - for (auto iter = fbs_paths->begin(); iter != fbs_paths->end(); iter++) { - out.insert(iter->str()); - } -} - -} // namespace +// Private +#include "target/private/schema_extract.h" +#include "target_generated.h" namespace buildcc::internal { From 0f424d5e1b8812e2cb74dbf80ef001abc91080ba Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 00:04:22 -0700 Subject: [PATCH 10/51] Updated schema_extract --- .../include/target/private/schema_extract.h | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/include/target/private/schema_extract.h b/buildcc/lib/target/include/target/private/schema_extract.h index e1b066f9..3d43bc78 100644 --- a/buildcc/lib/target/include/target/private/schema_extract.h +++ b/buildcc/lib/target/include/target/private/schema_extract.h @@ -39,11 +39,24 @@ inline void ExtractPath( } } +inline void +Extract(const flatbuffers::Vector> + *fbs_paths, + fs_unordered_set &out) { + if (fbs_paths == nullptr) { + return; + } + + for (auto iter = fbs_paths->begin(); iter != fbs_paths->end(); iter++) { + out.insert(iter->str()); + } +} + template inline void Extract(const flatbuffers::Vector> *fbs_paths, - T &out) { + std::unordered_set &out) { if (fbs_paths == nullptr) { return; } @@ -53,6 +66,20 @@ Extract(const flatbuffers::Vector> } } +template +inline void +Extract(const flatbuffers::Vector> + *fbs_paths, + std::vector &out) { + if (fbs_paths == nullptr) { + return; + } + + for (auto iter = fbs_paths->begin(); iter != fbs_paths->end(); iter++) { + out.push_back(iter->str()); + } +} + } // namespace buildcc::internal #endif From 000371dbccd848d88f00f0ecbe6bd28434a462b7 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 00:07:53 -0700 Subject: [PATCH 11/51] Update generator.fbs --- buildcc/schema/generator.fbs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildcc/schema/generator.fbs b/buildcc/schema/generator.fbs index b796c509..3cc63ab8 100644 --- a/buildcc/schema/generator.fbs +++ b/buildcc/schema/generator.fbs @@ -20,10 +20,10 @@ table Generator { name:string (key); // Input and output files are both tracked via timestamp - input:[Path]; - output:[Path]; + inputs:[Path]; + outputs:[Path]; - command:[string]; + commands:[string]; } root_type Generator; From d23787c40a9e6f9b399422ae883f5bb50172a89f Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 00:53:28 -0700 Subject: [PATCH 12/51] Updated generator_loader --- .../target/include/target/generator_loader.h | 6 ++++ .../target/src/generator/generator_loader.cpp | 31 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/include/target/generator_loader.h b/buildcc/lib/target/include/target/generator_loader.h index 195147cd..5b22dec2 100644 --- a/buildcc/lib/target/include/target/generator_loader.h +++ b/buildcc/lib/target/include/target/generator_loader.h @@ -21,6 +21,8 @@ #include "fmt/format.h" +#include "target/path.h" + namespace buildcc::internal { class GeneratorLoader : public LoaderInterface { @@ -40,6 +42,10 @@ class GeneratorLoader : public LoaderInterface { private: std::string name_; fs::path path_; + + path_unordered_set loaded_inputs_; + path_unordered_set loaded_outputs_; + std::vector loaded_commands_; }; } // namespace buildcc::internal diff --git a/buildcc/lib/target/src/generator/generator_loader.cpp b/buildcc/lib/target/src/generator/generator_loader.cpp index 636db61b..f470b3d1 100644 --- a/buildcc/lib/target/src/generator/generator_loader.cpp +++ b/buildcc/lib/target/src/generator/generator_loader.cpp @@ -16,10 +16,39 @@ #include "target/generator_loader.h" +#include "env/logging.h" +#include "env/util.h" + +// Private +#include "target/private/schema_extract.h" + +#include "generator_generated.h" + namespace buildcc::internal { bool GeneratorLoader::Load() { - // TODO, + env::log_trace(name_, __FUNCTION__); + auto file_path = GetBinaryPath(); + std::string buffer; + bool is_loaded = env::LoadFile(file_path.string().c_str(), true, &buffer); + if (!is_loaded) { + return false; + } + + flatbuffers::Verifier verifier((const uint8_t *)buffer.c_str(), + buffer.length()); + const bool is_verified = fbs::VerifyGeneratorBuffer(verifier); + if (!is_verified) { + return false; + } + + const auto *generator = fbs::GetGenerator((const void *)buffer.c_str()); + + ExtractPath(generator->inputs(), loaded_inputs_); + ExtractPath(generator->outputs(), loaded_outputs_); + Extract(generator->commands(), loaded_commands_); + loaded_ = true; + return true; } From f826dd4f07bb2bc88222236bf8f8ba9811c5266b Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 04:04:16 -0700 Subject: [PATCH 13/51] Updated current_object_files_ for target --- buildcc/lib/target/include/target/target.h | 2 +- buildcc/lib/target/src/target/source.cpp | 3 +-- buildcc/lib/target/src/target/target.cpp | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/buildcc/lib/target/include/target/target.h b/buildcc/lib/target/include/target/target.h index 6c539f30..87308a16 100644 --- a/buildcc/lib/target/include/target/target.h +++ b/buildcc/lib/target/include/target/target.h @@ -282,7 +282,7 @@ class Target { internal::path_unordered_set current_source_files_; // NOTE, Always store the absolute source path -> absolute compiled source // path here - std::unordered_map + std::unordered_map current_object_files_; internal::path_unordered_set current_header_files_; diff --git a/buildcc/lib/target/src/target/source.cpp b/buildcc/lib/target/src/target/source.cpp index 6ba19a9b..4cb85e9d 100644 --- a/buildcc/lib/target/src/target/source.cpp +++ b/buildcc/lib/target/src/target/source.cpp @@ -39,8 +39,7 @@ void Target::AddSourceAbsolute(const fs::path &absolute_input_filepath, const auto absolute_compiled_source = internal::Path::CreateNewPath(absolute_output_filepath); fs::create_directories(absolute_compiled_source.GetPathname().parent_path()); - current_object_files_.insert( - {absolute_source.native(), absolute_compiled_source}); + current_object_files_.insert({absolute_source, absolute_compiled_source}); } void Target::GlobSourcesAbsolute(const fs::path &absolute_input_path, diff --git a/buildcc/lib/target/src/target/target.cpp b/buildcc/lib/target/src/target/target.cpp index a652ad16..f96ed511 100644 --- a/buildcc/lib/target/src/target/target.cpp +++ b/buildcc/lib/target/src/target/target.cpp @@ -157,10 +157,10 @@ const std::string &Target::GetCompiler(const fs::path &source) const { const internal::Path & Target::GetCompiledSourcePath(const fs::path &source) const { - const auto fiter = current_object_files_.find(source.native()); + const auto fiter = current_object_files_.find(source); env::assert_fatal(fiter != current_object_files_.end(), fmt::format("{} not found", source.string())); - return current_object_files_.at(source.native()); + return current_object_files_.at(source); } internal::path_unordered_set Target::GetCompiledSources() const { From c5302549468978511b4dae17827ea3c75601d07a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 08:06:18 -0700 Subject: [PATCH 14/51] Added BuildInterface --- buildcc/lib/target/cmake/target.cmake | 10 ++-- .../target/include/target/builder_interface.h | 51 +++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 buildcc/lib/target/include/target/builder_interface.h diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 9dd48f6d..246fda92 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -1,27 +1,23 @@ set(TARGET_SRCS - src/generator/generator_loader.cpp + include/target/loader_interface.h + include/target/builder_interface.h + src/generator/generator_loader.cpp include/target/generator_loader.h src/util/util.cpp - src/target/target_loader.cpp src/target/target_storer.cpp - src/target/target.cpp src/target/source.cpp src/target/include_dir.cpp src/target/lib.cpp src/target/build.cpp src/target/flags.cpp - src/target/recheck_states.cpp src/target/tasks.cpp - include/target/path.h include/target/util.h - include/target/loader_interface.h - include/target/target_loader.h include/target/target.h ) diff --git a/buildcc/lib/target/include/target/builder_interface.h b/buildcc/lib/target/include/target/builder_interface.h new file mode 100644 index 00000000..e1dff01a --- /dev/null +++ b/buildcc/lib/target/include/target/builder_interface.h @@ -0,0 +1,51 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TARGET_BUILDER_INTERFACE_H_ +#define TARGET_BUILDER_INTERFACE_H_ + +#include + +#include "target/path.h" + +namespace buildcc::base { + +class BuilderInterface { + +public: + virtual void Build() = 0; + +protected: + void RecheckChanged(const std::unordered_set &previous, + const std::unordered_set ¤t, + const std::function &callback) { + if (dirty_) { + return; + } + + if (previous != current) { + callback(); + dirty_ = true; + } + } + +protected: + bool dirty_{false}; +}; + +} // namespace buildcc::base + +#endif From a1a14ae5ddf67c6d5afcd2d181cb701cb7d83a3d Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 08:07:05 -0700 Subject: [PATCH 15/51] Updated Target with initial BuildInterface --- buildcc/lib/target/include/target/target.h | 12 ++++-------- buildcc/lib/target/src/target/target.cpp | 13 ------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/buildcc/lib/target/include/target/target.h b/buildcc/lib/target/include/target/target.h index 87308a16..7009e666 100644 --- a/buildcc/lib/target/include/target/target.h +++ b/buildcc/lib/target/include/target/target.h @@ -26,6 +26,8 @@ #include #include +#include "target/builder_interface.h" + // Internal #include "target/path.h" #include "target/target_loader.h" @@ -55,7 +57,7 @@ enum class TargetType { DynamicLibrary, }; -class Target { +class Target : public BuilderInterface { public: // TODO, Consider making these std::string_view for string literals @@ -92,7 +94,7 @@ class Target { Target(const Target &target) = delete; // Builders - void Build(); + void Build() override; // Setters @@ -246,11 +248,6 @@ class Target { const std::unordered_set &previous_external_libs, const std::unordered_set ¤t_external_libs); - // Helper function - void RecheckChanged(const std::unordered_set &previous, - const std::unordered_set ¤t, - const std::function &callback); - // Tasks void CompileTask(); void LinkTask(); @@ -314,7 +311,6 @@ class Target { Command command_; // Build states - bool dirty_ = false; bool first_build_ = false; bool rebuild_ = false; diff --git a/buildcc/lib/target/src/target/target.cpp b/buildcc/lib/target/src/target/target.cpp index f96ed511..89f5b7a3 100644 --- a/buildcc/lib/target/src/target/target.cpp +++ b/buildcc/lib/target/src/target/target.cpp @@ -248,17 +248,4 @@ void Target::RecheckExternalLib( std::bind(&Target::ExternalLibChanged, this)); } -void Target::RecheckChanged(const std::unordered_set &previous, - const std::unordered_set ¤t, - const std::function &callback) { - if (dirty_) { - return; - } - - if (previous != current) { - callback(); - dirty_ = true; - } -} - } // namespace buildcc::base From f23936714331c725ae1d6eb3bc634e441188a0e0 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 08:14:02 -0700 Subject: [PATCH 16/51] Updated builder_interface RecheckChanged API to template - Updated target.cpp with RecheckChanged API --- buildcc/lib/target/include/target/builder_interface.h | 4 ++-- buildcc/lib/target/src/target/target.cpp | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/buildcc/lib/target/include/target/builder_interface.h b/buildcc/lib/target/include/target/builder_interface.h index e1dff01a..a9ee8e5e 100644 --- a/buildcc/lib/target/include/target/builder_interface.h +++ b/buildcc/lib/target/include/target/builder_interface.h @@ -29,8 +29,8 @@ class BuilderInterface { virtual void Build() = 0; protected: - void RecheckChanged(const std::unordered_set &previous, - const std::unordered_set ¤t, + template + void RecheckChanged(const T &previous, const T ¤t, const std::function &callback) { if (dirty_) { return; diff --git a/buildcc/lib/target/src/target/target.cpp b/buildcc/lib/target/src/target/target.cpp index 89f5b7a3..164e6f08 100644 --- a/buildcc/lib/target/src/target/target.cpp +++ b/buildcc/lib/target/src/target/target.cpp @@ -224,14 +224,8 @@ void Target::RecheckPaths(const internal::path_unordered_set &previous_path, void Target::RecheckDirs(const internal::fs_unordered_set &previous_dirs, const internal::fs_unordered_set ¤t_dirs) { - if (dirty_) { - return; - } - - if (previous_dirs != current_dirs) { - DirChanged(); - dirty_ = true; - } + RecheckChanged(previous_dirs, current_dirs, + std::bind(&Target::DirChanged, this)); } void Target::RecheckFlags( From aec6ed7652ace902f8bb7f71691f85b6e4e5ccbc Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 19:44:02 -0700 Subject: [PATCH 17/51] Update target.cmake --- buildcc/lib/target/cmake/target.cmake | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 246fda92..55d33a3c 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -1,11 +1,18 @@ set(TARGET_SRCS + # Interfaces include/target/loader_interface.h include/target/builder_interface.h + # Utils + src/util/util.cpp + include/target/path.h + include/target/util.h + + # Generator src/generator/generator_loader.cpp include/target/generator_loader.h - src/util/util.cpp + # Target src/target/target_loader.cpp src/target/target_storer.cpp src/target/target.cpp @@ -16,8 +23,6 @@ set(TARGET_SRCS src/target/flags.cpp src/target/recheck_states.cpp src/target/tasks.cpp - include/target/path.h - include/target/util.h include/target/target_loader.h include/target/target.h ) From bd643e84665e2333b1822faf1429c5a220615214 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 25 Aug 2021 19:53:42 -0700 Subject: [PATCH 18/51] Updated build_interface - RecheckPaths - virtual Store - Updated RecheckChanged API with default std::function --- .../target/include/target/builder_interface.h | 50 ++++++++++++++++++- buildcc/lib/target/include/target/target.h | 2 +- buildcc/lib/target/src/target/target.cpp | 38 ++------------ 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/buildcc/lib/target/include/target/builder_interface.h b/buildcc/lib/target/include/target/builder_interface.h index a9ee8e5e..43e66221 100644 --- a/buildcc/lib/target/include/target/builder_interface.h +++ b/buildcc/lib/target/include/target/builder_interface.h @@ -20,6 +20,7 @@ #include #include "target/path.h" +#include "target/util.h" namespace buildcc::base { @@ -30,8 +31,9 @@ class BuilderInterface { protected: template - void RecheckChanged(const T &previous, const T ¤t, - const std::function &callback) { + void RecheckChanged( + const T &previous, const T ¤t, + const std::function &callback = []() {}) { if (dirty_) { return; } @@ -42,6 +44,50 @@ class BuilderInterface { } } + void RecheckPaths(const internal::path_unordered_set &previous_path, + const internal::path_unordered_set ¤t_path, + const std::function &path_removed_cb, + const std::function &path_added_cb, + const std::function &path_updated_cb) { + if (dirty_) { + return; + } + + // * Old path is removed + const bool removed = + internal::is_previous_paths_different(previous_path, current_path); + if (removed) { + path_removed_cb(); + dirty_ = true; + return; + } + + for (const auto &path : current_path) { + auto iter = previous_path.find(path); + + if (iter == previous_path.end()) { + // * New path added + path_added_cb(); + dirty_ = true; + } else { + // * Path is updated + if (path.GetLastWriteTimestamp() > iter->GetLastWriteTimestamp()) { + path_updated_cb(); + dirty_ = true; + } else { + // * Do nothing + } + } + + if (dirty_) { + break; + } + } + } + +private: + virtual bool Store() = 0; + protected: bool dirty_{false}; }; diff --git a/buildcc/lib/target/include/target/target.h b/buildcc/lib/target/include/target/target.h index 7009e666..6f7a4d9e 100644 --- a/buildcc/lib/target/include/target/target.h +++ b/buildcc/lib/target/include/target/target.h @@ -253,7 +253,7 @@ class Target : public BuilderInterface { void LinkTask(); // Fbs - bool Store(); + bool Store() override; // Callbacks void SourceRemoved(); diff --git a/buildcc/lib/target/src/target/target.cpp b/buildcc/lib/target/src/target/target.cpp index 164e6f08..a2e92b7b 100644 --- a/buildcc/lib/target/src/target/target.cpp +++ b/buildcc/lib/target/src/target/target.cpp @@ -185,41 +185,9 @@ void Target::Initialize() { // Rechecks void Target::RecheckPaths(const internal::path_unordered_set &previous_path, const internal::path_unordered_set ¤t_path) { - // * Compile sources / Target already requires rebuild - if (dirty_) { - return; - } - - // * Old path is removed - const bool removed = - internal::is_previous_paths_different(previous_path, current_path); - if (removed) { - PathRemoved(); - dirty_ = true; - return; - } - - for (const auto &path : current_path) { - auto iter = previous_path.find(path); - - if (iter == previous_path.end()) { - // * New path added - PathAdded(); - dirty_ = true; - } else { - // * Path is updated - if (path.GetLastWriteTimestamp() > iter->GetLastWriteTimestamp()) { - PathUpdated(); - dirty_ = true; - } else { - // * Do nothing - } - } - - if (dirty_) { - break; - } - } + BuilderInterface::RecheckPaths( + previous_path, current_path, [&]() { PathRemoved(); }, + [&]() { PathAdded(); }, [&]() { PathUpdated(); }); } void Target::RecheckDirs(const internal::fs_unordered_set &previous_dirs, From 4594ce8298603488b85e5348d389c8a23f03a539 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 01:42:30 -0700 Subject: [PATCH 19/51] Updated generator.fbs with GenInfo table Stores relational information for inputs, ouputs and commands --- buildcc/schema/generator.fbs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/buildcc/schema/generator.fbs b/buildcc/schema/generator.fbs index 3cc63ab8..2f729023 100644 --- a/buildcc/schema/generator.fbs +++ b/buildcc/schema/generator.fbs @@ -16,14 +16,16 @@ include "path.fbs"; namespace schema.internal; -table Generator { - name:string (key); - - // Input and output files are both tracked via timestamp +table GenInfo { inputs:[Path]; - outputs:[Path]; - + outputs:[string]; commands:[string]; } +// Each generator consists of many relational files of [input] - [output] - [commands] +table Generator { + name:string (key); + info:[GenInfo]; +} + root_type Generator; From f1bc281746643795c93106f428c123a59b50fa73 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:33:36 -0700 Subject: [PATCH 20/51] Update generator.fbs - Added name key and parallel to GenInfo --- buildcc/schema/generator.fbs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildcc/schema/generator.fbs b/buildcc/schema/generator.fbs index 2f729023..0eef1713 100644 --- a/buildcc/schema/generator.fbs +++ b/buildcc/schema/generator.fbs @@ -17,9 +17,11 @@ include "path.fbs"; namespace schema.internal; table GenInfo { + name:string (key); inputs:[Path]; outputs:[string]; commands:[string]; + parallel:bool; } // Each generator consists of many relational files of [input] - [output] - [commands] From 77f25a05289221bf025f758e6538c62c03431a92 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:35:13 -0700 Subject: [PATCH 21/51] Updated generator_loader with updated generator.fbs - Added GenInfo --- .../target/include/target/generator_loader.h | 28 ++++++++++++++++--- .../target/src/generator/generator_loader.cpp | 17 ++++++++--- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/buildcc/lib/target/include/target/generator_loader.h b/buildcc/lib/target/include/target/generator_loader.h index 5b22dec2..fe657c0c 100644 --- a/buildcc/lib/target/include/target/generator_loader.h +++ b/buildcc/lib/target/include/target/generator_loader.h @@ -17,7 +17,11 @@ #ifndef TARGET_GENERATOR_LOADER_H_ #define TARGET_GENERATOR_LOADER_H_ -#include "loader_interface.h" +#include "target/loader_interface.h" + +#include +#include +#include #include "fmt/format.h" @@ -25,6 +29,22 @@ namespace buildcc::internal { +struct GenInfo { + std::string name; + path_unordered_set inputs; + std::unordered_set outputs; + std::vector commands; + bool parallel{false}; + + GenInfo() {} + GenInfo(const std::string &n, const path_unordered_set &i, + const std::unordered_set &o, + const std::vector &c, bool p) + : name(n), inputs(i), outputs(o), commands(c), parallel(p) {} +}; + +typedef std::unordered_map geninfo_unordered_map; + class GeneratorLoader : public LoaderInterface { public: GeneratorLoader(const std::string &name, const fs::path &path) @@ -39,13 +59,13 @@ class GeneratorLoader : public LoaderInterface { return path_ / fmt::format("{}.bin", name_); } + const geninfo_unordered_map &GetLoadedInfo() { return loaded_info_; } + private: std::string name_; fs::path path_; - path_unordered_set loaded_inputs_; - path_unordered_set loaded_outputs_; - std::vector loaded_commands_; + geninfo_unordered_map loaded_info_; }; } // namespace buildcc::internal diff --git a/buildcc/lib/target/src/generator/generator_loader.cpp b/buildcc/lib/target/src/generator/generator_loader.cpp index f470b3d1..7b1ddac1 100644 --- a/buildcc/lib/target/src/generator/generator_loader.cpp +++ b/buildcc/lib/target/src/generator/generator_loader.cpp @@ -28,6 +28,7 @@ namespace buildcc::internal { bool GeneratorLoader::Load() { env::log_trace(name_, __FUNCTION__); + auto file_path = GetBinaryPath(); std::string buffer; bool is_loaded = env::LoadFile(file_path.string().c_str(), true, &buffer); @@ -44,11 +45,19 @@ bool GeneratorLoader::Load() { const auto *generator = fbs::GetGenerator((const void *)buffer.c_str()); - ExtractPath(generator->inputs(), loaded_inputs_); - ExtractPath(generator->outputs(), loaded_outputs_); - Extract(generator->commands(), loaded_commands_); - loaded_ = true; + const auto *info = generator->info(); + for (auto iter = info->cbegin(); iter != info->cend(); iter++) { + path_unordered_set i; + std::unordered_set o; + std::vector c; + ExtractPath(iter->inputs(), i); + Extract(iter->outputs(), o); + Extract(iter->commands(), c); + loaded_info_[iter->name()->c_str()] = + GenInfo(iter->name()->c_str(), i, o, c, iter->parallel()); + } + loaded_ = true; return true; } From f6624a93e20aef7023ffdbbcaa5711357992df99 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:40:17 -0700 Subject: [PATCH 22/51] Added initial generator files --- buildcc/lib/target/cmake/target.cmake | 3 + buildcc/lib/target/include/target/generator.h | 64 +++++++++++ .../lib/target/src/generator/generator.cpp | 108 ++++++++++++++++++ .../target/src/generator/generator_storer.cpp | 29 +++++ 4 files changed, 204 insertions(+) create mode 100644 buildcc/lib/target/include/target/generator.h create mode 100644 buildcc/lib/target/src/generator/generator.cpp create mode 100644 buildcc/lib/target/src/generator/generator_storer.cpp diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 55d33a3c..b4d51c54 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -10,7 +10,10 @@ set(TARGET_SRCS # Generator src/generator/generator_loader.cpp + src/generator/generator_storer.cpp + src/generator/generator.cpp include/target/generator_loader.h + include/target/generator.h # Target src/target/target_loader.cpp diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h new file mode 100644 index 00000000..a516eade --- /dev/null +++ b/buildcc/lib/target/include/target/generator.h @@ -0,0 +1,64 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TARGET_GENERATOR_H_ +#define TARGET_GENERATOR_H_ + +#include +#include +#include + +#include "taskflow/taskflow.hpp" + +#include "env/env.h" + +#include "target/builder_interface.h" + +#include "target/generator_loader.h" +#include "target/path.h" + +namespace buildcc::base { + +class Generator : public BuilderInterface { +public: + Generator(const std::string &name, const fs::path &path) + : loader_(name, path) {} + Generator(const Generator &generator) = delete; + + void AddGenInfo(const internal::GenInfo &info); + + void Build() override; + + // Getter + tf::Taskflow &GetTaskflow() { return tf_; } + +private: + std::vector BuildGenerate(); + bool Store() override; + +private: + std::string name_; + internal::geninfo_unordered_map current_info_; + + internal::GeneratorLoader loader_; + + tf::Taskflow tf_; + tf::Task build_task_; +}; + +} // namespace buildcc::base + +#endif diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp new file mode 100644 index 00000000..6e06317d --- /dev/null +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -0,0 +1,108 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/generator.h" + +#include + +#include "env/assert_fatal.h" + +#include "command/command.h" + +namespace buildcc::base { + +void Generator::AddGenInfo(const internal::GenInfo &info) { + env::assert_fatal( + current_info_.find(info.name) == current_info_.end(), + fmt::format("'{}' information already registered", info.name)); + current_info_[info.name] = info; +} + +void Generator::Build() { + // TODO, Handle parallel case + build_task_ = tf_.emplace([&](tf::Subflow &subflow) { + const auto &generated_files = BuildGenerate(); + if (!dirty_) { + return; + } + + for (const auto &info : generated_files) { + if (info->parallel) { + subflow + .for_each(info->commands.cbegin(), info->commands.cend(), + [](const std::string &command) { + bool success = Command::Execute(command); + env::assert_fatal(success, + fmt::format("{} failed", command)); + }) + .name(info->name); + } else { + subflow + .emplace([&]() { + for (const auto &command : info->commands) { + bool success = Command::Execute(command); + env::assert_fatal(success, fmt::format("{} failed", command)); + } + }) + .name(info->name); + } + } + Store(); + }); + build_task_.name(fmt::format("BuildTask:{}", name_)); +} + +// PRIVATE + +std::vector Generator::BuildGenerate() { + const bool loaded = loader_.Load(); + + std::vector generated_files; + if (!loaded) { + std::for_each(current_info_.cbegin(), current_info_.cend(), + [&](const std::pair &p) { + generated_files.push_back(&(p.second)); + }); + } else { + std::for_each( + current_info_.cbegin(), current_info_.cend(), + [&](const std::pair &p) { + // Recheck inputs for change in timestamp + RecheckPaths( + p.second.inputs, current_info_[p.first].inputs, []() {}, []() {}, + []() {}); + + // Recheck presence of outputs + RecheckChanged(p.second.outputs, current_info_[p.first].outputs); + + // Recheck change in commands + RecheckChanged(p.second.commands, current_info_[p.first].commands); + + if (dirty_) { + generated_files.push_back(&(p.second)); + } + dirty_ = false; + }); + } + + if (!generated_files.empty()) { + dirty_ = true; + } + + return generated_files; +} + +} // namespace buildcc::base diff --git a/buildcc/lib/target/src/generator/generator_storer.cpp b/buildcc/lib/target/src/generator/generator_storer.cpp new file mode 100644 index 00000000..a22c49d2 --- /dev/null +++ b/buildcc/lib/target/src/generator/generator_storer.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/generator.h" + +namespace buildcc::base { + +bool Generator::Store() { + env::log_trace(name_, __FUNCTION__); + + // TODO, + + return true; +} + +} // namespace buildcc::base From 0d71d54ec66a2391181ebc663cae279715bb7743 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:40:35 -0700 Subject: [PATCH 23/51] Removed fs from target.h --- buildcc/lib/target/include/target/target.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/buildcc/lib/target/include/target/target.h b/buildcc/lib/target/include/target/target.h index 6f7a4d9e..ed25deaa 100644 --- a/buildcc/lib/target/include/target/target.h +++ b/buildcc/lib/target/include/target/target.h @@ -41,8 +41,6 @@ namespace buildcc::base { -namespace fs = std::filesystem; - enum class FileExtType { Asm, C, From f0f147a31f24858997ddcbad0352afee7691b4c2 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:46:20 -0700 Subject: [PATCH 24/51] Update generator.cpp --- buildcc/lib/target/src/generator/generator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index 6e06317d..ceccd1c4 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -34,7 +34,7 @@ void Generator::AddGenInfo(const internal::GenInfo &info) { void Generator::Build() { // TODO, Handle parallel case build_task_ = tf_.emplace([&](tf::Subflow &subflow) { - const auto &generated_files = BuildGenerate(); + const auto generated_files = BuildGenerate(); if (!dirty_) { return; } From c35e4d30602d5735ca7231ef228cc1d0ce06dc9f Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:50:41 -0700 Subject: [PATCH 25/51] Renamed schema_extract to schema_util --- .../target/private/{schema_extract.h => schema_util.h} | 4 ++-- buildcc/lib/target/src/generator/generator_loader.cpp | 2 +- buildcc/lib/target/src/target/target_loader.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename buildcc/lib/target/include/target/private/{schema_extract.h => schema_util.h} (96%) diff --git a/buildcc/lib/target/include/target/private/schema_extract.h b/buildcc/lib/target/include/target/private/schema_util.h similarity index 96% rename from buildcc/lib/target/include/target/private/schema_extract.h rename to buildcc/lib/target/include/target/private/schema_util.h index 3d43bc78..f3478c83 100644 --- a/buildcc/lib/target/include/target/private/schema_extract.h +++ b/buildcc/lib/target/include/target/private/schema_util.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef TARGET_PRIVATE_SCHEMA_EXTRACT_H_ -#define TARGET_PRIVATE_SCHEMA_EXTRACT_H_ +#ifndef TARGET_PRIVATE_SCHEMA_UTIL_H_ +#define TARGET_PRIVATE_SCHEMA_UTIL_H_ #include "path_generated.h" diff --git a/buildcc/lib/target/src/generator/generator_loader.cpp b/buildcc/lib/target/src/generator/generator_loader.cpp index 7b1ddac1..a6a069e5 100644 --- a/buildcc/lib/target/src/generator/generator_loader.cpp +++ b/buildcc/lib/target/src/generator/generator_loader.cpp @@ -20,7 +20,7 @@ #include "env/util.h" // Private -#include "target/private/schema_extract.h" +#include "target/private/schema_util.h" #include "generator_generated.h" diff --git a/buildcc/lib/target/src/target/target_loader.cpp b/buildcc/lib/target/src/target/target_loader.cpp index 018378d6..71acaa70 100644 --- a/buildcc/lib/target/src/target/target_loader.cpp +++ b/buildcc/lib/target/src/target/target_loader.cpp @@ -20,7 +20,7 @@ #include "env/util.h" // Private -#include "target/private/schema_extract.h" +#include "target/private/schema_util.h" #include "target_generated.h" namespace buildcc::internal { From 2bf4cd49abc1383da3e9f044c0c2fa0782c121ff Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 18:58:52 -0700 Subject: [PATCH 26/51] Shifted store APIs to schema_util.h --- .../include/target/private/schema_util.h | 30 +++++++++++++++ .../lib/target/src/target/target_storer.cpp | 38 ++++--------------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/buildcc/lib/target/include/target/private/schema_util.h b/buildcc/lib/target/include/target/private/schema_util.h index f3478c83..d9ea2c42 100644 --- a/buildcc/lib/target/include/target/private/schema_util.h +++ b/buildcc/lib/target/include/target/private/schema_util.h @@ -19,12 +19,16 @@ #include "path_generated.h" +#include + #include "target/path.h" namespace fbs = schema::internal; namespace buildcc::internal { +// Extract APIs for LOAD + inline void ExtractPath( const flatbuffers::Vector> *fbs_paths, @@ -80,6 +84,32 @@ Extract(const flatbuffers::Vector> } } +// + +inline std::vector> +get_fbs_vector_path(flatbuffers::FlatBufferBuilder &builder, + const buildcc::internal::path_unordered_set &pathlist) { + std::vector> paths; + for (const auto &p : pathlist) { + auto fbs_file = fbs::CreatePathDirect( + builder, p.GetPathname().string().c_str(), p.GetLastWriteTimestamp()); + paths.push_back(fbs_file); + } + return paths; +} + +inline std::vector> +get_fbs_vector_string(flatbuffers::FlatBufferBuilder &builder, + const std::unordered_set &strlist) { + std::vector> strs; + std::transform( + strlist.begin(), strlist.end(), std::back_inserter(strs), + [&](const std::string &str) -> flatbuffers::Offset { + return builder.CreateString(str); + }); + return strs; +} + } // namespace buildcc::internal #endif diff --git a/buildcc/lib/target/src/target/target_storer.cpp b/buildcc/lib/target/src/target/target_storer.cpp index 6cc32e30..3143b5b6 100644 --- a/buildcc/lib/target/src/target/target_storer.cpp +++ b/buildcc/lib/target/src/target/target_storer.cpp @@ -22,6 +22,7 @@ #include "env/logging.h" #include "env/util.h" +#include "target/private/schema_util.h" #include "target_generated.h" namespace fbs = schema::internal; @@ -32,30 +33,6 @@ fbs::TargetType get_fbs_target_type(buildcc::base::TargetType type) { return (fbs::TargetType)type; } -std::vector> -get_fbs_vector_path(flatbuffers::FlatBufferBuilder &builder, - const buildcc::internal::path_unordered_set &pathlist) { - std::vector> paths; - for (const auto &p : pathlist) { - auto fbs_file = fbs::CreatePathDirect( - builder, p.GetPathname().string().c_str(), p.GetLastWriteTimestamp()); - paths.push_back(fbs_file); - } - return paths; -} - -std::vector> -get_fbs_vector_string(flatbuffers::FlatBufferBuilder &builder, - const std::unordered_set &strlist) { - std::vector> strs; - std::transform( - strlist.begin(), strlist.end(), std::back_inserter(strs), - [&](const std::string &str) -> flatbuffers::Offset { - return builder.CreateString(str); - }); - return strs; -} - std::vector> get_fbs_vector_string(flatbuffers::FlatBufferBuilder &builder, const buildcc::internal::fs_unordered_set &fslist) { @@ -84,20 +61,21 @@ bool Target::Store() { auto fbs_lib_deps = get_fbs_vector_path(builder, current_lib_deps_); auto fbs_external_lib_deps = - get_fbs_vector_string(builder, current_external_lib_deps_); + internal::get_fbs_vector_string(builder, current_external_lib_deps_); auto fbs_include_dirs = get_fbs_vector_string(builder, current_include_dirs_); auto fbs_lib_dirs = get_fbs_vector_string(builder, current_lib_dirs_); auto fbs_preprocessor_flags = - get_fbs_vector_string(builder, current_preprocessor_flags_); + internal::get_fbs_vector_string(builder, current_preprocessor_flags_); auto fbs_common_compile_flags = - get_fbs_vector_string(builder, current_common_compile_flags_); + internal::get_fbs_vector_string(builder, current_common_compile_flags_); auto fbs_c_compile_flags = - get_fbs_vector_string(builder, current_c_compile_flags_); + internal::get_fbs_vector_string(builder, current_c_compile_flags_); auto fbs_cpp_compile_flags = - get_fbs_vector_string(builder, current_cpp_compile_flags_); - auto fbs_link_flags = get_fbs_vector_string(builder, current_link_flags_); + internal::get_fbs_vector_string(builder, current_cpp_compile_flags_); + auto fbs_link_flags = + internal::get_fbs_vector_string(builder, current_link_flags_); auto fbs_compile_dependencies = get_fbs_vector_path(builder, current_compile_dependencies_); From 4b53b4260c3b6a678d79b0480de214275e706111 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 19:04:25 -0700 Subject: [PATCH 27/51] Updated naming convention for private APIs --- .../include/target/private/schema_util.h | 18 ++++++-- .../lib/target/src/target/target_storer.cpp | 46 ++++++++----------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/buildcc/lib/target/include/target/private/schema_util.h b/buildcc/lib/target/include/target/private/schema_util.h index d9ea2c42..7b73526f 100644 --- a/buildcc/lib/target/include/target/private/schema_util.h +++ b/buildcc/lib/target/include/target/private/schema_util.h @@ -84,10 +84,10 @@ Extract(const flatbuffers::Vector> } } -// +// Create APIs for STORE inline std::vector> -get_fbs_vector_path(flatbuffers::FlatBufferBuilder &builder, +CreateFbsVectorPath(flatbuffers::FlatBufferBuilder &builder, const buildcc::internal::path_unordered_set &pathlist) { std::vector> paths; for (const auto &p : pathlist) { @@ -99,7 +99,7 @@ get_fbs_vector_path(flatbuffers::FlatBufferBuilder &builder, } inline std::vector> -get_fbs_vector_string(flatbuffers::FlatBufferBuilder &builder, +CreateFbsVectorString(flatbuffers::FlatBufferBuilder &builder, const std::unordered_set &strlist) { std::vector> strs; std::transform( @@ -110,6 +110,18 @@ get_fbs_vector_string(flatbuffers::FlatBufferBuilder &builder, return strs; } +inline std::vector> +CreateFbsVectorString(flatbuffers::FlatBufferBuilder &builder, + const buildcc::internal::fs_unordered_set &fslist) { + std::vector> strs; + std::transform( + fslist.begin(), fslist.end(), std::back_inserter(strs), + [&](const fs::path &p) -> flatbuffers::Offset { + return builder.CreateString(p.string()); + }); + return strs; +} + } // namespace buildcc::internal #endif diff --git a/buildcc/lib/target/src/target/target_storer.cpp b/buildcc/lib/target/src/target/target_storer.cpp index 3143b5b6..e2903b00 100644 --- a/buildcc/lib/target/src/target/target_storer.cpp +++ b/buildcc/lib/target/src/target/target_storer.cpp @@ -29,22 +29,10 @@ namespace fbs = schema::internal; namespace { -fbs::TargetType get_fbs_target_type(buildcc::base::TargetType type) { +fbs::TargetType CreateFbsTargetType(buildcc::base::TargetType type) { return (fbs::TargetType)type; } -std::vector> -get_fbs_vector_string(flatbuffers::FlatBufferBuilder &builder, - const buildcc::internal::fs_unordered_set &fslist) { - std::vector> strs; - std::transform( - fslist.begin(), fslist.end(), std::back_inserter(strs), - [&](const fs::path &p) -> flatbuffers::Offset { - return builder.CreateString(p.string()); - }); - return strs; -} - } // namespace namespace buildcc::base { @@ -54,33 +42,37 @@ bool Target::Store() { flatbuffers::FlatBufferBuilder builder; - auto fbs_target_type = get_fbs_target_type(type_); + auto fbs_target_type = CreateFbsTargetType(type_); - auto fbs_source_files = get_fbs_vector_path(builder, current_source_files_); - auto fbs_header_files = get_fbs_vector_path(builder, current_header_files_); - auto fbs_lib_deps = get_fbs_vector_path(builder, current_lib_deps_); + auto fbs_source_files = + internal::CreateFbsVectorPath(builder, current_source_files_); + auto fbs_header_files = + internal::CreateFbsVectorPath(builder, current_header_files_); + auto fbs_lib_deps = internal::CreateFbsVectorPath(builder, current_lib_deps_); auto fbs_external_lib_deps = - internal::get_fbs_vector_string(builder, current_external_lib_deps_); + internal::CreateFbsVectorString(builder, current_external_lib_deps_); - auto fbs_include_dirs = get_fbs_vector_string(builder, current_include_dirs_); - auto fbs_lib_dirs = get_fbs_vector_string(builder, current_lib_dirs_); + auto fbs_include_dirs = + internal::CreateFbsVectorString(builder, current_include_dirs_); + auto fbs_lib_dirs = + internal::CreateFbsVectorString(builder, current_lib_dirs_); auto fbs_preprocessor_flags = - internal::get_fbs_vector_string(builder, current_preprocessor_flags_); + internal::CreateFbsVectorString(builder, current_preprocessor_flags_); auto fbs_common_compile_flags = - internal::get_fbs_vector_string(builder, current_common_compile_flags_); + internal::CreateFbsVectorString(builder, current_common_compile_flags_); auto fbs_c_compile_flags = - internal::get_fbs_vector_string(builder, current_c_compile_flags_); + internal::CreateFbsVectorString(builder, current_c_compile_flags_); auto fbs_cpp_compile_flags = - internal::get_fbs_vector_string(builder, current_cpp_compile_flags_); + internal::CreateFbsVectorString(builder, current_cpp_compile_flags_); auto fbs_link_flags = - internal::get_fbs_vector_string(builder, current_link_flags_); + internal::CreateFbsVectorString(builder, current_link_flags_); auto fbs_compile_dependencies = - get_fbs_vector_path(builder, current_compile_dependencies_); + internal::CreateFbsVectorPath(builder, current_compile_dependencies_); auto fbs_link_dependencies = - get_fbs_vector_path(builder, current_link_dependencies_); + internal::CreateFbsVectorPath(builder, current_link_dependencies_); auto fbs_target = fbs::CreateTargetDirect( builder, name_.c_str(), fbs_target_type, &fbs_source_files, From 55e396a6f90144050bf08fd8e08f2564de71bdc6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 19:24:51 -0700 Subject: [PATCH 28/51] Updated generator_storer.cpp --- buildcc/lib/target/include/target/generator.h | 1 + .../include/target/private/schema_util.h | 3 +- .../target/src/generator/generator_storer.cpp | 34 +++++++++++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index a516eade..88662236 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -43,6 +43,7 @@ class Generator : public BuilderInterface { void Build() override; // Getter + fs::path GetBinaryPath() const { return loader_.GetBinaryPath(); } tf::Taskflow &GetTaskflow() { return tf_; } private: diff --git a/buildcc/lib/target/include/target/private/schema_util.h b/buildcc/lib/target/include/target/private/schema_util.h index 7b73526f..0fd15a1c 100644 --- a/buildcc/lib/target/include/target/private/schema_util.h +++ b/buildcc/lib/target/include/target/private/schema_util.h @@ -98,9 +98,10 @@ CreateFbsVectorPath(flatbuffers::FlatBufferBuilder &builder, return paths; } +template inline std::vector> CreateFbsVectorString(flatbuffers::FlatBufferBuilder &builder, - const std::unordered_set &strlist) { + const T &strlist) { std::vector> strs; std::transform( strlist.begin(), strlist.end(), std::back_inserter(strs), diff --git a/buildcc/lib/target/src/generator/generator_storer.cpp b/buildcc/lib/target/src/generator/generator_storer.cpp index a22c49d2..833e06b2 100644 --- a/buildcc/lib/target/src/generator/generator_storer.cpp +++ b/buildcc/lib/target/src/generator/generator_storer.cpp @@ -16,14 +16,44 @@ #include "target/generator.h" +#include "flatbuffers/flatbuffers.h" + +#include "env/util.h" + +#include "generator_generated.h" +#include "target/private/schema_util.h" + +namespace fbs = schema::internal; + namespace buildcc::base { bool Generator::Store() { env::log_trace(name_, __FUNCTION__); - // TODO, + flatbuffers::FlatBufferBuilder builder; + + std::vector> fbs_generator_list; + for (const auto &info : current_info_) { + const auto &geninfo = info.second; + auto fbs_inputs = internal::CreateFbsVectorPath(builder, geninfo.inputs); + auto fbs_outputs = + internal::CreateFbsVectorString(builder, geninfo.outputs); + auto fbs_commands = + internal::CreateFbsVectorString(builder, geninfo.commands); + auto fbs_geninfo = + fbs::CreateGenInfoDirect(builder, geninfo.name.c_str(), &fbs_inputs, + &fbs_outputs, &fbs_commands, geninfo.parallel); + fbs_generator_list.push_back(fbs_geninfo); + } + + auto fbs_generator = + fbs::CreateGeneratorDirect(builder, name_.c_str(), &fbs_generator_list); + fbs::FinishGeneratorBuffer(builder, fbs_generator); - return true; + auto file_path = GetBinaryPath(); + return env::SaveFile(file_path.string().c_str(), + (const char *)builder.GetBufferPointer(), + builder.GetSize(), true); } } // namespace buildcc::base From 7592f39c2498e3421500ae670d3dab178ad76153 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 23:02:43 -0700 Subject: [PATCH 29/51] Added task.cpp for unit tests --- buildcc/lib/target/cmake/target.cmake | 1 + buildcc/lib/target/include/target/generator.h | 1 + .../lib/target/src/generator/generator.cpp | 36 +----------- buildcc/lib/target/src/generator/task.cpp | 56 +++++++++++++++++++ 4 files changed, 59 insertions(+), 35 deletions(-) create mode 100644 buildcc/lib/target/src/generator/task.cpp diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index b4d51c54..9d476908 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -12,6 +12,7 @@ set(TARGET_SRCS src/generator/generator_loader.cpp src/generator/generator_storer.cpp src/generator/generator.cpp + src/generator/task.cpp include/target/generator_loader.h include/target/generator.h diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 88662236..d2a7b92b 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -48,6 +48,7 @@ class Generator : public BuilderInterface { private: std::vector BuildGenerate(); + void GenerateTask(); bool Store() override; private: diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index ceccd1c4..f783d6fc 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -20,8 +20,6 @@ #include "env/assert_fatal.h" -#include "command/command.h" - namespace buildcc::base { void Generator::AddGenInfo(const internal::GenInfo &info) { @@ -31,39 +29,7 @@ void Generator::AddGenInfo(const internal::GenInfo &info) { current_info_[info.name] = info; } -void Generator::Build() { - // TODO, Handle parallel case - build_task_ = tf_.emplace([&](tf::Subflow &subflow) { - const auto generated_files = BuildGenerate(); - if (!dirty_) { - return; - } - - for (const auto &info : generated_files) { - if (info->parallel) { - subflow - .for_each(info->commands.cbegin(), info->commands.cend(), - [](const std::string &command) { - bool success = Command::Execute(command); - env::assert_fatal(success, - fmt::format("{} failed", command)); - }) - .name(info->name); - } else { - subflow - .emplace([&]() { - for (const auto &command : info->commands) { - bool success = Command::Execute(command); - env::assert_fatal(success, fmt::format("{} failed", command)); - } - }) - .name(info->name); - } - } - Store(); - }); - build_task_.name(fmt::format("BuildTask:{}", name_)); -} +void Generator::Build() { GenerateTask(); } // PRIVATE diff --git a/buildcc/lib/target/src/generator/task.cpp b/buildcc/lib/target/src/generator/task.cpp new file mode 100644 index 00000000..4b96cca6 --- /dev/null +++ b/buildcc/lib/target/src/generator/task.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/generator.h" + +#include "command/command.h" + +namespace buildcc::base { + +void Generator::GenerateTask() { + build_task_ = tf_.emplace([&](tf::Subflow &subflow) { + const auto generated_files = BuildGenerate(); + if (!dirty_) { + return; + } + + for (const auto &info : generated_files) { + if (info->parallel) { + subflow + .for_each(info->commands.cbegin(), info->commands.cend(), + [](const std::string &command) { + bool success = Command::Execute(command); + env::assert_fatal(success, + fmt::format("{} failed", command)); + }) + .name(info->name); + } else { + subflow + .emplace([&]() { + for (const auto &command : info->commands) { + bool success = Command::Execute(command); + env::assert_fatal(success, fmt::format("{} failed", command)); + } + }) + .name(info->name); + } + } + Store(); + }); + build_task_.name(fmt::format("BuildTask:{}", name_)); +} + +} // namespace buildcc::base From d51cfcbed3097b6b99f789c9f0bd8f8acc8ecd4d Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Thu, 26 Aug 2021 23:27:35 -0700 Subject: [PATCH 30/51] Updated BuildGenerate logic --- .../lib/target/src/generator/generator.cpp | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index f783d6fc..7a0ca906 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -37,37 +37,38 @@ std::vector Generator::BuildGenerate() { const bool loaded = loader_.Load(); std::vector generated_files; + bool build = false; if (!loaded) { std::for_each(current_info_.cbegin(), current_info_.cend(), [&](const std::pair &p) { generated_files.push_back(&(p.second)); }); + build = true; } else { + const auto &previous_info = loader_.GetLoadedInfo(); std::for_each( current_info_.cbegin(), current_info_.cend(), [&](const std::pair &p) { - // Recheck inputs for change in timestamp - RecheckPaths( - p.second.inputs, current_info_[p.first].inputs, []() {}, []() {}, - []() {}); - - // Recheck presence of outputs - RecheckChanged(p.second.outputs, current_info_[p.first].outputs); - - // Recheck change in commands - RecheckChanged(p.second.commands, current_info_[p.first].commands); - - if (dirty_) { - generated_files.push_back(&(p.second)); + try { + const internal::GenInfo &loaded_geninfo = previous_info.at(p.first); + RecheckPaths( + loaded_geninfo.inputs, p.second.inputs, []() {}, []() {}, + []() {}); + RecheckChanged(loaded_geninfo.outputs, p.second.outputs); + RecheckChanged(loaded_geninfo.commands, p.second.commands); + if (dirty_) { + generated_files.push_back(&(p.second)); + build = true; + } + dirty_ = false; + } catch (const std::out_of_range &e) { + // This means that current_info has more items than previous_info + build = true; } - dirty_ = false; }); } - if (!generated_files.empty()) { - dirty_ = true; - } - + dirty_ = build; return generated_files; } From 3b082320289c026963ea04d5d607e5c6d7d06715 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 27 Aug 2021 20:10:38 -0700 Subject: [PATCH 31/51] Added recheck states to generator --- buildcc/lib/target/cmake/target.cmake | 1 + buildcc/lib/target/include/target/generator.h | 9 ++++ .../lib/target/src/generator/generator.cpp | 52 +++++++++++-------- .../target/src/generator/recheck_states.cpp | 28 ++++++++++ 4 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 buildcc/lib/target/src/generator/recheck_states.cpp diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 9d476908..3be095ff 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -13,6 +13,7 @@ set(TARGET_SRCS src/generator/generator_storer.cpp src/generator/generator.cpp src/generator/task.cpp + src/generator/recheck_states.cpp include/target/generator_loader.h include/target/generator.h diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index d2a7b92b..5c978dda 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -49,8 +49,17 @@ class Generator : public BuilderInterface { private: std::vector BuildGenerate(); void GenerateTask(); + bool Regenerate(std::vector &generated_files); bool Store() override; + // Recheck states + void InputRemoved(); + void InputAdded(); + void InputUpdated(); + + void OutputChanged(); + void CommandChanged(); + private: std::string name_; internal::geninfo_unordered_map current_info_; diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index 7a0ca906..ceedbcc3 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -45,31 +45,41 @@ std::vector Generator::BuildGenerate() { }); build = true; } else { - const auto &previous_info = loader_.GetLoadedInfo(); - std::for_each( - current_info_.cbegin(), current_info_.cend(), - [&](const std::pair &p) { - try { - const internal::GenInfo &loaded_geninfo = previous_info.at(p.first); - RecheckPaths( - loaded_geninfo.inputs, p.second.inputs, []() {}, []() {}, - []() {}); - RecheckChanged(loaded_geninfo.outputs, p.second.outputs); - RecheckChanged(loaded_geninfo.commands, p.second.commands); - if (dirty_) { - generated_files.push_back(&(p.second)); - build = true; - } - dirty_ = false; - } catch (const std::out_of_range &e) { - // This means that current_info has more items than previous_info - build = true; - } - }); + build = Regenerate(generated_files); } dirty_ = build; return generated_files; } +bool Generator::Regenerate( + std::vector &generated_files) { + bool build = false; + const auto &previous_info = loader_.GetLoadedInfo(); + + for (const auto &p : current_info_) { + try { + const internal::GenInfo &loaded_geninfo = previous_info.at(p.first); + RecheckPaths( + loaded_geninfo.inputs, p.second.inputs, [&]() { InputRemoved(); }, + [&]() { InputAdded(); }, [&]() { InputUpdated(); }); + RecheckChanged(loaded_geninfo.outputs, p.second.outputs, + [&]() { OutputChanged(); }); + RecheckChanged(loaded_geninfo.commands, p.second.commands, + [&]() { CommandChanged(); }); + + if (dirty_) { + generated_files.push_back(&(p.second)); + build = true; + } + dirty_ = false; + } catch (const std::out_of_range &e) { + // This means that current_info has more items than + // previous_info + build = true; + } + } + return build; +} + } // namespace buildcc::base diff --git a/buildcc/lib/target/src/generator/recheck_states.cpp b/buildcc/lib/target/src/generator/recheck_states.cpp new file mode 100644 index 00000000..57149745 --- /dev/null +++ b/buildcc/lib/target/src/generator/recheck_states.cpp @@ -0,0 +1,28 @@ +/* + * Copyright 2021 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/generator.h" + +namespace buildcc::base { + +void Generator::InputRemoved() {} +void Generator::InputAdded() {} +void Generator::InputUpdated() {} + +void Generator::OutputChanged() {} +void Generator::CommandChanged() {} + +} // namespace buildcc::base From 0307585bc7ebc55e88f4b1c8747dbd83411898a2 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sat, 28 Aug 2021 14:01:15 -0700 Subject: [PATCH 32/51] Updated GenInfo to fs_unordered_set --- buildcc/lib/target/include/target/generator_loader.h | 9 ++++----- buildcc/lib/target/src/generator/generator_loader.cpp | 7 ++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/buildcc/lib/target/include/target/generator_loader.h b/buildcc/lib/target/include/target/generator_loader.h index fe657c0c..d6344120 100644 --- a/buildcc/lib/target/include/target/generator_loader.h +++ b/buildcc/lib/target/include/target/generator_loader.h @@ -32,14 +32,13 @@ namespace buildcc::internal { struct GenInfo { std::string name; path_unordered_set inputs; - std::unordered_set outputs; + fs_unordered_set outputs; std::vector commands; bool parallel{false}; - GenInfo() {} - GenInfo(const std::string &n, const path_unordered_set &i, - const std::unordered_set &o, - const std::vector &c, bool p) + explicit GenInfo(const std::string &n, const path_unordered_set &i, + const fs_unordered_set &o, const std::vector &c, + bool p) : name(n), inputs(i), outputs(o), commands(c), parallel(p) {} }; diff --git a/buildcc/lib/target/src/generator/generator_loader.cpp b/buildcc/lib/target/src/generator/generator_loader.cpp index a6a069e5..6adb5f06 100644 --- a/buildcc/lib/target/src/generator/generator_loader.cpp +++ b/buildcc/lib/target/src/generator/generator_loader.cpp @@ -48,13 +48,14 @@ bool GeneratorLoader::Load() { const auto *info = generator->info(); for (auto iter = info->cbegin(); iter != info->cend(); iter++) { path_unordered_set i; - std::unordered_set o; + fs_unordered_set o; std::vector c; ExtractPath(iter->inputs(), i); Extract(iter->outputs(), o); Extract(iter->commands(), c); - loaded_info_[iter->name()->c_str()] = - GenInfo(iter->name()->c_str(), i, o, c, iter->parallel()); + loaded_info_.insert( + std::make_pair(iter->name()->c_str(), GenInfo(iter->name()->c_str(), i, + o, c, iter->parallel()))); } loaded_ = true; From 108e13f2a79f9a31c463a6ece1fc14a16ad385d6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sat, 28 Aug 2021 14:09:19 -0700 Subject: [PATCH 33/51] Added UserGenInfo to be used by users instead of internal::GenInfo --- buildcc/lib/target/include/target/generator.h | 18 ++++++++++++++++-- buildcc/lib/target/src/generator/generator.cpp | 6 +++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 5c978dda..fef473a7 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -32,14 +32,27 @@ namespace buildcc::base { +struct UserGenInfo { + std::string name; + internal::fs_unordered_set inputs; + internal::fs_unordered_set outputs; + std::vector commands; + bool parallel{false}; + + explicit UserGenInfo(const std::string &n, + const internal::fs_unordered_set &i, + const internal::fs_unordered_set &o, + const std::vector &c, bool p) + : name(n), inputs(i), outputs(o), commands(c), parallel(p) {} +}; + class Generator : public BuilderInterface { public: Generator(const std::string &name, const fs::path &path) : loader_(name, path) {} Generator(const Generator &generator) = delete; - void AddGenInfo(const internal::GenInfo &info); - + void AddGenInfo(const UserGenInfo &info); void Build() override; // Getter @@ -62,6 +75,7 @@ class Generator : public BuilderInterface { private: std::string name_; + std::unordered_map user_info_; internal::geninfo_unordered_map current_info_; internal::GeneratorLoader loader_; diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index ceedbcc3..f0708066 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -22,11 +22,11 @@ namespace buildcc::base { -void Generator::AddGenInfo(const internal::GenInfo &info) { +void Generator::AddGenInfo(const UserGenInfo &info) { env::assert_fatal( - current_info_.find(info.name) == current_info_.end(), + user_info_.find(info.name) == user_info_.end(), fmt::format("'{}' information already registered", info.name)); - current_info_[info.name] = info; + user_info_.insert(std::make_pair(info.name, info)); } void Generator::Build() { GenerateTask(); } From 1cc9a6f8ea719c5ff49f7380e3209ff40688f3c6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sat, 28 Aug 2021 14:53:47 -0700 Subject: [PATCH 34/51] Added Convert API --- buildcc/lib/target/include/target/generator.h | 7 +++++-- buildcc/lib/target/src/generator/generator.cpp | 15 +++++++++++++++ buildcc/lib/target/src/generator/task.cpp | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index fef473a7..b70b3bc0 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -60,9 +60,12 @@ class Generator : public BuilderInterface { tf::Taskflow &GetTaskflow() { return tf_; } private: - std::vector BuildGenerate(); void GenerateTask(); + // Convert from UserGenInfo to internal::GenInfo + void Convert(); + std::vector BuildGenerate(); bool Regenerate(std::vector &generated_files); + bool Store() override; // Recheck states @@ -76,7 +79,7 @@ class Generator : public BuilderInterface { private: std::string name_; std::unordered_map user_info_; - internal::geninfo_unordered_map current_info_; + std::unordered_map current_info_; internal::GeneratorLoader loader_; diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index f0708066..4ef24ec4 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -33,6 +33,21 @@ void Generator::Build() { GenerateTask(); } // PRIVATE +void Generator::Convert() { + for (const auto &user_info : user_info_) { + internal::path_unordered_set current_inputs; + for (const auto &user_inputs : user_info.second.inputs) { + current_inputs.insert(internal::Path::CreateExistingPath(user_inputs)); + } + + current_info_.insert(std::make_pair( + user_info.first, + internal::GenInfo(user_info.second.name, current_inputs, + user_info.second.outputs, user_info.second.commands, + user_info.second.parallel))); + } +} + std::vector Generator::BuildGenerate() { const bool loaded = loader_.Load(); diff --git a/buildcc/lib/target/src/generator/task.cpp b/buildcc/lib/target/src/generator/task.cpp index 4b96cca6..b720cfb8 100644 --- a/buildcc/lib/target/src/generator/task.cpp +++ b/buildcc/lib/target/src/generator/task.cpp @@ -22,6 +22,7 @@ namespace buildcc::base { void Generator::GenerateTask() { build_task_ = tf_.emplace([&](tf::Subflow &subflow) { + Convert(); const auto generated_files = BuildGenerate(); if (!dirty_) { return; From 0e47c8fe7f2cfd290d1f4ff200dd60566d15c610 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 13:34:19 -0700 Subject: [PATCH 35/51] Update generator.cpp --- buildcc/lib/target/src/generator/generator.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index 4ef24ec4..cc008142 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -39,7 +39,6 @@ void Generator::Convert() { for (const auto &user_inputs : user_info.second.inputs) { current_inputs.insert(internal::Path::CreateExistingPath(user_inputs)); } - current_info_.insert(std::make_pair( user_info.first, internal::GenInfo(user_info.second.name, current_inputs, @@ -54,10 +53,9 @@ std::vector Generator::BuildGenerate() { std::vector generated_files; bool build = false; if (!loaded) { - std::for_each(current_info_.cbegin(), current_info_.cend(), - [&](const std::pair &p) { - generated_files.push_back(&(p.second)); - }); + for (const auto &ci : current_info_) { + generated_files.push_back(&(ci.second)); + } build = true; } else { build = Regenerate(generated_files); From 781977935254ff5279bcb8732f2fe7e820da5abd Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 14:33:02 -0700 Subject: [PATCH 36/51] Update generator.cpp --- buildcc/lib/target/src/generator/generator.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index cc008142..596c9c54 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -26,7 +26,7 @@ void Generator::AddGenInfo(const UserGenInfo &info) { env::assert_fatal( user_info_.find(info.name) == user_info_.end(), fmt::format("'{}' information already registered", info.name)); - user_info_.insert(std::make_pair(info.name, info)); + user_info_.emplace(info.name, info); } void Generator::Build() { GenerateTask(); } @@ -37,13 +37,13 @@ void Generator::Convert() { for (const auto &user_info : user_info_) { internal::path_unordered_set current_inputs; for (const auto &user_inputs : user_info.second.inputs) { - current_inputs.insert(internal::Path::CreateExistingPath(user_inputs)); + current_inputs.emplace(internal::Path::CreateExistingPath(user_inputs)); } - current_info_.insert(std::make_pair( + current_info_.emplace( user_info.first, internal::GenInfo(user_info.second.name, current_inputs, user_info.second.outputs, user_info.second.commands, - user_info.second.parallel))); + user_info.second.parallel)); } } From 50b928225ccb0a5d2e97a7ce80381ca7038a5ded Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 14:33:37 -0700 Subject: [PATCH 37/51] Added unit test mocks --- buildcc/lib/target/cmake/mock_target.cmake | 22 +++++-- buildcc/lib/target/mock/expect_generator.h | 17 ++++++ .../target/mock/generator/recheck_states.cpp | 59 +++++++++++++++++++ buildcc/lib/target/mock/generator/task.cpp | 27 +++++++++ 4 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 buildcc/lib/target/mock/expect_generator.h create mode 100644 buildcc/lib/target/mock/generator/recheck_states.cpp create mode 100644 buildcc/lib/target/mock/generator/task.cpp diff --git a/buildcc/lib/target/cmake/mock_target.cmake b/buildcc/lib/target/cmake/mock_target.cmake index 8d1a2286..4ca4566a 100644 --- a/buildcc/lib/target/cmake/mock_target.cmake +++ b/buildcc/lib/target/cmake/mock_target.cmake @@ -1,17 +1,29 @@ add_library(mock_target STATIC + # Utils + src/util/util.cpp + + # Generator + src/generator/generator_loader.cpp + src/generator/generator_storer.cpp + src/generator/generator.cpp + + # Generator mocks + mock/generator/task.cpp + mock/generator/recheck_states.cpp + + # Target + src/target/target_loader.cpp + src/target/target_storer.cpp src/target/target.cpp src/target/source.cpp src/target/include_dir.cpp src/target/lib.cpp src/target/build.cpp src/target/flags.cpp + + # Target mocks mock/target/recheck_states.cpp mock/target/tasks.cpp - - src/target/target_loader.cpp - src/target/target_storer.cpp - - src/util/util.cpp ) target_include_directories(mock_target PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include diff --git a/buildcc/lib/target/mock/expect_generator.h b/buildcc/lib/target/mock/expect_generator.h new file mode 100644 index 00000000..30c4d2ac --- /dev/null +++ b/buildcc/lib/target/mock/expect_generator.h @@ -0,0 +1,17 @@ +#ifndef TARGET_MOCK_EXPECT_GENERATOR_H_ +#define TARGET_MOCK_EXPECT_GENERATOR_H_ + +#include "target/generator.h" + +namespace buildcc::base::m { + +void GeneratorExpect_InputRemoved(unsigned int calls, Generator *generator); +void GeneratorExpect_InputAdded(unsigned int calls, Generator *generator); +void GeneratorExpect_InputUpdated(unsigned int calls, Generator *generator); + +void GeneratorExpect_OutputChanged(unsigned int calls, Generator *generator); +void GeneratorExpect_CommandChanged(unsigned int calls, Generator *generator); + +} // namespace buildcc::base::m + +#endif diff --git a/buildcc/lib/target/mock/generator/recheck_states.cpp b/buildcc/lib/target/mock/generator/recheck_states.cpp new file mode 100644 index 00000000..1b4a0cf1 --- /dev/null +++ b/buildcc/lib/target/mock/generator/recheck_states.cpp @@ -0,0 +1,59 @@ +#include "target/generator.h" + +#include "expect_generator.h" + +#include "CppUTestExt/MockSupport.h" + +namespace buildcc::base { + +static constexpr const char *const INPUT_REMOVED_FUNCTION = + "Generator::InputRemoved"; +static constexpr const char *const INPUT_ADDED_FUNCTION = + "Generator::InputAdded"; +static constexpr const char *const INPUT_UPDATED_FUNCTION = + "Generator::InputUpdated"; + +static constexpr const char *const OUTPUT_CHANGED_FUNCTION = + "Generator::OutputChanged"; +static constexpr const char *const COMMAND_CHANGED_FUNCTION = + "Generator::CommandChanged"; + +void Generator::InputRemoved() { + mock().actualCall(INPUT_REMOVED_FUNCTION).onObject(this); +} +void Generator::InputAdded() { + mock().actualCall(INPUT_ADDED_FUNCTION).onObject(this); +} +void Generator::InputUpdated() { + mock().actualCall(INPUT_UPDATED_FUNCTION).onObject(this); +} + +void Generator::OutputChanged() { + mock().actualCall(OUTPUT_CHANGED_FUNCTION).onObject(this); +} +void Generator::CommandChanged() { + mock().actualCall(COMMAND_CHANGED_FUNCTION).onObject(this); +} + +namespace m { + +void GeneratorExpect_InputRemoved(unsigned int calls, Generator *generator) { + mock().expectNCalls(calls, INPUT_REMOVED_FUNCTION).onObject(generator); +} +void GeneratorExpect_InputAdded(unsigned int calls, Generator *generator) { + mock().expectNCalls(calls, INPUT_ADDED_FUNCTION).onObject(generator); +} +void GeneratorExpect_InputUpdated(unsigned int calls, Generator *generator) { + mock().expectNCalls(calls, INPUT_UPDATED_FUNCTION).onObject(generator); +} + +void GeneratorExpect_OutputChanged(unsigned int calls, Generator *generator) { + mock().expectNCalls(calls, OUTPUT_CHANGED_FUNCTION).onObject(generator); +} +void GeneratorExpect_CommandChanged(unsigned int calls, Generator *generator) { + mock().expectNCalls(calls, COMMAND_CHANGED_FUNCTION).onObject(generator); +} + +} // namespace m + +} // namespace buildcc::base diff --git a/buildcc/lib/target/mock/generator/task.cpp b/buildcc/lib/target/mock/generator/task.cpp new file mode 100644 index 00000000..4bad5c62 --- /dev/null +++ b/buildcc/lib/target/mock/generator/task.cpp @@ -0,0 +1,27 @@ +#include "target/generator.h" + +#include "command/command.h" + +namespace buildcc::base { + +void Generator::GenerateTask() { + Convert(); + + const auto generated_files = BuildGenerate(); + + if (!dirty_) { + return; + } + + // NOTE, info->parallel is not checked + for (const auto &info : generated_files) { + for (const auto &command : info->commands) { + bool success = Command::Execute(command); + env::assert_fatal(success, fmt::format("{} failed", command)); + } + } + + Store(); +} + +} // namespace buildcc::base From a1bc1b1176c00f6a515c9445d2aefe4c5a701b2e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 14:33:54 -0700 Subject: [PATCH 38/51] Added test_generator basic unit tests --- buildcc/lib/target/test/target/CMakeLists.txt | 9 ++ .../lib/target/test/target/test_generator.cpp | 115 ++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 buildcc/lib/target/test/target/test_generator.cpp diff --git a/buildcc/lib/target/test/target/CMakeLists.txt b/buildcc/lib/target/test/target/CMakeLists.txt index 978dc499..f63b0b27 100644 --- a/buildcc/lib/target/test/target/CMakeLists.txt +++ b/buildcc/lib/target/test/target/CMakeLists.txt @@ -8,6 +8,15 @@ target_link_libraries(target_interface INTERFACE mock_target ) +# Generator +add_executable(test_generator + test_generator.cpp +) +target_link_libraries(test_generator PRIVATE target_interface) + +add_test(NAME test_generator COMMAND test_generator + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) # Test base target add_executable(test_base_target diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp new file mode 100644 index 00000000..21f03ff9 --- /dev/null +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -0,0 +1,115 @@ +#include "target/generator.h" + +#include "expect_command.h" +#include "expect_generator.h" + +// NOTE, Make sure all these includes are AFTER the system and header includes +#include "CppUTest/CommandLineTestRunner.h" +#include "CppUTest/MemoryLeakDetectorNewMacros.h" +#include "CppUTest/TestHarness.h" +#include "CppUTest/Utest.h" +#include "CppUTestExt/MockSupport.h" + +// clang-format off +TEST_GROUP(GeneratorTestGroup) +{ + void teardown() { + mock().clear(); + } +}; +// clang-format on + +fs::path BUILD_DIR = "intermediate/generator"; + +TEST(GeneratorTestGroup, Generator_AddInfo) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "AddInfo"; + fs::create_directories(TEST_BUILD_DIR); + + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + CHECK_THROWS(buildcc::env::assert_exception, + generator.AddGenInfo( + buildcc::base::UserGenInfo("gcc_1", {}, {}, {}, true))); +} + +TEST(GeneratorTestGroup, Generator_Build) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "Build"; + fs::create_directories(TEST_BUILD_DIR); + + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + + mock().checkExpectations(); +} + +TEST(GeneratorTestGroup, Generator_Rebuild) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "Rebuild"; + fs::create_directories(TEST_BUILD_DIR); + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + generator.Build(); + } + + mock().checkExpectations(); +} + +TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Remove) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Add) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Update) {} + +// Input for second generator is output of first generator +TEST(GeneratorTestGroup, Generator_DoubleDependency) {} + +int main(int ac, char **av) { + fs::remove_all(BUILD_DIR); + fs::create_directories(BUILD_DIR); + return CommandLineTestRunner::RunAllTests(ac, av); +} From ecae12210c350199149435f5c1c439ceebddbbdf Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 14:56:01 -0700 Subject: [PATCH 39/51] Update generator.cpp --- buildcc/lib/target/src/generator/generator.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index 596c9c54..aa637b49 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -89,6 +89,7 @@ bool Generator::Regenerate( } catch (const std::out_of_range &e) { // This means that current_info has more items than // previous_info + generated_files.push_back(&(p.second)); build = true; } } From 9785d6fc4c8a40dd77b1c1e78bbe15cec8efe4b9 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 14:56:13 -0700 Subject: [PATCH 40/51] Update test_generator.cpp --- .../lib/target/test/target/test_generator.cpp | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 21f03ff9..33d0513f 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -100,7 +100,56 @@ TEST(GeneratorTestGroup, Generator_Rebuild) { mock().checkExpectations(); } -TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) {} +TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "Rebuild_PreviousNotFound"; + fs::create_directories(TEST_BUILD_DIR); + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + // Current Info is NEWER than old + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_2", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + mock().checkExpectations(); +} + TEST(GeneratorTestGroup, Generator_Rebuild_Remove) {} TEST(GeneratorTestGroup, Generator_Rebuild_Add) {} TEST(GeneratorTestGroup, Generator_Rebuild_Update) {} From 56557302e5fe1d5ee627b66d152c5fd2fbf79fe9 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 15:14:59 -0700 Subject: [PATCH 41/51] Update test_generator.cpp - Added Rebuild for Inputs --- .../lib/target/test/target/test_generator.cpp | 88 ++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 33d0513f..207bac1c 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -1,8 +1,12 @@ #include "target/generator.h" +#include + #include "expect_command.h" #include "expect_generator.h" +#include "env/util.h" + // NOTE, Make sure all these includes are AFTER the system and header includes #include "CppUTest/CommandLineTestRunner.h" #include "CppUTest/MemoryLeakDetectorNewMacros.h" @@ -150,9 +154,87 @@ TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) { mock().checkExpectations(); } -TEST(GeneratorTestGroup, Generator_Rebuild_Remove) {} -TEST(GeneratorTestGroup, Generator_Rebuild_Add) {} -TEST(GeneratorTestGroup, Generator_Rebuild_Update) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Inputs) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "Rebuild_Inputs"; + fs::create_directories(TEST_BUILD_DIR); + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + "data/new_source.cpp", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::base::m::GeneratorExpect_InputAdded(1, &generator); + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/new_source.cpp", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::base::m::GeneratorExpect_InputRemoved(1, &generator); + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + sleep(1); + bool saved = buildcc::env::SaveFile("data/new_source.cpp", "", false); + CHECK_TRUE(saved); + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/new_source.cpp", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::base::m::GeneratorExpect_InputUpdated(1, &generator); + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + mock().checkExpectations(); +} + +TEST(GeneratorTestGroup, Generator_Rebuild_Outputs) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Commands) {} // Input for second generator is output of first generator TEST(GeneratorTestGroup, Generator_DoubleDependency) {} From 291eeaad22109239b6de971cebf15056a1b3c67c Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 15:20:41 -0700 Subject: [PATCH 42/51] Update test_generator.cpp --- .../lib/target/test/target/test_generator.cpp | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 207bac1c..4df95899 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -233,7 +233,49 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Inputs) { mock().checkExpectations(); } -TEST(GeneratorTestGroup, Generator_Rebuild_Outputs) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Outputs) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "Rebuild_Outputs"; + fs::create_directories(TEST_BUILD_DIR); + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + { + TEST_BUILD_DIR / "dummy_main.o", + TEST_BUILD_DIR / "dummy_main.exe", + }, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.exe"}, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::base::m::GeneratorExpect_OutputChanged(1, &generator); + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + mock().checkExpectations(); +} + TEST(GeneratorTestGroup, Generator_Rebuild_Commands) {} // Input for second generator is output of first generator From 18438ca4f21c6a2e77fe3f70ba38a6a7e79cedeb Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 15:39:45 -0700 Subject: [PATCH 43/51] Update test_generator.cpp --- .../lib/target/test/target/test_generator.cpp | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 4df95899..6693136f 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -276,7 +276,56 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Outputs) { mock().checkExpectations(); } -TEST(GeneratorTestGroup, Generator_Rebuild_Commands) {} +TEST(GeneratorTestGroup, Generator_Rebuild_Commands) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "Rebuild_Commands"; + fs::create_directories(TEST_BUILD_DIR); + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + { + TEST_BUILD_DIR / "dummy_main.o", + TEST_BUILD_DIR / "dummy_main.exe", + }, + {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + { + buildcc::base::Generator generator("custom_file_generator", TEST_BUILD_DIR); + generator.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + { + TEST_BUILD_DIR / "dummy_main.o", + TEST_BUILD_DIR / "dummy_main.exe", + }, + { + "gcc -c -o intermediate/generator/AddInfo/dummy_main.o " + "data/dummy_main.c", + "gcc -o intermediate/generator/AddInfo/dummy_main.exe " + "intermediate/generator/AddInfo/dummy_main.o", + }, + true)); + + buildcc::base::m::GeneratorExpect_CommandChanged(1, &generator); + buildcc::m::CommandExpect_Execute(1, true); + buildcc::m::CommandExpect_Execute(1, true); + generator.Build(); + } + + mock().checkExpectations(); +} // Input for second generator is output of first generator TEST(GeneratorTestGroup, Generator_DoubleDependency) {} From 126a717ec78d21695dfd89fea7c256a88318bb6e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:19:47 -0700 Subject: [PATCH 44/51] Update test_generator.cpp --- .../lib/target/test/target/test_generator.cpp | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 6693136f..c3ed2b4f 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -328,7 +328,44 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Commands) { } // Input for second generator is output of first generator -TEST(GeneratorTestGroup, Generator_DoubleDependency) {} +TEST(GeneratorTestGroup, Generator_DoubleDependency) { + fs::path TEST_BUILD_DIR = BUILD_DIR / "DoubleDependency"; + fs::create_directories(TEST_BUILD_DIR); + + buildcc::base::Generator ogen("custom_object_generator", TEST_BUILD_DIR); + ogen.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + "data/dummy_main.c", + }, + {TEST_BUILD_DIR / "dummy_main.o"}, + {"gcc -c -o intermediate/generator/DoubleDependency/dummy_main.o " + "data/dummy_main.c"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + ogen.Build(); + + // * NOTE, dummy_main.o is an input to `custom_exe_generator` + buildcc::env::SaveFile((TEST_BUILD_DIR / "dummy_main.o").string().c_str(), "", + false); + + buildcc::base::Generator egen("custom_exe_generator", TEST_BUILD_DIR); + egen.AddGenInfo(buildcc::base::UserGenInfo( + "gcc_1", + { + TEST_BUILD_DIR / "dummy_main.o", + }, + {TEST_BUILD_DIR / "dummy_main.o"}, + {"gcc -o intermediate/generator/DoubleDependency/dummy_main.exe " + "intermediate/generator/DoubleDependency/dummy_main.o"}, + true)); + + buildcc::m::CommandExpect_Execute(1, true); + egen.Build(); + + mock().checkExpectations(); +} int main(int ac, char **av) { fs::remove_all(BUILD_DIR); From 3d428d48245b4a1e0fc45ea36c1733681432a149 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:20:39 -0700 Subject: [PATCH 45/51] Update test_generator.cpp --- buildcc/lib/target/test/target/test_generator.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index c3ed2b4f..a06d6d58 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -291,7 +291,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Commands) { TEST_BUILD_DIR / "dummy_main.o", TEST_BUILD_DIR / "dummy_main.exe", }, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Commands/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -311,10 +311,10 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Commands) { TEST_BUILD_DIR / "dummy_main.exe", }, { - "gcc -c -o intermediate/generator/AddInfo/dummy_main.o " + "gcc -c -o intermediate/generator/Rebuild_Commands/dummy_main.o " "data/dummy_main.c", - "gcc -o intermediate/generator/AddInfo/dummy_main.exe " - "intermediate/generator/AddInfo/dummy_main.o", + "gcc -o intermediate/generator/Rebuild_Commands/dummy_main.exe " + "intermediate/generator/Rebuild_Commands/dummy_main.o", }, true)); From eb2d1514b4ffc7c081e6397163d43973945d51cc Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:20:58 -0700 Subject: [PATCH 46/51] Update test_generator.cpp --- buildcc/lib/target/test/target/test_generator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index a06d6d58..202cc142 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -248,7 +248,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Outputs) { TEST_BUILD_DIR / "dummy_main.o", TEST_BUILD_DIR / "dummy_main.exe", }, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Outputs/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -264,7 +264,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Outputs) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Outputs/dummy_main.exe " "data/dummy_main.c"}, true)); From 3bdff245ca1f32db5ba40b271c294d6b34fde4b6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:21:24 -0700 Subject: [PATCH 47/51] Update test_generator.cpp --- buildcc/lib/target/test/target/test_generator.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 202cc142..78f0eec0 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -166,7 +166,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Inputs) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Inputs/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -183,7 +183,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Inputs) { "data/new_source.cpp", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Inputs/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -200,7 +200,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Inputs) { "data/new_source.cpp", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Inputs/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -221,7 +221,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild_Inputs) { "data/new_source.cpp", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild_Inputs/dummy_main.exe " "data/dummy_main.c"}, true)); From 8417dd41228b88ab1d8f36c2892fa2d78e61537f Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:21:41 -0700 Subject: [PATCH 48/51] Update test_generator.cpp --- buildcc/lib/target/test/target/test_generator.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 78f0eec0..a7d71b9e 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -116,7 +116,8 @@ TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o " + "intermediate/generator/Rebuild_PreviousNotFound/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -132,7 +133,8 @@ TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o " + "intermediate/generator/Rebuild_PreviousNotFound/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -143,7 +145,8 @@ TEST(GeneratorTestGroup, Generator_Rebuild_PreviousNotFound) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o " + "intermediate/generator/Rebuild_PreviousNotFound/dummy_main.exe " "data/dummy_main.c"}, true)); From 3cc096f690141d86c5fe629a55845aad8e5bb5f0 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:21:57 -0700 Subject: [PATCH 49/51] Update test_generator.cpp --- buildcc/lib/target/test/target/test_generator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index a7d71b9e..557d7332 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -78,7 +78,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild/dummy_main.exe " "data/dummy_main.c"}, true)); @@ -94,7 +94,7 @@ TEST(GeneratorTestGroup, Generator_Rebuild) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Rebuild/dummy_main.exe " "data/dummy_main.c"}, true)); From 2a0e0a3bc91a178d724a3494a0c9ef1962beeb9c Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 16:22:11 -0700 Subject: [PATCH 50/51] Update test_generator.cpp --- buildcc/lib/target/test/target/test_generator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildcc/lib/target/test/target/test_generator.cpp b/buildcc/lib/target/test/target/test_generator.cpp index 557d7332..ced24dfe 100644 --- a/buildcc/lib/target/test/target/test_generator.cpp +++ b/buildcc/lib/target/test/target/test_generator.cpp @@ -56,7 +56,7 @@ TEST(GeneratorTestGroup, Generator_Build) { "data/dummy_main.c", }, {TEST_BUILD_DIR / "dummy_main.exe"}, - {"gcc -o intermediate/generator/AddInfo/dummy_main.exe " + {"gcc -o intermediate/generator/Build/dummy_main.exe " "data/dummy_main.c"}, true)); From 3cde7ec26a369e119ad4649a1f77035cfbb15433 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 29 Aug 2021 17:23:26 -0700 Subject: [PATCH 51/51] Update generator.cpp --- buildcc/lib/target/src/generator/generator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/src/generator/generator.cpp b/buildcc/lib/target/src/generator/generator.cpp index aa637b49..2c72012a 100644 --- a/buildcc/lib/target/src/generator/generator.cpp +++ b/buildcc/lib/target/src/generator/generator.cpp @@ -53,9 +53,9 @@ std::vector Generator::BuildGenerate() { std::vector generated_files; bool build = false; if (!loaded) { - for (const auto &ci : current_info_) { - generated_files.push_back(&(ci.second)); - } + std::for_each( + current_info_.cbegin(), current_info_.cend(), + [&](const auto &p) { generated_files.push_back(&(p.second)); }); build = true; } else { build = Regenerate(generated_files);