Generator (#98)

One stage generator for one or more inputs generating one or more outputs using custom commands

- Complete rework to Targets
- Interfaces design for common functionality between Generator and Target

Author: coder137

buildcc/buildcc.h

@@ -24,8 +24,12 @@
 #include "env/assert_fatal.h"
 #include "env/logging.h"
 #include "env/host_os.h"
+#include "env/host_compiler.h"
 #include "env/util.h"
 
+// 
+#include "command/command.h"
+
 // Base
 #include "toolchain/toolchain.h"
 #include "target/target.h"

buildcc/lib/target/cmake/mock_target.cmake

@@ -1,17 +1,29 @@
 add_library(mock_target STATIC
+    # Utils
+    src/util/util.cpp
+
+    # Generator
+    src/generator/generator_loader.cpp
+    src/generator/generator_storer.cpp
+    src/generator/generator.cpp
+
+    # Generator mocks
+    mock/generator/task.cpp
+    mock/generator/recheck_states.cpp
+
+    # Target
+    src/target/target_loader.cpp
+    src/target/target_storer.cpp
     src/target/target.cpp
     src/target/source.cpp
     src/target/include_dir.cpp
     src/target/lib.cpp
     src/target/build.cpp
     src/target/flags.cpp
+
+    # Target mocks
     mock/target/recheck_states.cpp
     mock/target/tasks.cpp
-
-    src/fbs/fbs_loader.cpp
-    src/fbs/fbs_storer.cpp
-
-    src/util/util.cpp
 )
 target_include_directories(mock_target PUBLIC
     ${CMAKE_CURRENT_SOURCE_DIR}/include

buildcc/lib/target/cmake/target.cmake

@@ -1,23 +1,35 @@
 set(TARGET_SRCS 
+    # Interfaces
+    include/target/loader_interface.h
+    include/target/builder_interface.h
+
+    # Utils
+    src/util/util.cpp
+    include/target/path.h
+    include/target/util.h
+
+    # Generator
+    src/generator/generator_loader.cpp
+    src/generator/generator_storer.cpp
+    src/generator/generator.cpp
+    src/generator/task.cpp
+    src/generator/recheck_states.cpp
+    include/target/generator_loader.h
+    include/target/generator.h
+
+    # Target
+    src/target/target_loader.cpp
+    src/target/target_storer.cpp
     src/target/target.cpp
     src/target/source.cpp
     src/target/include_dir.cpp
     src/target/lib.cpp
     src/target/build.cpp
     src/target/flags.cpp
-
     src/target/recheck_states.cpp
     src/target/tasks.cpp
-
-    src/fbs/fbs_loader.cpp
-    src/fbs/fbs_storer.cpp
-
-    src/util/util.cpp
-
+    include/target/target_loader.h
     include/target/target.h
-    include/target/fbs_loader.h
-    include/target/path.h
-    include/target/util.h
 )
 
 if(${BUILDCC_BUILD_AS_SINGLE_LIB})

buildcc/lib/target/include/target/builder_interface.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2021 Niket Naidu. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TARGET_BUILDER_INTERFACE_H_
+#define TARGET_BUILDER_INTERFACE_H_
+
+#include <unordered_set>
+
+#include "target/path.h"
+#include "target/util.h"
+
+namespace buildcc::base {
+
+class BuilderInterface {
+
+public:
+  virtual void Build() = 0;
+
+protected:
+  template <typename T>
+  void RecheckChanged(
+      const T &previous, const T &current,
+      const std::function<void(void)> &callback = []() {}) {
+    if (dirty_) {
+      return;
+    }
+
+    if (previous != current) {
+      callback();
+      dirty_ = true;
+    }
+  }
+
+  void RecheckPaths(const internal::path_unordered_set &previous_path,
+                    const internal::path_unordered_set &current_path,
+                    const std::function<void(void)> &path_removed_cb,
+                    const std::function<void(void)> &path_added_cb,
+                    const std::function<void(void)> &path_updated_cb) {
+    if (dirty_) {
+      return;
+    }
+
+    // * Old path is removed
+    const bool removed =
+        internal::is_previous_paths_different(previous_path, current_path);
+    if (removed) {
+      path_removed_cb();
+      dirty_ = true;
+      return;
+    }
+
+    for (const auto &path : current_path) {
+      auto iter = previous_path.find(path);
+
+      if (iter == previous_path.end()) {
+        // * New path added
+        path_added_cb();
+        dirty_ = true;
+      } else {
+        // * Path is updated
+        if (path.GetLastWriteTimestamp() > iter->GetLastWriteTimestamp()) {
+          path_updated_cb();
+          dirty_ = true;
+        } else {
+          // * Do nothing
+        }
+      }
+
+      if (dirty_) {
+        break;
+      }
+    }
+  }
+
+private:
+  virtual bool Store() = 0;
+
+protected:
+  bool dirty_{false};
+};
+
+} // namespace buildcc::base
+
+#endif

buildcc/lib/target/include/target/generator.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2021 Niket Naidu. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TARGET_GENERATOR_H_
+#define TARGET_GENERATOR_H_
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "taskflow/taskflow.hpp"
+
+#include "env/env.h"
+
+#include "target/builder_interface.h"
+
+#include "target/generator_loader.h"
+#include "target/path.h"
+
+namespace buildcc::base {
+
+struct UserGenInfo {
+  std::string name;
+  internal::fs_unordered_set inputs;
+  internal::fs_unordered_set outputs;
+  std::vector<std::string> commands;
+  bool parallel{false};
+
+  explicit UserGenInfo(const std::string &n,
+                       const internal::fs_unordered_set &i,
+                       const internal::fs_unordered_set &o,
+                       const std::vector<std::string> &c, bool p)
+      : name(n), inputs(i), outputs(o), commands(c), parallel(p) {}
+};
+
+class Generator : public BuilderInterface {
+public:
+  Generator(const std::string &name, const fs::path &path)
+      : loader_(name, path) {}
+  Generator(const Generator &generator) = delete;
+
+  void AddGenInfo(const UserGenInfo &info);
+  void Build() override;
+
+  // Getter
+  fs::path GetBinaryPath() const { return loader_.GetBinaryPath(); }
+  tf::Taskflow &GetTaskflow() { return tf_; }
+
+private:
+  void GenerateTask();
+  // Convert from UserGenInfo to internal::GenInfo
+  void Convert();
+  std::vector<const internal::GenInfo *> BuildGenerate();
+  bool Regenerate(std::vector<const internal::GenInfo *> &generated_files);
+
+  bool Store() override;
+
+  // Recheck states
+  void InputRemoved();
+  void InputAdded();
+  void InputUpdated();
+
+  void OutputChanged();
+  void CommandChanged();
+
+private:
+  std::string name_;
+  std::unordered_map<std::string, UserGenInfo> user_info_;
+  std::unordered_map<std::string, internal::GenInfo> current_info_;
+
+  internal::GeneratorLoader loader_;
+
+  tf::Taskflow tf_;
+  tf::Task build_task_;
+};
+
+} // namespace buildcc::base
+
+#endif

buildcc/lib/target/include/target/generator_loader.h

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2021 Niket Naidu. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TARGET_GENERATOR_LOADER_H_
+#define TARGET_GENERATOR_LOADER_H_
+
+#include "target/loader_interface.h"
+
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "fmt/format.h"
+
+#include "target/path.h"
+
+namespace buildcc::internal {
+
+struct GenInfo {
+  std::string name;
+  path_unordered_set inputs;
+  fs_unordered_set outputs;
+  std::vector<std::string> commands;
+  bool parallel{false};
+
+  explicit GenInfo(const std::string &n, const path_unordered_set &i,
+                   const fs_unordered_set &o, const std::vector<std::string> &c,
+                   bool p)
+      : name(n), inputs(i), outputs(o), commands(c), parallel(p) {}
+};
+
+typedef std::unordered_map<std::string, GenInfo> geninfo_unordered_map;
+
+class GeneratorLoader : public LoaderInterface {
+public:
+  GeneratorLoader(const std::string &name, const fs::path &path)
+      : name_(name), path_(path) {}
+
+  GeneratorLoader(const GeneratorLoader &loader) = delete;
+
+  bool Load() override;
+
+  // Getters
+  fs::path GetBinaryPath() const override {
+    return path_ / fmt::format("{}.bin", name_);
+  }
+
+  const geninfo_unordered_map &GetLoadedInfo() { return loaded_info_; }
+
+private:
+  std::string name_;
+  fs::path path_;
+
+  geninfo_unordered_map loaded_info_;
+};
+
+} // namespace buildcc::internal
+
+#endif

buildcc/lib/target/include/target/loader_interface.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2021 Niket Naidu. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TARGET_LOADER_INTERFACE_H_
+#define TARGET_LOADER_INTERFACE_H_
+
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace buildcc::internal {
+
+class LoaderInterface {
+public:
+  virtual bool Load() = 0;
+  virtual fs::path GetBinaryPath() const = 0;
+
+  bool IsLoaded() const { return loaded_; };
+
+protected:
+  bool loaded_{false};
+};
+
+} // namespace buildcc::internal
+
+#endif