From 32d412437ba1b9bdfa46814e0bc6985082cd4266 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Wed, 12 Nov 2025 10:15:21 -0600 Subject: [PATCH 01/49] introduce test --- test/CMakeLists.txt | 3 +++ test/provider_test.cpp | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 test/provider_test.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 83533721..3d93011a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -164,6 +164,9 @@ cet_test( cet_test(product_query USE_CATCH2_MAIN SOURCE product_query.cpp LIBRARIES phlex::core ) +cet_test(provider_test USE_CATCH2_MAIN SOURCE provider_test.cpp LIBRARIES + phlex::core + ) cet_test( type_distinction USE_CATCH2_MAIN diff --git a/test/provider_test.cpp b/test/provider_test.cpp new file mode 100644 index 00000000..a042879b --- /dev/null +++ b/test/provider_test.cpp @@ -0,0 +1,47 @@ +#include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" +#include "phlex/model/product_store.hpp" + +#include "catch2/catch_test_macros.hpp" +#include "fmt/std.h" +#include "spdlog/spdlog.h" + +using namespace phlex::experimental; + +namespace { + // Provider algorithms + // int give_me_an_a(data_cell_index) { return 1; }; +} + +namespace { + unsigned pass_on(unsigned number) { return number; } +} + +TEST_CASE("provider_test") +{ + constexpr auto max_events{100'000u}; + // constexpr auto max_events{1'000'000u}; + // spdlog::flush_on(spdlog::level::trace); + + auto levels_to_process = [](framework_driver& driver) { + auto job_store = product_store::base(); + driver.yield(job_store); + + for (unsigned int i : std::views::iota(1u, max_events + 1)) { + auto store = job_store->make_child(i, "spill", "Source"); + store->add_product("number", i); + driver.yield(store); + } + }; + + framework_graph g{levels_to_process}; + +// g.provider(give_me_an_a).input_indices("spill").output_product("Gauss:number"); + + g.transform("pass_on", pass_on, concurrency::unlimited) + .input_family("number"_in("spill")) + .output_products("different"); + + g.execute(); + CHECK(g.execution_counts("count") == 1); +} From 635874bd310ed5530d2bc586b6d60569c316aea9 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Mon, 17 Nov 2025 16:03:08 -0600 Subject: [PATCH 02/49] make provider_test work --- test/provider_test.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index a042879b..9ea10be5 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -14,22 +14,24 @@ namespace { } namespace { - unsigned pass_on(unsigned number) { return number; } + unsigned pass_on(unsigned number) { spdlog::info("pass_on({})", number); return number; } } TEST_CASE("provider_test") { - constexpr auto max_events{100'000u}; + constexpr auto max_events{1'000u}; // constexpr auto max_events{1'000'000u}; // spdlog::flush_on(spdlog::level::trace); auto levels_to_process = [](framework_driver& driver) { auto job_store = product_store::base(); + spdlog::info("job_store: is about to be yielded"); driver.yield(job_store); for (unsigned int i : std::views::iota(1u, max_events + 1)) { auto store = job_store->make_child(i, "spill", "Source"); store->add_product("number", i); + spdlog::info("store: is about to be yielded"); driver.yield(store); } }; @@ -43,5 +45,5 @@ TEST_CASE("provider_test") .output_products("different"); g.execute(); - CHECK(g.execution_counts("count") == 1); + CHECK(g.execution_counts("pass_on") == max_events); } From e696b2dc686876bfab6d00558f2f8061f523eb19 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Mon, 17 Nov 2025 16:10:56 -0600 Subject: [PATCH 03/49] Add development plan notes to provider_test.cpp --- test/provider_test.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 9ea10be5..c6ad2f7a 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -47,3 +47,20 @@ TEST_CASE("provider_test") g.execute(); CHECK(g.execution_counts("pass_on") == max_events); } + +/* + +Planned development flow: + +[x] Get initial test working. +[ ] Introduce stub `provider` that takes product_store_ptr as input and returns a product_store_ptr. + Wire the provider into the graph. +[ ] Modify the `provider` to take data_cell_index as input. + The `multiplexer` will then need to emit data_cell_index, stripped from the `product_store` it currently returns. +[ ] Modify the `multiplexer` to accept data_cell_index as input. + The Input will then need to emit data_cell_index, stripped from the `product_store` it currently returns. +[ ] Modify `Input` to accept data_cell_index is input. + This will mean `framework_driver` will have to emit data_cell_index instead of product_store. + +Then we're done. +*/ From 29546d1ac6e4a5b54b95264eeeeffe75f54d1107 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Mon, 17 Nov 2025 16:42:18 -0600 Subject: [PATCH 04/49] Introduce non-trivial data product --- test/provider_test.cpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index c6ad2f7a..2c03bd5e 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -8,13 +8,21 @@ using namespace phlex::experimental; +namespace toy { + struct VertexCollection { + unsigned data; + }; +} + namespace { // Provider algorithms // int give_me_an_a(data_cell_index) { return 1; }; } namespace { - unsigned pass_on(unsigned number) { spdlog::info("pass_on({})", number); return number; } + unsigned pass_on(toy::VertexCollection const& vertices) { + return vertices.data; + } } TEST_CASE("provider_test") @@ -30,7 +38,7 @@ TEST_CASE("provider_test") for (unsigned int i : std::views::iota(1u, max_events + 1)) { auto store = job_store->make_child(i, "spill", "Source"); - store->add_product("number", i); + store->add_product("happy_vertices", toy::VertexCollection{i}); spdlog::info("store: is about to be yielded"); driver.yield(store); } @@ -38,14 +46,13 @@ TEST_CASE("provider_test") framework_graph g{levels_to_process}; -// g.provider(give_me_an_a).input_indices("spill").output_product("Gauss:number"); - g.transform("pass_on", pass_on, concurrency::unlimited) - .input_family("number"_in("spill")) - .output_products("different"); + g.transform("passer", pass_on, concurrency::unlimited) + .input_family("happy_vertices"_in("spill")) + .output_products("vertex_data"); g.execute(); - CHECK(g.execution_counts("pass_on") == max_events); + CHECK(g.execution_counts("passer") == max_events); } /* @@ -53,6 +60,7 @@ TEST_CASE("provider_test") Planned development flow: [x] Get initial test working. +[ ] Make the data product be a simple struct not a fundamental type. [ ] Introduce stub `provider` that takes product_store_ptr as input and returns a product_store_ptr. Wire the provider into the graph. [ ] Modify the `provider` to take data_cell_index as input. @@ -63,4 +71,5 @@ Planned development flow: This will mean `framework_driver` will have to emit data_cell_index instead of product_store. Then we're done. + */ From 81dc8618ac3e2f1489eeb097ed1ab16e87f6f179 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Wed, 19 Nov 2025 13:57:46 -0600 Subject: [PATCH 05/49] Add stub provider with struct data product Implement give_me_vertices provider algorithm that returns toy::VertexCollection struct instead of fundamental type. Wire stub provider into framework_graph with unlimited concurrency. Update VertexCollection::data to use std::size_t. --- test/provider_test.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 2c03bd5e..866c94ad 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -10,13 +10,15 @@ using namespace phlex::experimental; namespace toy { struct VertexCollection { - unsigned data; + std::size_t data; }; } namespace { // Provider algorithms - // int give_me_an_a(data_cell_index) { return 1; }; + toy::VertexCollection give_me_vertices(data_cell_index const& id) { + return toy::VertexCollection{id.number()}; + } } namespace { @@ -46,7 +48,7 @@ TEST_CASE("provider_test") framework_graph g{levels_to_process}; - + g.provider("happy_vertices", give_me_vertices, concurrency::unlimited); g.transform("passer", pass_on, concurrency::unlimited) .input_family("happy_vertices"_in("spill")) .output_products("vertex_data"); @@ -60,7 +62,7 @@ TEST_CASE("provider_test") Planned development flow: [x] Get initial test working. -[ ] Make the data product be a simple struct not a fundamental type. +[x] Make the data product be a simple struct not a fundamental type. [ ] Introduce stub `provider` that takes product_store_ptr as input and returns a product_store_ptr. Wire the provider into the graph. [ ] Modify the `provider` to take data_cell_index as input. From dc16810510602e0d0f1a2857b6b3500face507aa Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Wed, 19 Nov 2025 16:29:49 -0600 Subject: [PATCH 06/49] Add provide Method And Documentation - Add provide() member to glue class for provider registration - Document glue class template with comprehensive description of its role in flow graph registration - Document make_glue() helper explaining void_tag behavior and state management - Update provider_test to use provide() instead of provider() for consistency with other node types --- phlex/core/framework_graph.hpp | 25 +++++++++++++++++++++++++ phlex/core/glue.hpp | 24 +++++++++++++++++++++++- test/provider_test.cpp | 2 +- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/phlex/core/framework_graph.hpp b/phlex/core/framework_graph.hpp index 9e0da3dd..9a5f00d0 100644 --- a/phlex/core/framework_graph.hpp +++ b/phlex/core/framework_graph.hpp @@ -115,6 +115,31 @@ namespace phlex::experimental { } private: + /** + * Creates a glue object that binds framework components together. + * + * @tparam T Type of object to construct and bind, defaults to void_tag + * @tparam Construct Boolean flag controlling whether to construct T, defaults to true + * @tparam Args Variadic template for constructor arguments + * + * @param args Arguments forwarded to T's constructor if Construct is true + * + * @return glue A glue object containing the constructed T (if any) and framework components + * + * This helper creates a glue object that connects framework components. If T is void_tag, + * no object is constructed and the glue acts as a pure connector between framework parts. + * This is useful for simple operations that don't require maintaining state or complex logic. + * + * If T is not void_tag and Construct is true, it will construct a shared_ptr to T using + * the provided args. This allows the glue to maintain state and complex behavior through + * the lifetime of the constructed T object. + * + * The glue object binds together: + * - The TBB flow graph + * - Node catalog for tracking registered nodes + * - Optional instance of T (if not void_tag) + * - Registration error collection + */ template glue make_glue(Args&&... args) { diff --git a/phlex/core/glue.hpp b/phlex/core/glue.hpp index 733e29a3..7ce93355 100644 --- a/phlex/core/glue.hpp +++ b/phlex/core/glue.hpp @@ -25,7 +25,16 @@ namespace phlex::experimental { // ============================================================================== // Registering user functions - +/** + * @brief A class template that provides a fluent interface for registering data processing nodes in a flow graph. + * + * The glue class acts as a registration helper that allows binding user-defined functions and algorithms + * to nodes in a TBB flow graph. It provides methods to create different types of processing nodes like + * fold, transform, observe, predicate etc. + * + * @tparam T The type of the object that contains the user-defined functions/algorithms to be registered. + * This object is stored as a shared pointer and its methods are bound to the created nodes. + */ template class glue { public: @@ -67,6 +76,19 @@ namespace phlex::experimental { errors_); } + template + auto provide(std::string name, FT f, concurrency c) + { + detail::verify_name(name, config_); + return make_registration(config_, + std::move(name), + algorithm_bits{bound_obj_, std::move(f)}, + c, + graph_, + nodes_, + errors_); + } + template auto transform(std::string name, FT f, concurrency c) { diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 866c94ad..8941df0c 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -48,7 +48,7 @@ TEST_CASE("provider_test") framework_graph g{levels_to_process}; - g.provider("happy_vertices", give_me_vertices, concurrency::unlimited); + g.provide("happy_vertices", give_me_vertices, concurrency::unlimited); g.transform("passer", pass_on, concurrency::unlimited) .input_family("happy_vertices"_in("spill")) .output_products("vertex_data"); From c5ad9de6ab51c0f527c9e746cbcdf611c47a3744 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Wed, 26 Nov 2025 10:22:56 -0600 Subject: [PATCH 07/49] Add provider node support to framework Introduce declared_provider class and provider_node template for handling data input into the graph. Providers accept level_id and generate products. Add provider registration to node_catalog and glue interface. Register providers using input_family syntax for consistent product association. Update provider_test to use level_id reference and specified_label syntax for provider registration. --- phlex/core/CMakeLists.txt | 5 +- phlex/core/declared_provider.cpp | 10 +++ phlex/core/declared_provider.hpp | 101 +++++++++++++++++++++++++++++++ phlex/core/framework_graph.hpp | 19 +++--- phlex/core/glue.hpp | 20 +++--- phlex/core/node_catalog.hpp | 2 + test/provider_test.cpp | 9 ++- 7 files changed, 142 insertions(+), 24 deletions(-) create mode 100644 phlex/core/declared_provider.cpp create mode 100644 phlex/core/declared_provider.hpp diff --git a/phlex/core/CMakeLists.txt b/phlex/core/CMakeLists.txt index 080fc6f5..5cc786ad 100644 --- a/phlex/core/CMakeLists.txt +++ b/phlex/core/CMakeLists.txt @@ -5,12 +5,13 @@ cet_make_library( SOURCE concurrency.cpp consumer.cpp + declared_fold.cpp declared_observer.cpp declared_output.cpp declared_predicate.cpp - declared_fold.cpp - declared_unfold.cpp + declared_provider.cpp declared_transform.cpp + declared_unfold.cpp detail/make_algorithm_name.cpp detail/maybe_predicates.cpp detail/filter_impl.cpp diff --git a/phlex/core/declared_provider.cpp b/phlex/core/declared_provider.cpp new file mode 100644 index 00000000..74767691 --- /dev/null +++ b/phlex/core/declared_provider.cpp @@ -0,0 +1,10 @@ +#include "phlex/core/declared_provider.hpp" + +namespace phlex::experimental { + declared_provider::declared_provider(algorithm_name name, product_queries output_products) : + products_consumer{std::move(name), {}, std::move(output_products)} + { + } + + declared_provider::~declared_provider() = default; +} diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp new file mode 100644 index 00000000..f27c5ec2 --- /dev/null +++ b/phlex/core/declared_provider.hpp @@ -0,0 +1,101 @@ +#ifndef PHLEX_CORE_DECLARED_PROVIDER_HPP +#define PHLEX_CORE_DECLARED_PROVIDER_HPP + +#include "phlex/core/concepts.hpp" +#include "phlex/core/fwd.hpp" +#include "phlex/core/message.hpp" +#include "phlex/core/products_consumer.hpp" +#include "phlex/metaprogramming/type_deduction.hpp" +#include "phlex/model/algorithm_name.hpp" +#include "phlex/model/data_cell_index.hpp" +#include "phlex/model/product_specification.hpp" +#include "phlex/model/product_store.hpp" +#include "phlex/utilities/simple_ptr_map.hpp" + +#include "oneapi/tbb/concurrent_hash_map.h" +#include "oneapi/tbb/concurrent_unordered_map.h" +#include "oneapi/tbb/flow_graph.h" + +#include +#include +#include +#include +#include +#include + +namespace phlex::experimental { + + class declared_provider : public products_consumer { + public: + declared_provider(algorithm_name name, product_queries output_products); + virtual ~declared_provider(); + + virtual tbb::flow::sender& sender() = 0; + virtual tbb::flow::sender& to_output() = 0; + virtual product_specifications const& output() const = 0; + virtual std::size_t num_calls() const = 0; + }; + + using declared_provider_ptr = std::unique_ptr; + using declared_providers = simple_ptr_map; + + // ===================================================================================== + + template + class provider_node : public declared_provider { + using function_t = typename AlgorithmBits::bound_type; + static constexpr auto M = number_output_objects; + + public: + using node_ptr_type = declared_provider_ptr; + // We are setting number_output_products to 0 because that will allow the + // registration mechanism to work, for now. + static constexpr auto number_output_products = 0; + + provider_node(algorithm_name name, + std::size_t concurrency, + std::vector /*ignored */, + tbb::flow::graph& g, + AlgorithmBits alg, + product_queries output) : + // Fixme: get rid of copying of output. + declared_provider{std::move(name), output}, + output_{output[0].name}, + provider_{ + g, concurrency, [this, ft = alg.release_algorithm()](message const& msg, auto& output) { + auto& [stay_in_graph, to_output] = output; + + auto result = std::invoke(ft, *msg.store->id()); + ++calls_; + + products new_products; + // Add all adds all products; we should only have one. Fix this later. + new_products.add_all(output_, std::move(result)); + auto store = msg.store->make_continuation(this->full_name(), std::move(new_products)); + + message const new_msg{store, msg.eom, msg.id}; + stay_in_graph.try_put(new_msg); + to_output.try_put(new_msg); + }} + { + } + + tbb::flow::receiver& receiver() { return provider_; } + + tbb::flow::receiver& port_for(product_query const&) override { return provider_; } + std::vector*> ports() override { return {&provider_}; } + + private: + tbb::flow::sender& sender() override { return output_port<0>(provider_); } + tbb::flow::sender& to_output() override { return output_port<1>(provider_); } + product_specifications const& output() const override { return output_; } + + std::size_t num_calls() const final { return calls_.load(); } + product_specifications output_; + tbb::flow::multifunction_node> provider_; + std::atomic calls_; + }; + +} + +#endif // PHLEX_CORE_DECLARED_PROVIDER_HPP diff --git a/phlex/core/framework_graph.hpp b/phlex/core/framework_graph.hpp index 9a5f00d0..6fd0d7f3 100644 --- a/phlex/core/framework_graph.hpp +++ b/phlex/core/framework_graph.hpp @@ -108,6 +108,11 @@ namespace phlex::experimental { return make_glue().transform(std::move(name), std::move(f), c); } + auto provide(product_query name, auto f, concurrency c = concurrency::serial) + { + return make_glue().provide(std::move(name), std::move(f), c); + } + template glue make(Args&&... args) { @@ -117,23 +122,23 @@ namespace phlex::experimental { private: /** * Creates a glue object that binds framework components together. - * + * * @tparam T Type of object to construct and bind, defaults to void_tag * @tparam Construct Boolean flag controlling whether to construct T, defaults to true * @tparam Args Variadic template for constructor arguments - * + * * @param args Arguments forwarded to T's constructor if Construct is true - * + * * @return glue A glue object containing the constructed T (if any) and framework components - * + * * This helper creates a glue object that connects framework components. If T is void_tag, * no object is constructed and the glue acts as a pure connector between framework parts. * This is useful for simple operations that don't require maintaining state or complex logic. - * - * If T is not void_tag and Construct is true, it will construct a shared_ptr to T using + * + * If T is not void_tag and Construct is true, it will construct a shared_ptr to T using * the provided args. This allows the glue to maintain state and complex behavior through * the lifetime of the constructed T object. - * + * * The glue object binds together: * - The TBB flow graph * - Node catalog for tracking registered nodes diff --git a/phlex/core/glue.hpp b/phlex/core/glue.hpp index 7ce93355..3c14cfd6 100644 --- a/phlex/core/glue.hpp +++ b/phlex/core/glue.hpp @@ -25,7 +25,7 @@ namespace phlex::experimental { // ============================================================================== // Registering user functions -/** + /** * @brief A class template that provides a fluent interface for registering data processing nodes in a flow graph. * * The glue class acts as a registration helper that allows binding user-defined functions and algorithms @@ -77,16 +77,16 @@ namespace phlex::experimental { } template - auto provide(std::string name, FT f, concurrency c) + void provide(product_query name, FT f, concurrency c) { - detail::verify_name(name, config_); - return make_registration(config_, - std::move(name), - algorithm_bits{bound_obj_, std::move(f)}, - c, - graph_, - nodes_, - errors_); + auto reg = make_registration(config_, + name.to_string(), + algorithm_bits{bound_obj_, std::move(f)}, + c, + graph_, + nodes_, + errors_); + reg.input_family(std::move(name)); } template diff --git a/phlex/core/node_catalog.hpp b/phlex/core/node_catalog.hpp index 36123c4e..f1edc09b 100644 --- a/phlex/core/node_catalog.hpp +++ b/phlex/core/node_catalog.hpp @@ -5,6 +5,7 @@ #include "phlex/core/declared_observer.hpp" #include "phlex/core/declared_output.hpp" #include "phlex/core/declared_predicate.hpp" +#include "phlex/core/declared_provider.hpp" #include "phlex/core/declared_transform.hpp" #include "phlex/core/declared_unfold.hpp" #include "phlex/core/registrar.hpp" @@ -32,6 +33,7 @@ namespace phlex::experimental { simple_ptr_map folds{}; simple_ptr_map unfolds{}; simple_ptr_map transforms{}; + simple_ptr_map providers{}; }; } diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 8941df0c..6b8e6027 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -16,15 +16,14 @@ namespace toy { namespace { // Provider algorithms - toy::VertexCollection give_me_vertices(data_cell_index const& id) { + toy::VertexCollection give_me_vertices(data_cell_index const& id) + { return toy::VertexCollection{id.number()}; } } namespace { - unsigned pass_on(toy::VertexCollection const& vertices) { - return vertices.data; - } + unsigned pass_on(toy::VertexCollection const& vertices) { return vertices.data; } } TEST_CASE("provider_test") @@ -48,7 +47,7 @@ TEST_CASE("provider_test") framework_graph g{levels_to_process}; - g.provide("happy_vertices", give_me_vertices, concurrency::unlimited); + g.provide("happy_vertices"_in("spill"), give_me_vertices, concurrency::unlimited); g.transform("passer", pass_on, concurrency::unlimited) .input_family("happy_vertices"_in("spill")) .output_products("vertex_data"); From a71595bdb27709a6515982718a12a4c34d1c8d7f Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Tue, 2 Dec 2025 10:19:09 -0600 Subject: [PATCH 08/49] Add provider node support to framework Add provider_node template class that creates data products on-demand from level IDs. Providers are registered via the provide() API and wired into the graph by edge_maker, which matches provider outputs to consumer inputs based on product labels and families. Modify multiplexer to route messages to provider nodes based on level information. Update node_catalog to track provider execution counts. Add provider_test.cpp demonstrating provider registration and execution with a toy VertexCollection example. This development is not yet complete. All the libraries compile and link, and all the tests compile and link. But most tests that use a graph fail, because they have not yet been updated to include the providers needed to work with this version of the infrastructure. --- phlex/core/declared_provider.hpp | 8 ++++++ phlex/core/edge_maker.hpp | 44 +++++++++++++++++++++++++++++++- phlex/core/framework_graph.cpp | 1 + phlex/core/framework_graph.hpp | 2 +- phlex/core/glue.hpp | 12 +++------ phlex/core/multiplexer.cpp | 38 ++++++--------------------- phlex/core/multiplexer.hpp | 1 + phlex/core/node_catalog.cpp | 9 ++++++- test/provider_test.cpp | 29 ++++++++++++++------- 9 files changed, 93 insertions(+), 51 deletions(-) diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index f27c5ec2..03c8f375 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -15,6 +15,7 @@ #include "oneapi/tbb/concurrent_hash_map.h" #include "oneapi/tbb/concurrent_unordered_map.h" #include "oneapi/tbb/flow_graph.h" +#include "spdlog/spdlog.h" #include #include @@ -65,6 +66,11 @@ namespace phlex::experimental { g, concurrency, [this, ft = alg.release_algorithm()](message const& msg, auto& output) { auto& [stay_in_graph, to_output] = output; + if (msg.store->is_flush()) { + stay_in_graph.try_put(msg); + return; + } + auto result = std::invoke(ft, *msg.store->id()); ++calls_; @@ -78,6 +84,8 @@ namespace phlex::experimental { to_output.try_put(new_msg); }} { + spdlog::info( + "Created provider node {} making output {}", this->full_name(), output[0].to_string()); } tbb::flow::receiver& receiver() { return provider_; } diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index 1f2e6acf..643e78aa 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -7,6 +7,7 @@ #include "phlex/core/multiplexer.hpp" #include "oneapi/tbb/flow_graph.h" +#include "spdlog/spdlog.h" #include #include @@ -18,6 +19,7 @@ #include namespace phlex::experimental { + using namespace std::string_literals; using product_name_t = std::string; @@ -31,6 +33,7 @@ namespace phlex::experimental { multiplexer& multi, std::map& filters, declared_outputs& outputs, + declared_providers& providers, Args&... consumers); private: @@ -77,6 +80,7 @@ namespace phlex::experimental { multiplexer& multi, std::map& filters, declared_outputs& outputs, + declared_providers& providers, Args&... consumers) { make_edge(source, multi); @@ -92,8 +96,46 @@ namespace phlex::experimental { // Create normal edges multiplexer::head_ports_t head_ports; (head_ports.merge(edges(filters, consumers)), ...); + // Eventually, we want to look at the filled-in head_ports and + // figure out what provider nodes are needed. + // For now, we take as input a mapping of declared_providers. + + // wire up the head_ports to the given providers. + // If any head_port does not have a matching provider, that is + // an error. + multiplexer::head_ports_t provider_ports; + assert(!head_ports.empty()); + for (auto const& [_, ports] : head_ports) { + for (auto const& port : ports) { + // Find the provider that has the right product name (hidden in the + // output port) and the right family (hidden in the input port). + bool found_match = false; + for (auto const& [_, p] : providers) { + if (port.product_label == p->input()[0]) { + auto& provider = *p; + assert(provider.ports().size() == 1); + multiplexer::named_input_ports_t::value_type v{port.product_label, provider.ports()[0]}; + auto& provider_input_ports = provider_ports[provider.full_name()]; + provider_input_ports.push_back(v); + spdlog::info( + "Connecting provider {} to {}", provider.full_name(), port.product_label.to_string()); + make_edge(provider.sender(), *(port.port)); + found_match = true; + break; + } + } + if (!found_match) { + throw std::runtime_error("No provider found for product: "s + + port.product_label.to_string()); + } + } + } - multi.finalize(std::move(head_ports)); + // We must have at least one provider port, or there can be no data to + // process. + assert(!provider_ports.empty()); + // Pass the providers to the multiplexer. + multi.finalize(std::move(provider_ports)); } } diff --git a/phlex/core/framework_graph.cpp b/phlex/core/framework_graph.cpp index 46194b3b..6308fc82 100644 --- a/phlex/core/framework_graph.cpp +++ b/phlex/core/framework_graph.cpp @@ -152,6 +152,7 @@ namespace phlex::experimental { multiplexer_, filters_, nodes_.outputs, + nodes_.providers, nodes_.predicates, nodes_.observers, nodes_.folds, diff --git a/phlex/core/framework_graph.hpp b/phlex/core/framework_graph.hpp index 6fd0d7f3..9b2cdae5 100644 --- a/phlex/core/framework_graph.hpp +++ b/phlex/core/framework_graph.hpp @@ -108,7 +108,7 @@ namespace phlex::experimental { return make_glue().transform(std::move(name), std::move(f), c); } - auto provide(product_query name, auto f, concurrency c = concurrency::serial) + auto provide(std::string name, auto f, concurrency c = concurrency::serial) { return make_glue().provide(std::move(name), std::move(f), c); } diff --git a/phlex/core/glue.hpp b/phlex/core/glue.hpp index 3c14cfd6..0bef23b6 100644 --- a/phlex/core/glue.hpp +++ b/phlex/core/glue.hpp @@ -77,16 +77,10 @@ namespace phlex::experimental { } template - void provide(product_query name, FT f, concurrency c) + auto provide(std::string name, FT f, concurrency c) { - auto reg = make_registration(config_, - name.to_string(), - algorithm_bits{bound_obj_, std::move(f)}, - c, - graph_, - nodes_, - errors_); - reg.input_family(std::move(name)); + return make_registration( + config_, name, algorithm_bits{bound_obj_, std::move(f)}, c, graph_, nodes_, errors_); } template diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index ffbfeed8..a2aa7f93 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -12,27 +12,6 @@ using namespace std::chrono; namespace { - phlex::experimental::product_store_const_ptr store_for( - phlex::experimental::product_store_const_ptr store, - phlex::experimental::product_query const& label) - { - auto const& [product_name, layer] = label; - if (layer.empty()) { - return store->store_for_product(product_name.full()); - } - if (store->layer_name() == layer and store->contains_product(product_name.full())) { - return store; - } - auto parent = store->parent(layer); - if (not parent) { - return nullptr; - } - if (parent->contains_product(product_name.full())) { - return parent; - } - throw std::runtime_error( - fmt::format("Store not available that provides product {}", label.to_string())); - } struct sender_slot { tbb::flow::receiver* port; @@ -43,6 +22,9 @@ namespace { } }; + // FIXME: this function should return optional to indicate + // because it can never return more than one slot, but can return no + // sender_slot. std::vector senders_for( phlex::experimental::product_store_const_ptr store, phlex::experimental::multiplexer::named_input_ports_t const& ports) @@ -50,20 +32,16 @@ namespace { std::vector result; result.reserve(ports.size()); for (auto const& [product_label, port] : ports) { - auto store_to_send = store_for(store, product_label); - if (not store_to_send) { - // This is fine if the store is not expected to contain the product. - continue; - } - + // Look for the product_label that matches our store's layer_name. if (auto const& allowed_layer = product_label.layer; not allowed_layer.empty()) { - if (store_to_send->layer_name() != allowed_layer) { + if (store->layer_name() != allowed_layer) { + // This store level does not match the required layer continue; } } - - result.push_back({port, store_to_send}); + result.push_back({port, store}); } + assert(result.size() <= 1); return result; } } diff --git a/phlex/core/multiplexer.hpp b/phlex/core/multiplexer.hpp index bb956f29..9717e198 100644 --- a/phlex/core/multiplexer.hpp +++ b/phlex/core/multiplexer.hpp @@ -27,6 +27,7 @@ namespace phlex::experimental { tbb::flow::receiver* port; }; using named_input_ports_t = std::vector; + // map of node name to its input ports using head_ports_t = std::map; explicit multiplexer(tbb::flow::graph& g, bool debug = false); diff --git a/phlex/core/node_catalog.cpp b/phlex/core/node_catalog.cpp index 113097b7..d44a3047 100644 --- a/phlex/core/node_catalog.cpp +++ b/phlex/core/node_catalog.cpp @@ -1,5 +1,9 @@ #include "phlex/core/node_catalog.hpp" +#include + +using namespace std::string_literals; + namespace phlex::experimental { std::size_t node_catalog::execution_counts(std::string const& node_name) const { @@ -19,7 +23,10 @@ namespace phlex::experimental { if (auto node = transforms.get(node_name)) { return node->num_calls(); } - return -1u; + if (auto node = providers.get(node_name)) { + return node->num_calls(); + } + throw std::runtime_error("Unknown node type with name: "s + node_name); } std::size_t node_catalog::product_counts(std::string const& node_name) const diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 6b8e6027..d2157de0 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -18,6 +18,7 @@ namespace { // Provider algorithms toy::VertexCollection give_me_vertices(data_cell_index const& id) { + spdlog::info("give_me_vertices: {}", id.number()); return toy::VertexCollection{id.number()}; } } @@ -28,32 +29,42 @@ namespace { TEST_CASE("provider_test") { - constexpr auto max_events{1'000u}; + constexpr auto max_events{3u}; // constexpr auto max_events{1'000'000u}; - // spdlog::flush_on(spdlog::level::trace); + spdlog::flush_on(spdlog::level::trace); auto levels_to_process = [](framework_driver& driver) { auto job_store = product_store::base(); - spdlog::info("job_store: is about to be yielded"); driver.yield(job_store); for (unsigned int i : std::views::iota(1u, max_events + 1)) { auto store = job_store->make_child(i, "spill", "Source"); - store->add_product("happy_vertices", toy::VertexCollection{i}); - spdlog::info("store: is about to be yielded"); driver.yield(store); } }; framework_graph g{levels_to_process}; - g.provide("happy_vertices"_in("spill"), give_me_vertices, concurrency::unlimited); + g.provide("my_name_here", give_me_vertices, concurrency::unlimited) + .input_family( + "happy_vertices"_in("spill")); // todo: fix 'input_family'; it should be 'output_product' + g.transform("passer", pass_on, concurrency::unlimited) .input_family("happy_vertices"_in("spill")) .output_products("vertex_data"); - g.execute(); - CHECK(g.execution_counts("passer") == max_events); + spdlog::info("graph: is about to be executed"); + try { + g.execute(); + CHECK(g.execution_counts("passer") == max_events); + CHECK(g.execution_counts("my_name_here") == max_events); + } catch (std::exception const& e) { + spdlog::error("Exception during graph execution: {}", e.what()); + throw; + } catch (...) { + spdlog::error("Unknown exception during graph execution"); + throw; + } } /* @@ -62,7 +73,7 @@ Planned development flow: [x] Get initial test working. [x] Make the data product be a simple struct not a fundamental type. -[ ] Introduce stub `provider` that takes product_store_ptr as input and returns a product_store_ptr. +[x] Introduce stub `provider` that takes product_store_ptr as input and returns a product_store_ptr. Wire the provider into the graph. [ ] Modify the `provider` to take data_cell_index as input. The `multiplexer` will then need to emit data_cell_index, stripped from the `product_store` it currently returns. From 6d96e2a3b134888cef026ddb685459df0b8d530f Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 2 Dec 2025 15:50:11 -0600 Subject: [PATCH 09/49] Adjust the matching between providers and downstream ports --- phlex/core/edge_creation_policy.cpp | 4 +++- phlex/core/edge_maker.hpp | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/phlex/core/edge_creation_policy.cpp b/phlex/core/edge_creation_policy.cpp index 39b9f7f7..bc1922b8 100644 --- a/phlex/core/edge_creation_policy.cpp +++ b/phlex/core/edge_creation_policy.cpp @@ -12,7 +12,9 @@ namespace phlex::experimental { auto const& spec = query.name; auto [b, e] = producers_.equal_range(spec.name()); if (b == e) { - spdlog::debug("Failed to find {}. Assuming it is provided by the driver", spec.name()); + spdlog::debug( + "Failed to find an algorithm that creates {} products. Assuming it comes from a provider", + spec.name()); return nullptr; } std::map candidates; diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index 643e78aa..1a5469c9 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -111,7 +111,10 @@ namespace phlex::experimental { // output port) and the right family (hidden in the input port). bool found_match = false; for (auto const& [_, p] : providers) { - if (port.product_label == p->input()[0]) { + // FIXME: The check should probably be more robust. Right now, the product_specification + // buried in the p->input()[0] call does not have its type set, which prevents us from + // doing a simpler comparison (e.g., port.product_label == p->input()[0]). + if (port.product_label.name.full() == p->input()[0].name.full()) { auto& provider = *p; assert(provider.ports().size() == 1); multiplexer::named_input_ports_t::value_type v{port.product_label, provider.ports()[0]}; From 5f74b750b1f121fb4592f827e6ec11c04082cfa1 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 2 Dec 2025 16:56:43 -0600 Subject: [PATCH 10/49] Make sure there is only one entry per unique provider name --- phlex/core/edge_maker.hpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index 1a5469c9..ad587650 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -105,7 +105,8 @@ namespace phlex::experimental { // an error. multiplexer::head_ports_t provider_ports; assert(!head_ports.empty()); - for (auto const& [_, ports] : head_ports) { + + for (auto const& [node_name, ports] : head_ports) { for (auto const& port : ports) { // Find the provider that has the right product name (hidden in the // output port) and the right family (hidden in the input port). @@ -117,11 +118,16 @@ namespace phlex::experimental { if (port.product_label.name.full() == p->input()[0].name.full()) { auto& provider = *p; assert(provider.ports().size() == 1); - multiplexer::named_input_ports_t::value_type v{port.product_label, provider.ports()[0]}; - auto& provider_input_ports = provider_ports[provider.full_name()]; - provider_input_ports.push_back(v); - spdlog::info( - "Connecting provider {} to {}", provider.full_name(), port.product_label.to_string()); + auto it = provider_ports.find(provider.full_name()); + if (it == provider_ports.cend()) { + multiplexer::named_input_ports_t provider_input_ports; + provider_input_ports.emplace_back(port.product_label, provider.ports()[0]); + provider_ports.try_emplace(provider.full_name(), std::move(provider_input_ports)); + } + spdlog::info("Connecting provider {} to node {} (product: {})", + provider.full_name(), + node_name, + port.product_label.to_string()); make_edge(provider.sender(), *(port.port)); found_match = true; break; From 697fcb31f9cab8905341a0d226aff388644af2c0 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 3 Dec 2025 09:02:31 -0600 Subject: [PATCH 11/49] Change provider API from input_family to output_product --- phlex/core/declared_provider.hpp | 10 +++---- phlex/core/glue.hpp | 10 +++++-- phlex/core/registration_api.hpp | 47 ++++++++++++++++++++++++++++++++ test/provider_test.cpp | 3 +- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index 03c8f375..7bdc665b 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -55,13 +55,11 @@ namespace phlex::experimental { provider_node(algorithm_name name, std::size_t concurrency, - std::vector /*ignored */, tbb::flow::graph& g, AlgorithmBits alg, - product_queries output) : - // Fixme: get rid of copying of output. - declared_provider{std::move(name), output}, - output_{output[0].name}, + product_query output) : + declared_provider{std::move(name), {output}}, + output_{output.name}, provider_{ g, concurrency, [this, ft = alg.release_algorithm()](message const& msg, auto& output) { auto& [stay_in_graph, to_output] = output; @@ -85,7 +83,7 @@ namespace phlex::experimental { }} { spdlog::info( - "Created provider node {} making output {}", this->full_name(), output[0].to_string()); + "Created provider node {} making output {}", this->full_name(), output.to_string()); } tbb::flow::receiver& receiver() { return provider_; } diff --git a/phlex/core/glue.hpp b/phlex/core/glue.hpp index 0bef23b6..6f29eff2 100644 --- a/phlex/core/glue.hpp +++ b/phlex/core/glue.hpp @@ -79,8 +79,14 @@ namespace phlex::experimental { template auto provide(std::string name, FT f, concurrency c) { - return make_registration( - config_, name, algorithm_bits{bound_obj_, std::move(f)}, c, graph_, nodes_, errors_); + detail::verify_name(name, config_); + return provider_api{config_, + std::move(name), + algorithm_bits{bound_obj_, std::move(f)}, + c, + graph_, + nodes_, + errors_}; } template diff --git a/phlex/core/registration_api.hpp b/phlex/core/registration_api.hpp index c331e4c7..80bf4412 100644 --- a/phlex/core/registration_api.hpp +++ b/phlex/core/registration_api.hpp @@ -106,6 +106,53 @@ namespace phlex::experimental { config, std::move(name), std::move(alg), c, g, nodes, errors}; } + // ==================================================================================== + // Provider API + + template + class provider_api { + using provider_type = provider_node; + + public: + provider_api(configuration const* config, + std::string name, + AlgorithmBits alg, + concurrency c, + tbb::flow::graph& g, + node_catalog& nodes, + std::vector& errors) : + config_{config}, + name_{detail::make_algorithm_name(config, std::move(name))}, + alg_{std::move(alg)}, + concurrency_{c}, + graph_{g}, + registrar_{nodes.registrar_for(errors)} + { + } + + auto output_product(product_query output) + { + registrar_.set_creator( + [this, output = std::move(output)](auto /* predicates */, auto /* output_products */) { + return std::make_unique( + std::move(name_), concurrency_.value, graph_, std::move(alg_), std::move(output)); + }); + } + + auto output_product(label_compatible auto output) + { + return output_product({product_query::create(std::forward(output))}); + } + + private: + configuration const* config_; + algorithm_name name_; + AlgorithmBits alg_; + concurrency concurrency_; + tbb::flow::graph& graph_; + registrar registrar_; + }; + // ==================================================================================== // Fold API diff --git a/test/provider_test.cpp b/test/provider_test.cpp index d2157de0..8e0ced0b 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -46,8 +46,7 @@ TEST_CASE("provider_test") framework_graph g{levels_to_process}; g.provide("my_name_here", give_me_vertices, concurrency::unlimited) - .input_family( - "happy_vertices"_in("spill")); // todo: fix 'input_family'; it should be 'output_product' + .output_product("happy_vertices"_in("spill")); g.transform("passer", pass_on, concurrency::unlimited) .input_family("happy_vertices"_in("spill")) From 144a67d348ce2f7fab5c4fc301474417d940660e Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Fri, 5 Dec 2025 12:57:19 -0600 Subject: [PATCH 12/49] Remove output_products from provider_api --- phlex/core/registration_api.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/phlex/core/registration_api.hpp b/phlex/core/registration_api.hpp index 80bf4412..14a23d15 100644 --- a/phlex/core/registration_api.hpp +++ b/phlex/core/registration_api.hpp @@ -139,11 +139,6 @@ namespace phlex::experimental { }); } - auto output_product(label_compatible auto output) - { - return output_product({product_query::create(std::forward(output))}); - } - private: configuration const* config_; algorithm_name name_; From 19c78f02abd73707e8de104a2c920d893fe31dc2 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 2 Dec 2025 16:07:28 -0600 Subject: [PATCH 13/49] Add providers for tests --- test/fold.cpp | 21 ++++++++++---------- test/hierarchical_nodes.cpp | 26 ++++++++++++++++++++----- test/multiple_function_registration.cpp | 11 +++++++---- test/unfold.cpp | 16 +++++++++++---- test/vector_of_abstract_types.cpp | 6 +++--- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/test/fold.cpp b/test/fold.cpp index 2cc17521..b388672b 100644 --- a/test/fold.cpp +++ b/test/fold.cpp @@ -54,9 +54,7 @@ TEST_CASE("Different data layers of fold", "[graph]") auto run_store = job_store->make_child(i, "run"); driver.yield(run_store); for (unsigned j : std::views::iota(0u, number_limit)) { - auto event_store = run_store->make_child(j, "event"); - event_store->add_product("number", j); - driver.yield(event_store); + driver.yield(run_store->make_child(j, "event")); } } }; @@ -64,6 +62,12 @@ TEST_CASE("Different data layers of fold", "[graph]") // framework_graph g{cells_to_process}; framework_graph g{cells_to_process}; + g.provide( + "provide_number", + [](data_cell_index const& index) -> unsigned int { return index.number(); }, + concurrency::unlimited) + .output_product("number"_in("event")); + g.fold("run_add", add, concurrency::unlimited, "run") .input_family("number"_in("event")) .output_products("run_sum"); @@ -75,16 +79,11 @@ TEST_CASE("Different data layers of fold", "[graph]") .input_family("run_sum"_in("run")) .output_products("two_layer_job_sum"); - g.observe( - "verify_run_sum", [](unsigned int actual) { CHECK(actual == 10u); }, concurrency::unlimited) + g.observe("verify_run_sum", [](unsigned int actual) { CHECK(actual == 10u); }) .input_family("run_sum"_in("run")); - g.observe( - "verify_two_layer_job_sum", - [](unsigned int actual) { CHECK(actual == 20u); }, - concurrency::unlimited) + g.observe("verify_two_layer_job_sum", [](unsigned int actual) { CHECK(actual == 20u); }) .input_family("two_layer_job_sum"_in("job")); - g.observe( - "verify_job_sum", [](unsigned int actual) { CHECK(actual == 20u); }, concurrency::unlimited) + g.observe("verify_job_sum", [](unsigned int actual) { CHECK(actual == 20u); }) .input_family("job_sum"_in("job")); g.execute(); diff --git a/test/hierarchical_nodes.cpp b/test/hierarchical_nodes.cpp index 773c042e..5ea3619e 100644 --- a/test/hierarchical_nodes.cpp +++ b/test/hierarchical_nodes.cpp @@ -44,12 +44,9 @@ namespace { driver.yield(job_store); for (unsigned i : std::views::iota(0u, index_limit)) { auto run_store = job_store->make_child(i, "run", "cells_to_process"); - run_store->add_product("time", std::time(nullptr)); driver.yield(run_store); for (unsigned j : std::views::iota(0u, number_limit)) { - auto event_store = run_store->make_child(j, "event", "cells_to_process"); - event_store->add_product("number", i + j); - driver.yield(event_store); + driver.yield(run_store->make_child(j, "event", "cells_to_process")); } } } @@ -102,6 +99,21 @@ TEST_CASE("Hierarchical nodes", "[graph]") { framework_graph g{cells_to_process}; + g.provide("provide_time", + [](data_cell_index const& index) -> std::time_t { + spdlog::info("Providing time for {}", index.to_string()); + return std::time(nullptr); + }) + .output_product("time"_in("run")); + + g.provide("provide_number", + [](data_cell_index const& index) -> unsigned int { + auto const event_number = index.number(); + auto const run_number = index.parent()->number(); + return event_number + run_number; + }) + .output_product("number"_in("event")); + g.transform("get_the_time", strtime, concurrency::unlimited) .input_family("time"_in("run")) .when() @@ -123,7 +135,11 @@ TEST_CASE("Hierarchical nodes", "[graph]") g.make().output("save", &test::products_for_output::save).when(); - g.execute(); + try { + g.execute(); + } catch (std::exception const& e) { + spdlog::error(e.what()); + } CHECK(g.execution_counts("square") == index_limit * number_limit); CHECK(g.execution_counts("add") == index_limit * number_limit); diff --git a/test/multiple_function_registration.cpp b/test/multiple_function_registration.cpp index 7f106c52..8c7ebf90 100644 --- a/test/multiple_function_registration.cpp +++ b/test/multiple_function_registration.cpp @@ -41,10 +41,13 @@ namespace { TEST_CASE("Call multiple functions", "[programming model]") { - auto store = product_store::base(); - store->add_product("numbers", std::vector{0, 1, 2, 3, 4}); - store->add_product("offset", 6u); - framework_graph g{store}; + framework_graph g{product_store::base()}; + + g.provide("provide_numbers", + [](data_cell_index const&) -> std::vector { return {0, 1, 2, 3, 4}; }) + .output_product("numbers"_in("job")); + g.provide("provide_offset", [](data_cell_index const&) -> unsigned { return 6u; }) + .output_product("offset"_in("job")); SECTION("All free functions") { diff --git a/test/unfold.cpp b/test/unfold.cpp index 89923987..31a51e32 100644 --- a/test/unfold.cpp +++ b/test/unfold.cpp @@ -88,15 +88,23 @@ TEST_CASE("Splitting the processing", "[graph]") auto job_store = product_store::base(); driver.yield(job_store); for (unsigned i : std::views::iota(0u, index_limit)) { - auto event_store = job_store->make_child(i, "event"); - event_store->add_product("max_number", 10u * (i + 1)); - event_store->add_product("ten_numbers", numbers_t(10, i + 1)); - driver.yield(event_store); + driver.yield(job_store->make_child(i, "event")); } }; framework_graph g{cells_to_process}; + g.provide( + "provide_max_number", + [](data_cell_index const& index) -> unsigned { return 10u * (index.number() + 1); }, + concurrency::unlimited) + .output_product("max_number"_in("event")); + g.provide( + "provide_ten_numbers", + [](data_cell_index const& index) -> numbers_t { return numbers_t(10, index.number() + 1); }, + concurrency::unlimited) + .output_product("ten_numbers"_in("event")); + g.unfold("iota", &iota::predicate, &iota::unfold, concurrency::unlimited, "lower1") .input_family("max_number"_in("event")) .output_products("new_number"); diff --git a/test/vector_of_abstract_types.cpp b/test/vector_of_abstract_types.cpp index 8588beba..f692a0c5 100644 --- a/test/vector_of_abstract_types.cpp +++ b/test/vector_of_abstract_types.cpp @@ -45,9 +45,7 @@ namespace { driver.yield(job_store); for (unsigned int i : std::views::iota(1u, max_ + 1)) { - auto store = job_store->make_child(i, "event"); - store->add_product("thing", make_derived_as_abstract()); - driver.yield(store); + driver.yield(job_store->make_child(i, "event")); } } @@ -60,6 +58,8 @@ namespace { TEST_CASE("Test vector of abstract types") { framework_graph g{source{1u}}; + g.provide("provide_thing", [](data_cell_index const&) { return make_derived_as_abstract(); }) + .output_product("thing"_in("event")); g.transform("read_thing", read_abstract).input_family("thing"_in("event")).output_products("sum"); g.observe( "verify_sum", [](int sum) { CHECK(sum == 3); }, concurrency::serial) From bed7e958f14eb6e542bd5e51a8c32e563cc859a5 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Fri, 5 Dec 2025 13:12:39 -0600 Subject: [PATCH 14/49] Add providers for tests. --- test/filter.cpp | 17 ++++++++++++++--- test/fold.cpp | 13 +++++++------ test/unfold.cpp | 21 ++++++++++++--------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/test/filter.cpp b/test/filter.cpp index 9e1efacc..533a356e 100644 --- a/test/filter.cpp +++ b/test/filter.cpp @@ -1,4 +1,5 @@ #include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/model/product_store.hpp" #include "catch2/catch_test_macros.hpp" @@ -18,9 +19,7 @@ namespace { driver.yield(job_store); for (unsigned int i : std::views::iota(1u, max_ + 1)) { - auto store = job_store->make_child(i, "event"); - store->add_product("num", i - 1); - store->add_product("other_num", 100 + i - 1); + auto store = job_store->make_child(i, "event", "Source"); driver.yield(store); } } @@ -29,6 +28,11 @@ namespace { unsigned const max_; }; + // Provider algorithms + unsigned int give_me_nums(data_cell_index const& id) { return id.number() - 1; } + + unsigned int give_me_other_nums(data_cell_index const& id) { return 100 + id.number() - 1; } + constexpr bool evens_only(unsigned int const value) { return value % 2u == 0u; } constexpr bool odds_only(unsigned int const value) { return not evens_only(value); } @@ -85,6 +89,7 @@ namespace { TEST_CASE("Two predicates", "[filtering]") { framework_graph g{source{10u}}; + g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); g.predicate("odds_only", odds_only, concurrency::unlimited).input_family("num"_in("event")); g.make(20u) @@ -105,6 +110,7 @@ TEST_CASE("Two predicates", "[filtering]") TEST_CASE("Two predicates in series", "[filtering]") { framework_graph g{source{10u}}; + g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); g.predicate("odds_only", odds_only, concurrency::unlimited) .input_family("num"_in("event")) @@ -122,6 +128,7 @@ TEST_CASE("Two predicates in series", "[filtering]") TEST_CASE("Two predicates in parallel", "[filtering]") { framework_graph g{source{10u}}; + g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); g.predicate("odds_only", odds_only, concurrency::unlimited).input_family("num"_in("event")); g.make(0u) @@ -146,6 +153,7 @@ TEST_CASE("Three predicates in parallel", "[filtering]") {.name = "exclude_gt_8", .begin = 8, .end = -1u}}; framework_graph g{source{10u}}; + g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); for (auto const& [name, b, e] : configs) { g.make(b, e) .predicate(name, ¬_in_range::eval, concurrency::unlimited) @@ -168,6 +176,9 @@ TEST_CASE("Three predicates in parallel", "[filtering]") TEST_CASE("Two predicates in parallel (each with multiple arguments)", "[filtering]") { framework_graph g{source{10u}}; + g.provide("provide_num", give_me_nums, concurrency::unlimited).output_product("num"_in("event")); + g.provide("provide_other_num", give_me_other_nums, concurrency::unlimited) + .output_product("other_num"_in("event")); g.predicate("evens_only", evens_only, concurrency::unlimited).input_family("num"_in("event")); g.predicate("odds_only", odds_only, concurrency::unlimited).input_family("num"_in("event")); g.make(5 * 100) diff --git a/test/fold.cpp b/test/fold.cpp index b388672b..7726b920 100644 --- a/test/fold.cpp +++ b/test/fold.cpp @@ -40,6 +40,9 @@ using namespace phlex::experimental; namespace { void add(std::atomic& counter, unsigned int number) { counter += number; } + + // Provider algorithm + unsigned int provide_number(data_cell_index const& id) { return id.number(); } } TEST_CASE("Different data layers of fold", "[graph]") @@ -51,10 +54,11 @@ TEST_CASE("Different data layers of fold", "[graph]") auto job_store = product_store::base(); driver.yield(job_store); for (unsigned i : std::views::iota(0u, index_limit)) { - auto run_store = job_store->make_child(i, "run"); + auto run_store = job_store->make_child(i, "run", "Source"); driver.yield(run_store); for (unsigned j : std::views::iota(0u, number_limit)) { - driver.yield(run_store->make_child(j, "event")); + auto event_store = run_store->make_child(j, "event", "Source"); + driver.yield(event_store); } } }; @@ -62,10 +66,7 @@ TEST_CASE("Different data layers of fold", "[graph]") // framework_graph g{cells_to_process}; framework_graph g{cells_to_process}; - g.provide( - "provide_number", - [](data_cell_index const& index) -> unsigned int { return index.number(); }, - concurrency::unlimited) + g.provide("provide_number", provide_number, concurrency::unlimited) .output_product("number"_in("event")); g.fold("run_add", add, concurrency::unlimited, "run") diff --git a/test/unfold.cpp b/test/unfold.cpp index 31a51e32..f1c6206f 100644 --- a/test/unfold.cpp +++ b/test/unfold.cpp @@ -78,6 +78,14 @@ namespace { auto const expected_sum = (sum.data_cell_index().number() + 1) * 10; CHECK(*sum == expected_sum); } + + // Provider algorithms + unsigned int provide_max_number(data_cell_index const& id) { return 10u * (id.number() + 1); } + + numbers_t provide_ten_numbers(data_cell_index const& id) + { + return numbers_t(10, id.number() + 1); + } } TEST_CASE("Splitting the processing", "[graph]") @@ -88,21 +96,16 @@ TEST_CASE("Splitting the processing", "[graph]") auto job_store = product_store::base(); driver.yield(job_store); for (unsigned i : std::views::iota(0u, index_limit)) { - driver.yield(job_store->make_child(i, "event")); + auto event_store = job_store->make_child(i, "event", "Source"); + driver.yield(event_store); } }; framework_graph g{cells_to_process}; - g.provide( - "provide_max_number", - [](data_cell_index const& index) -> unsigned { return 10u * (index.number() + 1); }, - concurrency::unlimited) + g.provide("provide_max_number", provide_max_number, concurrency::unlimited) .output_product("max_number"_in("event")); - g.provide( - "provide_ten_numbers", - [](data_cell_index const& index) -> numbers_t { return numbers_t(10, index.number() + 1); }, - concurrency::unlimited) + g.provide("provide_ten_numbers", provide_ten_numbers, concurrency::unlimited) .output_product("ten_numbers"_in("event")); g.unfold("iota", &iota::predicate, &iota::unfold, concurrency::unlimited, "lower1") From b1b217f96f693554966e0b3d9f63fbc77a459abe Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Fri, 5 Dec 2025 13:42:15 -0600 Subject: [PATCH 15/49] Modify tests to use providers --- test/allowed_families.cpp | 19 +++++++++++++++---- test/class_registration.cpp | 26 ++++++++++++++++++++------ test/function_registration.cpp | 20 +++++++++++++++----- test/type_distinction.cpp | 14 ++++++++++++-- 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index ee64cf34..ff373f75 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -14,22 +14,25 @@ namespace { void cells_to_process(framework_driver& driver) { auto job_store = product_store::base(); - job_store->add_product("id", *job_store->id()); driver.yield(job_store); auto run_store = job_store->make_child(0, "run"); - run_store->add_product("id", *run_store->id()); driver.yield(run_store); auto subrun_store = run_store->make_child(0, "subrun"); - subrun_store->add_product("id", *subrun_store->id()); driver.yield(subrun_store); auto event_store = subrun_store->make_child(0, "event"); - event_store->add_product("id", *event_store->id()); driver.yield(event_store); } + // Provider functions that return data_cell_index for each level + data_cell_index provide_run_id(data_cell_index const& index) { return index; } + + data_cell_index provide_subrun_id(data_cell_index const& index) { return index; } + + data_cell_index provide_event_id(data_cell_index const& index) { return index; } + void check_two_ids(data_cell_index const& parent_id, data_cell_index const& id) { CHECK(parent_id.depth() + 1ull == id.depth()); @@ -53,6 +56,14 @@ namespace { TEST_CASE("Testing families", "[data model]") { framework_graph g{cells_to_process, 2}; + + // Wire up providers for each level + g.provide("run_id_provider", provide_run_id, concurrency::unlimited).output_product("id"_in("run")); + g.provide("subrun_id_provider", provide_subrun_id, concurrency::unlimited) + .output_product("id"_in("subrun")); + g.provide("event_id_provider", provide_event_id, concurrency::unlimited) + .output_product("id"_in("event")); + g.observe("se", check_two_ids).input_family("id"_in("subrun"), "id"_in("event")); g.observe("rs", check_two_ids).input_family("id"_in("run"), "id"_in("subrun")); g.observe("rse", check_three_ids) diff --git a/test/class_registration.cpp b/test/class_registration.cpp index a5c2d334..366ab24e 100644 --- a/test/class_registration.cpp +++ b/test/class_registration.cpp @@ -1,4 +1,5 @@ #include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/model/product_store.hpp" #include "catch2/catch_test_macros.hpp" @@ -10,6 +11,15 @@ using namespace std::string_literals; using namespace phlex::experimental; +namespace { + // Provider functions + int provide_number(data_cell_index const&) { return 3; } + + double provide_temperature(data_cell_index const&) { return 98.5; } + + std::string provide_name(data_cell_index const&) { return "John"; } +} + namespace { struct A { auto no_framework(int num, double temp, std::string const& name) const @@ -52,12 +62,16 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const product_names{"number"_in("job"), "temperature"_in("job"), "name"_in("job")}; std::array const oproduct_names{"onumber"s, "otemperature"s, "oname"s}; - auto store = product_store::base(); - store->add_product("number", 3); - store->add_product("temperature", 98.5); - store->add_product("name", std::string{"John"}); - - framework_graph g{store}; + framework_graph g{product_store::base()}; + + // Register providers for the input products + g.provide("provide_number", provide_number, concurrency::unlimited) + .output_product("number"_in("job")); + g.provide("provide_temperature", provide_temperature, concurrency::unlimited) + .output_product("temperature"_in("job")); + g.provide("provide_name", provide_name, concurrency::unlimited) + .output_product("name"_in("job")); + auto glueball = g.make(); SECTION("No framework") { diff --git a/test/function_registration.cpp b/test/function_registration.cpp index 5a0a0d29..6ce7585e 100644 --- a/test/function_registration.cpp +++ b/test/function_registration.cpp @@ -1,4 +1,5 @@ #include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/model/product_store.hpp" #include "catch2/catch_test_macros.hpp" @@ -11,6 +12,11 @@ using namespace phlex::experimental; using namespace std::string_literals; namespace { + // Provider functions + int provide_number(data_cell_index const&) { return 3; } + double provide_temperature(data_cell_index const&) { return 98.5; } + std::string provide_name(data_cell_index const&) { return "John"; } + auto no_framework(int num, double temp, std::string const& name) { return std::make_tuple(num, temp, name); @@ -52,12 +58,16 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const oproduct_names = {"onumber"s, "otemperature"s, "oname"s}; std::array const result{"result"s}; - auto store = product_store::base(); - store->add_product("number", 3); - store->add_product("temperature", 98.5); - store->add_product("name", std::string{"John"}); + framework_graph g {product_store::base()}; + + // Register providers + g.provide("provide_number", provide_number, concurrency::unlimited) + .output_product("number"_in("job")); + g.provide("provide_temperature", provide_temperature, concurrency::unlimited) + .output_product("temperature"_in("job")); + g.provide("provide_name", provide_name, concurrency::unlimited) + .output_product("name"_in("job")); - framework_graph g{store}; SECTION("No framework") { g.transform("no_framework", no_framework) diff --git a/test/type_distinction.cpp b/test/type_distinction.cpp index e9330623..a905b8cf 100644 --- a/test/type_distinction.cpp +++ b/test/type_distinction.cpp @@ -1,4 +1,5 @@ #include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/model/product_store.hpp" #include "spdlog/spdlog.h" @@ -11,6 +12,11 @@ using namespace phlex::experimental; namespace { + // Provider functions + int provide_numbers(data_cell_index const& index) { return static_cast(index.number()); } + + std::size_t provide_length(data_cell_index const& index) { return index.number(); } + auto add_numbers(int x, int y) { return x + y; } auto triple(int x) { return 3 * x; } @@ -43,14 +49,18 @@ TEST_CASE("Distinguish products with same name and different types", "[programmi driver.yield(job_store); for (int i : numbers) { auto event_store = job_store->make_child(unsigned(i), "event"); - event_store->add_product("numbers", +i); - event_store->add_product("length", +i); driver.yield(event_store); } }; framework_graph g{gen}; + // Register providers + g.provide("provide_numbers", provide_numbers, concurrency::unlimited) + .output_product("numbers"_in("event")); + g.provide("provide_length", provide_length, concurrency::unlimited) + .output_product("length"_in("event")); + SECTION("Duplicate product name but differ in producer name") { g.observe("starter", [](int num) { spdlog::info("Received {}", num); }) From 223dc18f8a62648c0050836e4b1535022be0772d Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Fri, 5 Dec 2025 14:20:23 -0600 Subject: [PATCH 16/49] Modify more tests to use providers --- test/cached_execution.cpp | 14 ++++++++++++++ test/cached_execution_source.hpp | 3 --- test/different_hierarchies.cpp | 10 ++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/test/cached_execution.cpp b/test/cached_execution.cpp index ed1d607c..2f188528 100644 --- a/test/cached_execution.cpp +++ b/test/cached_execution.cpp @@ -29,6 +29,7 @@ // ======================================================================================= #include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/source.hpp" #include "test/cached_execution_source.hpp" @@ -38,6 +39,11 @@ using namespace phlex::experimental; using namespace test; namespace { + // Provider functions + int provide_number(data_cell_index const& index) { return 2 * index.number(); } + int provide_another(data_cell_index const& index) { return 3 * index.number(); } + int provide_still(data_cell_index const& index) { return 4 * index.number(); } + int call_one(int) noexcept { return 1; } int call_two(int, int) noexcept { return 2; } } @@ -46,6 +52,14 @@ TEST_CASE("Cached function calls", "[data model]") { framework_graph g{detail::create_next()}; + // Register providers + g.provide("provide_number", provide_number, concurrency::unlimited) + .output_product("number"_in("run")); + g.provide("provide_another", provide_another, concurrency::unlimited) + .output_product("another"_in("subrun")); + g.provide("provide_still", provide_still, concurrency::unlimited) + .output_product("still"_in("event")); + g.transform("A1", call_one, concurrency::unlimited) .input_family("number"_in("run")) .output_products("one"); diff --git a/test/cached_execution_source.hpp b/test/cached_execution_source.hpp index 41643edc..14968e80 100644 --- a/test/cached_execution_source.hpp +++ b/test/cached_execution_source.hpp @@ -29,17 +29,14 @@ namespace test { for (std::size_t i : std::views::iota(0u, n_runs)) { auto run_store = job_store->make_child(i, "run"); - run_store->add_product("number", 2 * i); driver.yield(run_store); for (std::size_t j : std::views::iota(0u, n_subruns)) { auto subrun_store = run_store->make_child(j, "subrun"); - subrun_store->add_product("another", 3 * j); driver.yield(subrun_store); for (std::size_t k : std::views::iota(0u, n_events)) { auto event_store = subrun_store->make_child(k, "event"); - event_store->add_product("still", 4 * k); driver.yield(event_store); } } diff --git a/test/different_hierarchies.cpp b/test/different_hierarchies.cpp index f7ce4a9d..32b203d6 100644 --- a/test/different_hierarchies.cpp +++ b/test/different_hierarchies.cpp @@ -31,6 +31,7 @@ // ======================================================================================= #include "phlex/core/framework_graph.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/model/product_store.hpp" #include "catch2/catch_test_macros.hpp" @@ -45,6 +46,9 @@ using namespace phlex::experimental; namespace { + // Provider function + unsigned int provide_number(data_cell_index const& index) { return index.number(); } + void add(std::atomic& counter, unsigned int number) { counter += number; } // job -> run -> event layers @@ -65,7 +69,6 @@ namespace { driver.yield(run_store); for (unsigned j : std::views::iota(0u, number_limit)) { auto event_store = run_store->make_child(j, "event"); - event_store->add_product("number", j); driver.yield(event_store); } } @@ -73,7 +76,6 @@ namespace { // job -> event layers for (unsigned i : std::views::iota(0u, top_level_event_limit)) { auto tp_store = job_store->make_child(i, "event"); - tp_store->add_product("number", i); driver.yield(tp_store); } } @@ -83,6 +85,10 @@ TEST_CASE("Different hierarchies used with fold", "[graph]") { framework_graph g{cells_to_process}; + // Register provider + g.provide("provide_number", provide_number, concurrency::unlimited) + .output_product("number"_in("event")); + g.fold("run_add", add, concurrency::unlimited, "run", 0u) .input_family("number"_in("event")) .output_products("run_sum"); From bcd4ecf3255e6ed25a50a53107c3b769d0cc8c76 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Mon, 8 Dec 2025 13:01:26 -0600 Subject: [PATCH 17/49] Add provider support to graph_proxy --- phlex/core/concepts.hpp | 4 ++++ phlex/core/graph_proxy.hpp | 5 +++++ test/benchmarks/CMakeLists.txt | 2 +- test/benchmarks/benchmarks_source.cpp | 9 ++++++++- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/phlex/core/concepts.hpp b/phlex/core/concepts.hpp index 8b4402b1..4fb112f7 100644 --- a/phlex/core/concepts.hpp +++ b/phlex/core/concepts.hpp @@ -56,6 +56,10 @@ namespace phlex::experimental { concept is_output_like = std::is_member_function_pointer_v && expects_input_parameters && returns; + template + concept is_provider_like = + expects_input_parameters && number_output_objects == 1ull; + template concept is_fold_like = at_least_two_input_parameters && first_input_parameter_is_non_const_lvalue_reference && diff --git a/phlex/core/graph_proxy.hpp b/phlex/core/graph_proxy.hpp index b73b7a27..34665171 100644 --- a/phlex/core/graph_proxy.hpp +++ b/phlex/core/graph_proxy.hpp @@ -66,6 +66,11 @@ namespace phlex::experimental { { return create_glue().predicate(std::move(name), std::move(f), c); } + + auto provide(std::string name, is_provider_like auto f, concurrency c = concurrency::serial) + { + return create_glue().provide(std::move(name), std::move(f), c); + } auto transform(std::string name, is_transform_like auto f, concurrency c = concurrency::serial) { diff --git a/test/benchmarks/CMakeLists.txt b/test/benchmarks/CMakeLists.txt index 8a0ae683..b5ca080b 100644 --- a/test/benchmarks/CMakeLists.txt +++ b/test/benchmarks/CMakeLists.txt @@ -2,7 +2,7 @@ add_library(fibonacci_numbers SHARED fibonacci_numbers.cpp) target_include_directories(fibonacci_numbers PRIVATE ${PROJECT_SOURCE_DIR}) add_library(benchmarks_source MODULE benchmarks_source.cpp) -target_link_libraries(benchmarks_source PRIVATE phlex::model Boost::json) +target_link_libraries(benchmarks_source PRIVATE phlex::model phlex::module) add_library(last_index MODULE last_index.cpp) target_link_libraries(last_index PRIVATE phlex::module) diff --git a/test/benchmarks/benchmarks_source.cpp b/test/benchmarks/benchmarks_source.cpp index 0bca7b92..0ae5be79 100644 --- a/test/benchmarks/benchmarks_source.cpp +++ b/test/benchmarks/benchmarks_source.cpp @@ -2,6 +2,7 @@ // This source creates 1M events. // =================================================================== +#include "phlex/module.hpp" #include "phlex/source.hpp" #include "fmt/std.h" @@ -29,7 +30,6 @@ namespace test { } auto store = job_store->make_child(i, "event"); - store->add_product("id", *store->id()); driver.yield(store); } } @@ -40,3 +40,10 @@ namespace test { } PHLEX_EXPERIMENTAL_REGISTER_SOURCE(test::benchmarks_source) +PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { + using namespace phlex::experimental; + m.provide("provide_id", [](data_cell_index const& id) { + return id; + }) + .output_product("id"_in("event")); +} From 9f63d3f38f0fb5d472bb4a9c1a29e97a1a79dd44 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 13:37:12 -0600 Subject: [PATCH 18/49] added providers to benchmark tests --- test/benchmarks/CMakeLists.txt | 3 +++ test/benchmarks/benchmark-01.jsonnet | 5 ++++- test/benchmarks/benchmark-02.jsonnet | 3 +++ test/benchmarks/benchmark-03.jsonnet | 3 +++ test/benchmarks/benchmark-04.jsonnet | 3 +++ test/benchmarks/benchmark-05.jsonnet | 3 +++ test/benchmarks/benchmark-06.jsonnet | 3 +++ test/benchmarks/benchmark-07.jsonnet | 7 +++++-- test/benchmarks/benchmark-08.jsonnet | 3 +++ test/benchmarks/benchmark-09.jsonnet | 3 +++ test/benchmarks/benchmarks_provider.cpp | 9 +++++++++ test/benchmarks/benchmarks_source.cpp | 8 -------- 12 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 test/benchmarks/benchmarks_provider.cpp diff --git a/test/benchmarks/CMakeLists.txt b/test/benchmarks/CMakeLists.txt index b5ca080b..dd8a9fd7 100644 --- a/test/benchmarks/CMakeLists.txt +++ b/test/benchmarks/CMakeLists.txt @@ -4,6 +4,9 @@ target_include_directories(fibonacci_numbers PRIVATE ${PROJECT_SOURCE_DIR}) add_library(benchmarks_source MODULE benchmarks_source.cpp) target_link_libraries(benchmarks_source PRIVATE phlex::model phlex::module) +add_library(benchmarks_provider MODULE benchmarks_provider.cpp) +target_link_libraries(benchmarks_provider PRIVATE phlex::model phlex::module) + add_library(last_index MODULE last_index.cpp) target_link_libraries(last_index PRIVATE phlex::module) diff --git a/test/benchmarks/benchmark-01.jsonnet b/test/benchmarks/benchmark-01.jsonnet index 305edec4..62433154 100644 --- a/test/benchmarks/benchmark-01.jsonnet +++ b/test/benchmarks/benchmark-01.jsonnet @@ -7,5 +7,8 @@ a_creator: { plugin: 'last_index', }, - }, + provider: { + plugin: 'benchmarks_provider' + } + }, } diff --git a/test/benchmarks/benchmark-02.jsonnet b/test/benchmarks/benchmark-02.jsonnet index d2369dd1..f248db6d 100644 --- a/test/benchmarks/benchmark-02.jsonnet +++ b/test/benchmarks/benchmark-02.jsonnet @@ -12,5 +12,8 @@ plugin: 'last_index', product_name: "a2" }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-03.jsonnet b/test/benchmarks/benchmark-03.jsonnet index ff207c34..f9a8c619 100644 --- a/test/benchmarks/benchmark-03.jsonnet +++ b/test/benchmarks/benchmark-03.jsonnet @@ -7,5 +7,8 @@ read_id: { plugin: 'read_id', }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-04.jsonnet b/test/benchmarks/benchmark-04.jsonnet index 2a597beb..5e3ccb13 100644 --- a/test/benchmarks/benchmark-04.jsonnet +++ b/test/benchmarks/benchmark-04.jsonnet @@ -11,5 +11,8 @@ plugin: 'read_index', consumes: 'a' }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-05.jsonnet b/test/benchmarks/benchmark-05.jsonnet index 9c686fd7..bc51596b 100644 --- a/test/benchmarks/benchmark-05.jsonnet +++ b/test/benchmarks/benchmark-05.jsonnet @@ -16,5 +16,8 @@ plugin: 'verify_difference', expected: 0 }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-06.jsonnet b/test/benchmarks/benchmark-06.jsonnet index ffbb9f37..a5500ab2 100644 --- a/test/benchmarks/benchmark-06.jsonnet +++ b/test/benchmarks/benchmark-06.jsonnet @@ -16,5 +16,8 @@ d: { plugin: 'verify_difference', }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-07.jsonnet b/test/benchmarks/benchmark-07.jsonnet index 830ecb74..f6eab7b4 100644 --- a/test/benchmarks/benchmark-07.jsonnet +++ b/test/benchmarks/benchmark-07.jsonnet @@ -11,16 +11,19 @@ b_creator: { plugin: 'last_index', when: ['even_filter:accept_even_ids'], - product_name: 'b', + produces: 'b', }, c_creator: { plugin: 'last_index', when: ['even_filter:accept_even_ids'], - product_name: 'c', + produces: 'c', }, d: { plugin: 'verify_difference', expected: 0 }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-08.jsonnet b/test/benchmarks/benchmark-08.jsonnet index aeac8e6c..bf89edae 100644 --- a/test/benchmarks/benchmark-08.jsonnet +++ b/test/benchmarks/benchmark-08.jsonnet @@ -25,5 +25,8 @@ local max_number = 100000; consumes: 'a', max_number: max_number, }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmark-09.jsonnet b/test/benchmarks/benchmark-09.jsonnet index b0aa7fac..2a39a34f 100644 --- a/test/benchmarks/benchmark-09.jsonnet +++ b/test/benchmarks/benchmark-09.jsonnet @@ -19,5 +19,8 @@ when: ['even_filter:accept_even_numbers'], consumes: 'b', }, + provider: { + plugin: 'benchmarks_provider' + } }, } diff --git a/test/benchmarks/benchmarks_provider.cpp b/test/benchmarks/benchmarks_provider.cpp new file mode 100644 index 00000000..295e01a4 --- /dev/null +++ b/test/benchmarks/benchmarks_provider.cpp @@ -0,0 +1,9 @@ +#include "phlex/module.hpp" + +PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { + using namespace phlex::experimental; + m.provide("provide_id", [](data_cell_index const& id) { + return id; + }) + .output_product("id"_in("event")); +} diff --git a/test/benchmarks/benchmarks_source.cpp b/test/benchmarks/benchmarks_source.cpp index 0ae5be79..582249b6 100644 --- a/test/benchmarks/benchmarks_source.cpp +++ b/test/benchmarks/benchmarks_source.cpp @@ -2,7 +2,6 @@ // This source creates 1M events. // =================================================================== -#include "phlex/module.hpp" #include "phlex/source.hpp" #include "fmt/std.h" @@ -40,10 +39,3 @@ namespace test { } PHLEX_EXPERIMENTAL_REGISTER_SOURCE(test::benchmarks_source) -PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { - using namespace phlex::experimental; - m.provide("provide_id", [](data_cell_index const& id) { - return id; - }) - .output_product("id"_in("event")); -} From fc8d46af7083ffea4d6dd09fc8f5a97f1960ba1b Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 13:42:54 -0600 Subject: [PATCH 19/49] added providers to check_parallelism tests --- test/max-parallelism/check_parallelism.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/max-parallelism/check_parallelism.cpp b/test/max-parallelism/check_parallelism.cpp index 4f14b0f3..6fca657a 100644 --- a/test/max-parallelism/check_parallelism.cpp +++ b/test/max-parallelism/check_parallelism.cpp @@ -22,7 +22,6 @@ namespace { void next(framework_driver& driver) { auto job_store = product_store::base(); - job_store->add_product("max_parallelism", max_allowed_parallelism::active_value()); driver.yield(job_store); } }; @@ -32,6 +31,12 @@ namespace { PHLEX_EXPERIMENTAL_REGISTER_SOURCE(send_parallelism) PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m, config) { + m.provide("provide_max_parallelism", + [](data_cell_index const&) { + return max_allowed_parallelism::active_value(); + }) + .output_product("max_parallelism"_in("job")); + m.observe("verify_expected", [expected = config.get("expected_parallelism")](std::size_t actual) { assert(actual == expected); From 2811ad1658a0d3dd5c7e6544ec3f926c731586a2 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 13:53:00 -0600 Subject: [PATCH 20/49] added providers to the job:add test --- test/plugins/module.cpp | 2 ++ test/plugins/source.cpp | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/plugins/module.cpp b/test/plugins/module.cpp index 2ed2a27b..61969554 100644 --- a/test/plugins/module.cpp +++ b/test/plugins/module.cpp @@ -9,6 +9,8 @@ using namespace phlex::experimental; PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { + m.provide("provide_i", [](data_cell_index const& id)->int{ return id.number(); }).output_product("i"_in("event")); + m.provide("provide_j", [](data_cell_index const& id)->int{ return -id.number(); }).output_product("j"_in("event")); m.transform("add", test::add, concurrency::unlimited) .input_family("i"_in("event"), "j"_in("event")) .output_products("sum"); diff --git a/test/plugins/source.cpp b/test/plugins/source.cpp index d0749725..45576613 100644 --- a/test/plugins/source.cpp +++ b/test/plugins/source.cpp @@ -18,8 +18,6 @@ namespace { for (int i : std::views::iota(1, n_ + 1)) { auto store = job_store->make_child(i, "event"); - store->add_product("i", i); - store->add_product("j", -i); driver.yield(store); } } From 9d8ce91691979c07f3f3e93daaca0744ed994139 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 13:57:32 -0600 Subject: [PATCH 21/49] added provider to the many_events test --- test/memory-checks/many_events.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/memory-checks/many_events.cpp b/test/memory-checks/many_events.cpp index 7a8703f8..4a1638e2 100644 --- a/test/memory-checks/many_events.cpp +++ b/test/memory-checks/many_events.cpp @@ -19,12 +19,12 @@ int main() for (unsigned int i : std::views::iota(1u, max_events + 1)) { auto event_store = job_store->make_child(i, "event", "Source"); - event_store->add_product("number", i); driver.yield(event_store); } }; framework_graph g{cells_to_process}; + g.provide("provide_number", [](data_cell_index const& id)->unsigned { return id.number(); }).output_product("number"_in("event")); g.transform("pass_on", pass_on, concurrency::unlimited) .input_family("number"_in("event")) .output_products("different"); From 53ff93a9e9989b31adaf05d724c88881a785470b Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 14:05:04 -0600 Subject: [PATCH 22/49] added provider to the mock-workflow test --- test/mock-workflow/CMakeLists.txt | 3 +++ test/mock-workflow/mock-workflow.jsonnet | 6 +++++- test/mock-workflow/source.cpp | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/mock-workflow/CMakeLists.txt b/test/mock-workflow/CMakeLists.txt index 23ccd850..1a19fdbc 100644 --- a/test/mock-workflow/CMakeLists.txt +++ b/test/mock-workflow/CMakeLists.txt @@ -10,6 +10,9 @@ target_link_libraries( add_library(mock_workflow_source MODULE source.cpp) target_link_libraries(mock_workflow_source PRIVATE timed_busy phlex::module) +add_library(id_provider MODULE id_provider.cpp) +target_link_libraries(id_provider PRIVATE phlex::module) + function(add_module NAME) add_library(${NAME} MODULE ${NAME}.cpp) target_link_libraries(${NAME} PRIVATE algorithm) diff --git a/test/mock-workflow/mock-workflow.jsonnet b/test/mock-workflow/mock-workflow.jsonnet index 3d68bc2e..b9d493ab 100644 --- a/test/mock-workflow/mock-workflow.jsonnet +++ b/test/mock-workflow/mock-workflow.jsonnet @@ -7,5 +7,9 @@ local g4stage2 = import 'G4Stage2.libsonnet'; plugin: 'mock_workflow_source', n_events: 1, }, - modules: singlesgen + g4stage1 + g4stage2, + modules: singlesgen + g4stage1 + g4stage2 { + provider: { + plugin: 'id_provider' + } + }, } diff --git a/test/mock-workflow/source.cpp b/test/mock-workflow/source.cpp index bda9d38b..12e366dd 100644 --- a/test/mock-workflow/source.cpp +++ b/test/mock-workflow/source.cpp @@ -26,7 +26,6 @@ namespace phlex::experimental::test { } auto store = job_store->make_child(i, "event"); - store->add_product("id", *store->id()); driver.yield(store); } } From a5a962cef0d86c34b4d0e24203645ae120df60d8 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 14:06:43 -0600 Subject: [PATCH 23/49] adding the provider --- test/mock-workflow/id_provider.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/mock-workflow/id_provider.cpp diff --git a/test/mock-workflow/id_provider.cpp b/test/mock-workflow/id_provider.cpp new file mode 100644 index 00000000..295e01a4 --- /dev/null +++ b/test/mock-workflow/id_provider.cpp @@ -0,0 +1,9 @@ +#include "phlex/module.hpp" + +PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { + using namespace phlex::experimental; + m.provide("provide_id", [](data_cell_index const& id) { + return id; + }) + .output_product("id"_in("event")); +} From 146eafc2945ca653f766a720760cfb4900da8803 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Mon, 8 Dec 2025 14:57:59 -0600 Subject: [PATCH 24/49] changing data_store to data_cell_index in graph --- phlex/app/load_module.cpp | 2 +- phlex/app/load_module.hpp | 2 +- phlex/core/framework_graph.cpp | 12 ++++++------ phlex/core/framework_graph.hpp | 4 ++-- phlex/core/fwd.hpp | 2 +- phlex/model/product_store.hpp | 11 ++++++----- phlex/source.hpp | 4 ++-- test/allowed_families.cpp | 18 +++++++++--------- test/benchmarks/benchmarks_source.cpp | 8 ++++---- test/cached_execution_source.hpp | 17 ++++++++--------- test/max-parallelism/check_parallelism.cpp | 4 ++-- test/memory-checks/many_events.cpp | 8 ++++---- test/mock-workflow/source.cpp | 8 ++++---- test/plugins/source.cpp | 8 ++++---- test/vector_of_abstract_types.cpp | 6 +++--- 15 files changed, 57 insertions(+), 57 deletions(-) diff --git a/phlex/app/load_module.cpp b/phlex/app/load_module.cpp index ffdbcfd8..42897bf9 100644 --- a/phlex/app/load_module.cpp +++ b/phlex/app/load_module.cpp @@ -57,7 +57,7 @@ namespace phlex::experimental { creator(module_proxy, config); } - detail::next_store_t load_source(boost::json::object const& raw_config) + detail::next_index_t load_source(boost::json::object const& raw_config) { configuration const config{raw_config}; auto const& spec = config.get("plugin"); diff --git a/phlex/app/load_module.hpp b/phlex/app/load_module.hpp index 89e818e9..93c9de83 100644 --- a/phlex/app/load_module.hpp +++ b/phlex/app/load_module.hpp @@ -10,7 +10,7 @@ namespace phlex::experimental { void load_module(framework_graph& g, std::string const& label, boost::json::object config); - detail::next_store_t load_source(boost::json::object const& config); + detail::next_index_t load_source(boost::json::object const& config); } #endif // PHLEX_APP_LOAD_MODULE_HPP diff --git a/phlex/core/framework_graph.cpp b/phlex/core/framework_graph.cpp index 6308fc82..d9fdaab5 100644 --- a/phlex/core/framework_graph.cpp +++ b/phlex/core/framework_graph.cpp @@ -34,15 +34,15 @@ namespace phlex::experimental { std::size_t layer_sentry::depth() const noexcept { return depth_; } - framework_graph::framework_graph(product_store_ptr store, int const max_parallelism) : - framework_graph{[store](framework_driver& driver) { driver.yield(store); }, max_parallelism} + framework_graph::framework_graph(data_cell_index_ptr index, int const max_parallelism) : + framework_graph{[index](framework_driver& driver) { driver.yield(index); }, max_parallelism} { } // FIXME: The algorithm below should support user-specified flush stores. - framework_graph::framework_graph(detail::next_store_t next_store, int const max_parallelism) : + framework_graph::framework_graph(detail::next_index_t next_index, int const max_parallelism) : parallelism_limit_{static_cast(max_parallelism)}, - driver_{std::move(next_store)}, + driver_{std::move(next_index)}, src_{graph_, [this](tbb::flow_control& fc) mutable -> message { auto item = driver_(); @@ -51,8 +51,8 @@ namespace phlex::experimental { fc.stop(); return {}; } - auto store = *item; - assert(not store->is_flush()); + auto index = *item; + auto store = std::make_shared(nullptr, index, "Source"); return sender_.make_message(accept(std::move(store))); }}, multiplexer_{graph_} diff --git a/phlex/core/framework_graph.hpp b/phlex/core/framework_graph.hpp index 9b2cdae5..f010bbf9 100644 --- a/phlex/core/framework_graph.hpp +++ b/phlex/core/framework_graph.hpp @@ -47,9 +47,9 @@ namespace phlex::experimental { class framework_graph { public: - explicit framework_graph(product_store_ptr store, + explicit framework_graph(data_cell_index_ptr index, int max_parallelism = oneapi::tbb::info::default_concurrency()); - explicit framework_graph(detail::next_store_t f, + explicit framework_graph(detail::next_index_t f, int max_parallelism = oneapi::tbb::info::default_concurrency()); ~framework_graph(); diff --git a/phlex/core/fwd.hpp b/phlex/core/fwd.hpp index 59448b57..45140754 100644 --- a/phlex/core/fwd.hpp +++ b/phlex/core/fwd.hpp @@ -18,7 +18,7 @@ namespace phlex::experimental { class products_consumer; using end_of_message_ptr = std::shared_ptr; - using framework_driver = async_driver; + using framework_driver = async_driver; } #endif // PHLEX_CORE_FWD_HPP diff --git a/phlex/model/product_store.hpp b/phlex/model/product_store.hpp index 12e404cf..beec4a2c 100644 --- a/phlex/model/product_store.hpp +++ b/phlex/model/product_store.hpp @@ -16,6 +16,12 @@ namespace phlex::experimental { class product_store : public std::enable_shared_from_this { public: + + explicit product_store(product_store_const_ptr parent, + data_cell_index_ptr id, + std::string source, + stage processing_stage = stage::process, + products new_products = {}); ~product_store(); static product_store_ptr base(std::string base_name = "Source"); @@ -60,11 +66,6 @@ namespace phlex::experimental { void add_product(std::string const& key, std::unique_ptr>&& t); private: - explicit product_store(product_store_const_ptr parent, - data_cell_index_ptr id, - std::string source, - stage processing_stage = stage::process, - products new_products = {}); explicit product_store(product_store_const_ptr parent, std::size_t data_cell_number, std::string const& child_layer_name, diff --git a/phlex/source.hpp b/phlex/source.hpp index 3182eab6..133b8d84 100644 --- a/phlex/source.hpp +++ b/phlex/source.hpp @@ -59,8 +59,8 @@ namespace phlex::experimental::detail { } } - using next_store_t = std::function; - using source_creator_t = next_store_t(configuration const&); + using next_index_t = std::function; + using source_creator_t = next_index_t(configuration const&); } #define PHLEX_EXPERIMENTAL_REGISTER_SOURCE(source) \ diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index ff373f75..f3e3f83e 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -13,17 +13,17 @@ namespace { void cells_to_process(framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); - auto run_store = job_store->make_child(0, "run"); - driver.yield(run_store); + auto run_index = job_index->make_child(0, "run"); + driver.yield(run_index); - auto subrun_store = run_store->make_child(0, "subrun"); - driver.yield(subrun_store); - - auto event_store = subrun_store->make_child(0, "event"); - driver.yield(event_store); + auto subrun_index = run_index->make_child(0, "subrun"); + driver.yield(subrun_index); + + auto event_index = subrun_index->make_child(0, "event"); + driver.yield(event_index); } // Provider functions that return data_cell_index for each level diff --git a/test/benchmarks/benchmarks_source.cpp b/test/benchmarks/benchmarks_source.cpp index 582249b6..8a5fa1c5 100644 --- a/test/benchmarks/benchmarks_source.cpp +++ b/test/benchmarks/benchmarks_source.cpp @@ -20,16 +20,16 @@ namespace test { void next(phlex::experimental::framework_driver& driver) const { - auto job_store = phlex::experimental::product_store::base(); - driver.yield(job_store); + auto job_index = phlex::experimental::data_cell_index::base_ptr(); + driver.yield(job_index); for (std::size_t i : std::views::iota(0u, max_)) { if (max_ > 10 and i % (max_ / 10) == 0) { spdlog::debug("Reached {} events", i); } - auto store = job_store->make_child(i, "event"); - driver.yield(store); + auto index = job_index->make_child(i, "event"); + driver.yield(index); } } diff --git a/test/cached_execution_source.hpp b/test/cached_execution_source.hpp index 14968e80..d93bb2ff 100644 --- a/test/cached_execution_source.hpp +++ b/test/cached_execution_source.hpp @@ -24,20 +24,19 @@ namespace test { { using namespace phlex::experimental; - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (std::size_t i : std::views::iota(0u, n_runs)) { - auto run_store = job_store->make_child(i, "run"); - driver.yield(run_store); + auto run_index = job_index->make_child(i, "run"); + driver.yield(run_index); for (std::size_t j : std::views::iota(0u, n_subruns)) { - auto subrun_store = run_store->make_child(j, "subrun"); - driver.yield(subrun_store); - + auto subrun_index = run_index->make_child(j, "subrun"); + driver.yield(subrun_index); for (std::size_t k : std::views::iota(0u, n_events)) { - auto event_store = subrun_store->make_child(k, "event"); - driver.yield(event_store); + auto event_index = subrun_index->make_child(k, "event"); + driver.yield(event_index); } } } diff --git a/test/max-parallelism/check_parallelism.cpp b/test/max-parallelism/check_parallelism.cpp index 6fca657a..739212a3 100644 --- a/test/max-parallelism/check_parallelism.cpp +++ b/test/max-parallelism/check_parallelism.cpp @@ -21,8 +21,8 @@ namespace { public: void next(framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); } }; } diff --git a/test/memory-checks/many_events.cpp b/test/memory-checks/many_events.cpp index 4a1638e2..2bc5c6fa 100644 --- a/test/memory-checks/many_events.cpp +++ b/test/memory-checks/many_events.cpp @@ -14,12 +14,12 @@ int main() // spdlog::flush_on(spdlog::level::trace); auto cells_to_process = [](framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned int i : std::views::iota(1u, max_events + 1)) { - auto event_store = job_store->make_child(i, "event", "Source"); - driver.yield(event_store); + auto event_index = job_index->make_child(i, "event"); + driver.yield(event_index); } }; diff --git a/test/mock-workflow/source.cpp b/test/mock-workflow/source.cpp index 12e366dd..b9d45523 100644 --- a/test/mock-workflow/source.cpp +++ b/test/mock-workflow/source.cpp @@ -17,16 +17,16 @@ namespace phlex::experimental::test { void next(framework_driver& driver) const { - auto job_store = phlex::experimental::product_store::base(); - driver.yield(job_store); + auto job_index = phlex::experimental::data_cell_index::base_ptr(); + driver.yield(job_index); for (std::size_t i : std::views::iota(0u, max_)) { if (max_ > 10 && (i % (max_ / 10) == 0)) { spdlog::debug("Reached {} events", i); } - auto store = job_store->make_child(i, "event"); - driver.yield(store); + auto index = job_index->make_child(i, "event"); + driver.yield(index); } } diff --git a/test/plugins/source.cpp b/test/plugins/source.cpp index 45576613..1a26e9b8 100644 --- a/test/plugins/source.cpp +++ b/test/plugins/source.cpp @@ -13,12 +13,12 @@ namespace { void next(phlex::experimental::framework_driver& driver) const { - auto job_store = phlex::experimental::product_store::base(); - driver.yield(job_store); + auto job_index = phlex::experimental::data_cell_index::base_ptr(); + driver.yield(job_index); for (int i : std::views::iota(1, n_ + 1)) { - auto store = job_store->make_child(i, "event"); - driver.yield(store); + auto index = job_index->make_child(i, "event"); + driver.yield(index); } } diff --git a/test/vector_of_abstract_types.cpp b/test/vector_of_abstract_types.cpp index f692a0c5..c4300280 100644 --- a/test/vector_of_abstract_types.cpp +++ b/test/vector_of_abstract_types.cpp @@ -41,11 +41,11 @@ namespace { void operator()(framework_driver& driver) { - auto job_store = phlex::experimental::product_store::base(); - driver.yield(job_store); + auto job_index = phlex::experimental::data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned int i : std::views::iota(1u, max_ + 1)) { - driver.yield(job_store->make_child(i, "event")); + driver.yield(job_index->make_child(i, "event")); } } From 15c324e43e992c88675dbe896314c5f59a8c8377 Mon Sep 17 00:00:00 2001 From: Marc Paterno Date: Mon, 8 Dec 2025 17:48:03 -0600 Subject: [PATCH 25/49] More work toward function providers --- test/class_registration.cpp | 2 +- test/demo-giantdata/log_record.hpp | 91 +------------------ test/demo-giantdata/unfold_transform_fold.cpp | 69 ++++---------- test/demo-giantdata/user_algorithms.cpp | 19 +--- test/demo-giantdata/user_algorithms.hpp | 14 +-- test/demo-giantdata/waveform_generator.cpp | 9 +- test/demo-giantdata/waveform_generator.hpp | 2 - .../waveform_generator_input.cpp | 7 -- test/demo-giantdata/waveforms.cpp | 7 -- test/different_hierarchies.cpp | 16 ++-- test/filter.cpp | 8 +- test/fold.cpp | 12 +-- test/function_registration.cpp | 2 +- test/hierarchical_nodes.cpp | 10 +- test/multiple_function_registration.cpp | 2 +- test/type_distinction.cpp | 8 +- test/unfold.cpp | 8 +- 17 files changed, 60 insertions(+), 226 deletions(-) diff --git a/test/class_registration.cpp b/test/class_registration.cpp index 366ab24e..a97900e8 100644 --- a/test/class_registration.cpp +++ b/test/class_registration.cpp @@ -62,7 +62,7 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const product_names{"number"_in("job"), "temperature"_in("job"), "name"_in("job")}; std::array const oproduct_names{"onumber"s, "otemperature"s, "oname"s}; - framework_graph g{product_store::base()}; + framework_graph g{data_cell_index::base_ptr()}; // Register providers for the input products g.provide("provide_number", provide_number, concurrency::unlimited) diff --git a/test/demo-giantdata/log_record.hpp b/test/demo-giantdata/log_record.hpp index b5b4c9a3..b48b814b 100644 --- a/test/demo-giantdata/log_record.hpp +++ b/test/demo-giantdata/log_record.hpp @@ -1,90 +1 @@ -#ifndef TEST_DEMO_GIANTDATA_LOG_RECORD_HPP -#define TEST_DEMO_GIANTDATA_LOG_RECORD_HPP - -#include "oneapi/tbb/concurrent_queue.h" -#include "oneapi/tbb/task_arena.h" -#include -#include -#include -#include - -static_assert(static_cast(-1ULL) == -1, "static_cast(-1ULL) is not -1"); - -namespace demo { - inline int constexpr convert_to_int(std::size_t value) { return static_cast(value); } - - std::size_t const EVENT_NAME_SIZE = 16; - struct record { - record(char const* event, - std::size_t run_id, - std::size_t subrun_id, - std::size_t spill_id, - std::size_t apa_id, - void const* active, - std::size_t data, - void const* orig) : - run_id(convert_to_int(run_id)), - subrun_id(convert_to_int(subrun_id)), - spill_id(convert_to_int(spill_id)), - apa_id(convert_to_int(apa_id)), - active(active), - data(data), - orig(orig), - timestamp(), - thread_index(tbb::this_task_arena::current_thread_index()) - { - auto const t = std::chrono::system_clock::now().time_since_epoch(); - double const ts = - static_cast(std::chrono::duration_cast(t).count()); - timestamp = ts * 1.0e-6; - std::strncpy(this->event, event, EVENT_NAME_SIZE); - this->event[EVENT_NAME_SIZE - 1] = '\0'; // ensure null-termination - } - - int run_id; - int subrun_id; - int spill_id; - int apa_id; - void const* active; - int data; - void const* orig; - double timestamp; - char event[EVENT_NAME_SIZE]; - int thread_index; - }; - - inline std::ostream& operator<<(std::ostream& os, record const& r) - { - os << std::scientific << std::setprecision(std::numeric_limits::max_digits10) - << r.timestamp << '\t' << r.thread_index << '\t' << r.event << '\t' << r.run_id << '\t' - << r.subrun_id << '\t' << r.spill_id << '\t' << r.apa_id << '\t' << r.active << '\t' - << r.data << '\t' << r.orig; - return os; - } - - // Define a global queue for logging records - inline oneapi::tbb::concurrent_queue global_queue; - - inline void log_record(char const* event, - std::size_t run_id = -1ULL, - std::size_t subrun_id = -1ULL, - std::size_t spill_id = -1ULL, - std::size_t apa_id = -1ULL, - void const* active = nullptr, - std::size_t data = 0, - void const* orig = nullptr) - { - record r(event, run_id, subrun_id, spill_id, apa_id, active, data, orig); - global_queue.push(r); - } - inline void write_log(std::string const& filename) - { - std::ofstream log_file(filename, std::ios_base::out | std::ios_base::trunc); - log_file << "time\tthread\tevent\trun\tsubrun\tspill\tapa\tactive\tdata\torig\n"; - for (auto it = global_queue.unsafe_cbegin(), e = global_queue.unsafe_cend(); it != e; ++it) { - log_file << *it << '\n'; - } - } -} // namespace demo - -#endif // TEST_DEMO_GIANTDATA_LOG_RECORD_HPP +#error DO NOT INCLUDE THIS FILE \ No newline at end of file diff --git a/test/demo-giantdata/unfold_transform_fold.cpp b/test/demo-giantdata/unfold_transform_fold.cpp index 1f14f797..0262a75d 100644 --- a/test/demo-giantdata/unfold_transform_fold.cpp +++ b/test/demo-giantdata/unfold_transform_fold.cpp @@ -4,7 +4,6 @@ #include "phlex/utilities/async_driver.hpp" #include "test/products_for_output.hpp" -#include "test/demo-giantdata/log_record.hpp" #include "test/demo-giantdata/user_algorithms.hpp" #include "test/demo-giantdata/waveform_generator.hpp" #include "test/demo-giantdata/waveform_generator_input.hpp" @@ -14,7 +13,6 @@ #include #include -using framework_driver = phlex::experimental::async_driver; using namespace phlex::experimental; // Call the program as follows: @@ -57,42 +55,23 @@ int main(int argc, char* argv[]) // We may or may not want to create pre-generated data layers. // Each data layer gets an index number in the hierarchy. - auto source = [n_runs, n_subruns, n_spills, wires_per_spill](framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto source = [n_runs, n_subruns, n_spills](framework_driver& driver) { + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); // job -> run -> subrun -> spill data layers for (unsigned runno : std::views::iota(0u, n_runs)) { - auto run_store = job_store->make_child(runno, "run"); - driver.yield(run_store); + auto run_index = job_index->make_child(runno, "run"); + driver.yield(run_index); for (unsigned subrunno : std::views::iota(0u, n_subruns)) { - auto subrun_store = run_store->make_child(subrunno, "subrun"); - driver.yield(subrun_store); + auto subrun_index = run_index->make_child(subrunno, "subrun"); + driver.yield(subrun_index); for (unsigned spillno : std::views::iota(0u, n_spills)) { - auto spill_store = subrun_store->make_child(spillno, "spill"); - - // Put the WGI product into the job, so that our CHOF can find it. - auto next_size = wires_per_spill; - demo::log_record("add_wgi", - run_store->id()->number(), - subrun_store->id()->number(), - spill_store->id()->number(), - -1, - &spill_store, - next_size, - nullptr); - // NOTE: the only reason that we are able to put the spill id into the WGI object - // is because we have access to the store - spill_store->add_product("wgen", - demo::WGI(next_size, - run_store->id()->number(), - subrun_store->id()->number(), - spill_store->id()->number())); - - driver.yield(spill_store); + auto spill_index = subrun_index->make_child(spillno, "spill"); + driver.yield(spill_index); } } } @@ -101,15 +80,22 @@ int main(int argc, char* argv[]) // Create the graph. The source tells us what data we will process. // We introduce a new scope to make sure the graph is destroyed before we // write out the logged records. - demo::log_record("create_graph"); { framework_graph g{source}; // Add the unfold node to the graph. We do not yet know how to provide the chunksize // to the constructor of the WaveformGenerator, so we will use the default value. - demo::log_record("add_unfold"); auto const chunksize = 256LL; // this could be read from a configuration file + g.provide("provide_wgen", [wires_per_spill](data_cell_index const& spill_index) { + return + demo::WGI(wires_per_spill, + spill_index.parent()->parent()->number(), // ugh + spill_index.parent()->number(), + spill_index.number()); + }) + .output_product("wget"_in("spill")); + g.unfold( "WaveformGenerator", &demo::WaveformGenerator::predicate, @@ -122,13 +108,8 @@ int main(int argc, char* argv[]) .output_products("waves_in_apa"); // Add the transform node to the graph. - demo::log_record("add_transform"); auto wrapped_user_function = [](phlex::experimental::handle hwf) { - auto apa_id = hwf.data_cell_index().number(); - auto spill_id = hwf.data_cell_index().parent()->number(); - auto subrun_id = hwf.data_cell_index().parent()->parent()->number(); - auto run_id = hwf.data_cell_index().parent()->parent()->parent()->number(); - return demo::clampWaveforms(*hwf, run_id, subrun_id, spill_id, apa_id); + return demo::clampWaveforms(*hwf); }; g.transform("clamp_node", wrapped_user_function, concurrency::unlimited) @@ -136,15 +117,10 @@ int main(int argc, char* argv[]) .output_products("clamped_waves"); // Add the fold node to the graph. - demo::log_record("add_fold"); g.fold( "accum_for_spill", [](demo::SummedClampedWaveforms& scw, phlex::experimental::handle hwf) { - auto apa_id = hwf.data_cell_index().number(); - auto spill_id = hwf.data_cell_index().parent()->number(); - auto subrun_id = hwf.data_cell_index().parent()->parent()->number(); - auto run_id = hwf.data_cell_index().parent()->parent()->parent()->number(); - demo::accumulateSCW(scw, *hwf, run_id, subrun_id, spill_id, apa_id); + demo::accumulateSCW(scw, *hwf); }, concurrency::unlimited, "spill" // partition the output by the spill @@ -152,15 +128,10 @@ int main(int argc, char* argv[]) .input_family("clamped_waves"_in("APA")) .output_products("summed_waveforms"); - demo::log_record("add_output"); g.make().output( "save", &test::products_for_output::save, concurrency::serial); // Execute the graph. - demo::log_record("execute_graph"); g.execute(); - demo::log_record("end_graph"); } - demo::log_record("graph_destroyed"); - demo::write_log("unfold_transform_fold.tsv"); } diff --git a/test/demo-giantdata/user_algorithms.cpp b/test/demo-giantdata/user_algorithms.cpp index 231099b3..15bd1c62 100644 --- a/test/demo-giantdata/user_algorithms.cpp +++ b/test/demo-giantdata/user_algorithms.cpp @@ -1,44 +1,29 @@ #include "user_algorithms.hpp" -#include "log_record.hpp" #include "summed_clamped_waveforms.hpp" #include "waveforms.hpp" // This function is used to transform an input Waveforms object into an // output Waveforms object. The output is a clamped version of the input. -demo::Waveforms demo::clampWaveforms(demo::Waveforms const& input, - std::size_t run_id, - std::size_t subrun_id, - std::size_t spill_id, - std::size_t apa_id) +demo::Waveforms demo::clampWaveforms(demo::Waveforms const& input) { - demo::log_record( - "start_clamp", run_id, subrun_id, spill_id, apa_id, &input, input.size(), nullptr); demo::Waveforms result(input); for (demo::Waveform& wf : result.waveforms) { for (double& x : wf.samples) { x = std::clamp(x, -10.0, 10.0); } } - demo::log_record("end_clamp", run_id, subrun_id, spill_id, apa_id, &input, input.size(), &result); return result; } // This is the fold operator that will accumulate a SummedClampedWaveforms object. void demo::accumulateSCW(demo::SummedClampedWaveforms& accumulator, - demo::Waveforms const& wf, - std::size_t run_id, - std::size_t subrun_id, - std::size_t spill_id, - std::size_t apa_id) + demo::Waveforms const& wf) { // This is the fold operator that will accumulate a SummedClampedWaveforms object. - demo::log_record( - "start_accSCW", run_id, subrun_id, spill_id, apa_id, &accumulator, wf.size(), &wf); accumulator.size += wf.size(); for (auto const& w : wf.waveforms) { for (double x : w.samples) { accumulator.sum += x; } } - demo::log_record("end_accSCW", run_id, subrun_id, spill_id, apa_id, &accumulator, wf.size(), &wf); } diff --git a/test/demo-giantdata/user_algorithms.hpp b/test/demo-giantdata/user_algorithms.hpp index 6c7e6759..9aec2a2e 100644 --- a/test/demo-giantdata/user_algorithms.hpp +++ b/test/demo-giantdata/user_algorithms.hpp @@ -8,20 +8,10 @@ namespace demo { // This function is used to transform an input Waveforms object into an // output Waveforms object. The output is a clamped version of the input. - Waveforms clampWaveforms(Waveforms const& input, - std::size_t run_id, - std::size_t subrun_id, - std::size_t spill_id, - std::size_t apa_id); + Waveforms clampWaveforms(Waveforms const& input); // This is the fold operator that will accumulate a SummedClampedWaveforms object. - void accumulateSCW(SummedClampedWaveforms& scw, - Waveforms const& wf, - std::size_t run_id, - std::size_t subrun_id, - std::size_t spill_id, - std::size_t apa_id); - + void accumulateSCW(SummedClampedWaveforms& scw, Waveforms const& wf); } // namespace demo #endif // TEST_DEMO_GIANTDATA_USER_ALGORITHMS_HPP diff --git a/test/demo-giantdata/waveform_generator.cpp b/test/demo-giantdata/waveform_generator.cpp index c5e8b638..53e6d1bc 100644 --- a/test/demo-giantdata/waveform_generator.cpp +++ b/test/demo-giantdata/waveform_generator.cpp @@ -1,29 +1,24 @@ #include "waveforms.hpp" -#include "log_record.hpp" #include "waveform_generator.hpp" #include "waveform_generator_input.hpp" #include demo::WaveformGenerator::WaveformGenerator(WGI const& wgi) : - maxsize_{wgi.size}, spill_id_(wgi.spill_id) + maxsize_{wgi.size} { - log_record("wgctor", -1, -1, spill_id_, -1, this, sizeof(*this), nullptr); } demo::WaveformGenerator::~WaveformGenerator() { - log_record("wgdtor", -1, -1, spill_id_, -1, this, sizeof(*this), nullptr); } std::size_t demo::WaveformGenerator::initial_value() const { return 0; } bool demo::WaveformGenerator::predicate(std::size_t made_so_far) const { - log_record("start_pred", -1, -1, spill_id_, -1, this, made_so_far, nullptr); bool const result = made_so_far < maxsize_; - log_record("end_pred", -1, -1, spill_id_, -1, this, made_so_far, nullptr); return result; } @@ -31,10 +26,8 @@ std::pair demo::WaveformGenerator::op(std::size_t std::size_t chunksize) const { // How many waveforms should go into this chunk? - log_record("start_op", -1, -1, spill_id_, -1, this, chunksize, nullptr); std::size_t const newsize = std::min(chunksize, maxsize_ - made_so_far); auto result = std::make_pair(made_so_far + newsize, Waveforms{newsize, 1.0 * made_so_far, -1, -1, -1, -1}); - log_record("end_op", -1, -1, spill_id_, -1, this, newsize, nullptr); return result; } \ No newline at end of file diff --git a/test/demo-giantdata/waveform_generator.hpp b/test/demo-giantdata/waveform_generator.hpp index 62d1485c..07d672d7 100644 --- a/test/demo-giantdata/waveform_generator.hpp +++ b/test/demo-giantdata/waveform_generator.hpp @@ -1,7 +1,6 @@ #ifndef TEST_DEMO_GIANTDATA_WAVEFORM_GENERATOR_HPP #define TEST_DEMO_GIANTDATA_WAVEFORM_GENERATOR_HPP -#include "log_record.hpp" #include "waveform_generator_input.hpp" #include "waveforms.hpp" @@ -37,7 +36,6 @@ namespace demo { private: std::size_t maxsize_; // total number of waveforms to make for the unfold - int spill_id_; // the id of the spill this object will process }; // class WaveformGenerator } // namespace demo #endif // TEST_DEMO_GIANTDATA_WAVEFORM_GENERATOR_HPP diff --git a/test/demo-giantdata/waveform_generator_input.cpp b/test/demo-giantdata/waveform_generator_input.cpp index de649083..990a8d47 100644 --- a/test/demo-giantdata/waveform_generator_input.cpp +++ b/test/demo-giantdata/waveform_generator_input.cpp @@ -1,5 +1,4 @@ #include "waveform_generator_input.hpp" -#include "log_record.hpp" demo::WaveformGeneratorInput::WaveformGeneratorInput(std::size_t size, std::size_t run_id, @@ -7,25 +6,21 @@ demo::WaveformGeneratorInput::WaveformGeneratorInput(std::size_t size, std::size_t spill_id) : size(size), run_id(run_id), subrun_id(subrun_id), spill_id(spill_id) { - log_record("wgictor", run_id, subrun_id, spill_id, -1, this, mysize(*this), nullptr); } demo::WaveformGeneratorInput::WaveformGeneratorInput(WaveformGeneratorInput const& other) : size(other.size), run_id(other.run_id), subrun_id(other.subrun_id), spill_id(other.spill_id) { - log_record("wgicopy", run_id, subrun_id, spill_id, -1, this, mysize(*this), &other); } demo::WaveformGeneratorInput::WaveformGeneratorInput(WaveformGeneratorInput&& other) : size(other.size), run_id(other.run_id), subrun_id(other.subrun_id), spill_id(other.spill_id) { - log_record("wgimove", run_id, subrun_id, spill_id, -1, this, mysize(*this), &other); } demo::WaveformGeneratorInput& demo::WaveformGeneratorInput::operator=( WaveformGeneratorInput const& other) { - log_record("wgicopy=", run_id, subrun_id, spill_id, -1, this, 0, &other); size = other.size; spill_id = other.spill_id; return *this; @@ -34,7 +29,6 @@ demo::WaveformGeneratorInput& demo::WaveformGeneratorInput::operator=( demo::WaveformGeneratorInput& demo::WaveformGeneratorInput::operator=( WaveformGeneratorInput&& other) { - log_record("wgimove=", run_id, subrun_id, spill_id, -1, this, 0, &other); size = other.size; spill_id = other.spill_id; return *this; @@ -42,5 +36,4 @@ demo::WaveformGeneratorInput& demo::WaveformGeneratorInput::operator=( demo::WaveformGeneratorInput::~WaveformGeneratorInput() { - log_record("wgidtor", run_id, subrun_id, spill_id, -1, this, mysize(*this), nullptr); } diff --git a/test/demo-giantdata/waveforms.cpp b/test/demo-giantdata/waveforms.cpp index bd538af8..62c8cc05 100644 --- a/test/demo-giantdata/waveforms.cpp +++ b/test/demo-giantdata/waveforms.cpp @@ -1,5 +1,4 @@ #include "waveforms.hpp" -#include "log_record.hpp" std::size_t demo::Waveforms::size() const { return waveforms.size(); } @@ -7,19 +6,16 @@ demo::Waveforms::Waveforms( std::size_t n, double val, int run_id, int subrun_id, int spill_id, int apa_id) : waveforms(n, {val}), run_id(run_id), subrun_id(subrun_id), spill_id(spill_id), apa_id(apa_id) { - log_record("wsctor", 0, 0, spill_id, apa_id, this, n, nullptr); } demo::Waveforms::Waveforms(Waveforms const& other) : waveforms(other.waveforms), spill_id(other.spill_id), apa_id(other.apa_id) { - log_record("wscopy", 0, 0, spill_id, apa_id, this, waveforms.size(), &other); } demo::Waveforms::Waveforms(Waveforms&& other) : waveforms(std::move(other.waveforms)), spill_id(other.spill_id), apa_id(other.apa_id) { - log_record("wsmove", 0, 0, spill_id, apa_id, this, waveforms.size(), &other); } demo::Waveforms& demo::Waveforms::operator=(Waveforms const& other) @@ -27,7 +23,6 @@ demo::Waveforms& demo::Waveforms::operator=(Waveforms const& other) waveforms = other.waveforms; spill_id = other.spill_id; apa_id = other.apa_id; - log_record("wscopy=", 0, 0, spill_id, apa_id, this, waveforms.size(), &other); return *this; } @@ -36,11 +31,9 @@ demo::Waveforms& demo::Waveforms::operator=(Waveforms&& other) waveforms = std::move(other.waveforms); spill_id = other.spill_id; apa_id = other.apa_id; - log_record("wsmove=", 0, 0, spill_id, apa_id, this, waveforms.size(), &other); return *this; } demo::Waveforms::~Waveforms() { - log_record("wsdtor", 0, 0, spill_id, apa_id, this, waveforms.size(), nullptr); }; diff --git a/test/different_hierarchies.cpp b/test/different_hierarchies.cpp index 32b203d6..529310ad 100644 --- a/test/different_hierarchies.cpp +++ b/test/different_hierarchies.cpp @@ -60,23 +60,23 @@ namespace { void cells_to_process(framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); // job -> run -> event layers for (unsigned i : std::views::iota(0u, index_limit)) { - auto run_store = job_store->make_child(i, "run"); - driver.yield(run_store); + auto run_index = job_index->make_child(i, "run"); + driver.yield(run_index); for (unsigned j : std::views::iota(0u, number_limit)) { - auto event_store = run_store->make_child(j, "event"); - driver.yield(event_store); + auto event_index = run_index->make_child(j, "event"); + driver.yield(event_index); } } // job -> event layers for (unsigned i : std::views::iota(0u, top_level_event_limit)) { - auto tp_store = job_store->make_child(i, "event"); - driver.yield(tp_store); + auto tp_event_index = job_index->make_child(i, "event"); + driver.yield(tp_event_index); } } } diff --git a/test/filter.cpp b/test/filter.cpp index 533a356e..d842fbd2 100644 --- a/test/filter.cpp +++ b/test/filter.cpp @@ -15,12 +15,12 @@ namespace { void operator()(framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned int i : std::views::iota(1u, max_ + 1)) { - auto store = job_store->make_child(i, "event", "Source"); - driver.yield(store); + auto index = job_index->make_child(i, "event"); + driver.yield(index); } } diff --git a/test/fold.cpp b/test/fold.cpp index 7726b920..1f73935f 100644 --- a/test/fold.cpp +++ b/test/fold.cpp @@ -51,14 +51,14 @@ TEST_CASE("Different data layers of fold", "[graph]") constexpr auto number_limit = 5u; auto cells_to_process = [index_limit, number_limit](framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned i : std::views::iota(0u, index_limit)) { - auto run_store = job_store->make_child(i, "run", "Source"); - driver.yield(run_store); + auto run_index = job_index->make_child(i, "run"); + driver.yield(run_index); for (unsigned j : std::views::iota(0u, number_limit)) { - auto event_store = run_store->make_child(j, "event", "Source"); - driver.yield(event_store); + auto event_index = run_index->make_child(j, "event"); + driver.yield(event_index); } } }; diff --git a/test/function_registration.cpp b/test/function_registration.cpp index 6ce7585e..ed3898a7 100644 --- a/test/function_registration.cpp +++ b/test/function_registration.cpp @@ -58,7 +58,7 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const oproduct_names = {"onumber"s, "otemperature"s, "oname"s}; std::array const result{"result"s}; - framework_graph g {product_store::base()}; + framework_graph g {data_cell_index::base_ptr()}; // Register providers g.provide("provide_number", provide_number, concurrency::unlimited) diff --git a/test/hierarchical_nodes.cpp b/test/hierarchical_nodes.cpp index 5ea3619e..63102ffe 100644 --- a/test/hierarchical_nodes.cpp +++ b/test/hierarchical_nodes.cpp @@ -40,13 +40,13 @@ namespace { void cells_to_process(framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned i : std::views::iota(0u, index_limit)) { - auto run_store = job_store->make_child(i, "run", "cells_to_process"); - driver.yield(run_store); + auto run_index = job_index->make_child(i, "run"); + driver.yield(run_index); for (unsigned j : std::views::iota(0u, number_limit)) { - driver.yield(run_store->make_child(j, "event", "cells_to_process")); + driver.yield(run_index->make_child(j, "event")); } } } diff --git a/test/multiple_function_registration.cpp b/test/multiple_function_registration.cpp index 8c7ebf90..a43bd793 100644 --- a/test/multiple_function_registration.cpp +++ b/test/multiple_function_registration.cpp @@ -41,7 +41,7 @@ namespace { TEST_CASE("Call multiple functions", "[programming model]") { - framework_graph g{product_store::base()}; + framework_graph g{data_cell_index::base_ptr()}; g.provide("provide_numbers", [](data_cell_index const&) -> std::vector { return {0, 1, 2, 3, 4}; }) diff --git a/test/type_distinction.cpp b/test/type_distinction.cpp index a905b8cf..24da1392 100644 --- a/test/type_distinction.cpp +++ b/test/type_distinction.cpp @@ -45,11 +45,11 @@ TEST_CASE("Distinguish products with same name and different types", "[programmi auto gen = [](auto& driver) { std::vector numbers{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (int i : numbers) { - auto event_store = job_store->make_child(unsigned(i), "event"); - driver.yield(event_store); + auto event_index = job_index->make_child(unsigned(i), "event"); + driver.yield(event_index); } }; diff --git a/test/unfold.cpp b/test/unfold.cpp index f1c6206f..6279aa89 100644 --- a/test/unfold.cpp +++ b/test/unfold.cpp @@ -93,11 +93,11 @@ TEST_CASE("Splitting the processing", "[graph]") constexpr auto index_limit = 2u; auto cells_to_process = [index_limit](auto& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned i : std::views::iota(0u, index_limit)) { - auto event_store = job_store->make_child(i, "event", "Source"); - driver.yield(event_store); + auto event_index = job_index->make_child(i, "event"); + driver.yield(event_index); } }; From f3b0b3e64d0fdf84e0c2f0631d85dc1e68debf12 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 09:47:11 -0600 Subject: [PATCH 26/49] Adjust fold bookkeeping; remove unnecessary product_store functionality --- phlex/core/declared_fold.hpp | 9 ++-- phlex/core/declared_provider.hpp | 3 +- phlex/core/declared_transform.hpp | 4 +- phlex/core/declared_unfold.cpp | 7 +-- phlex/core/framework_graph.cpp | 2 +- phlex/model/product_store.cpp | 87 ++----------------------------- phlex/model/product_store.hpp | 33 ++---------- test/data_cell_counting.cpp | 33 ++++++------ test/product_store.cpp | 12 ++--- 9 files changed, 42 insertions(+), 148 deletions(-) diff --git a/phlex/core/declared_fold.hpp b/phlex/core/declared_fold.hpp index fdabc8e3..04c9dd7a 100644 --- a/phlex/core/declared_fold.hpp +++ b/phlex/core/declared_fold.hpp @@ -97,9 +97,10 @@ namespace phlex::experimental { } } - auto const& fold_store = store->is_flush() ? store : store->parent(partition_); - assert(fold_store); - auto const& id_hash_for_counter = fold_store->id()->hash(); + auto const& fold_index = + store->is_flush() ? store->id() : store->id()->parent(partition_); + assert(fold_index); + auto const& id_hash_for_counter = fold_index->hash(); if (store->is_flush()) { counter_for(id_hash_for_counter).set_flush_value(store, original_message_id); @@ -109,7 +110,7 @@ namespace phlex::experimental { } if (auto counter = done_with(id_hash_for_counter)) { - auto parent = fold_store->make_continuation(this->full_name()); + auto parent = std::make_shared(fold_index, this->full_name()); commit_(*parent); ++product_count_; // FIXME: This msg.eom value may be wrong! diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index 7bdc665b..1c624c37 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -75,7 +75,8 @@ namespace phlex::experimental { products new_products; // Add all adds all products; we should only have one. Fix this later. new_products.add_all(output_, std::move(result)); - auto store = msg.store->make_continuation(this->full_name(), std::move(new_products)); + auto store = std::make_shared( + msg.store->id(), this->full_name(), stage::process, std::move(new_products)); message const new_msg{store, msg.eom, msg.id}; stay_in_graph.try_put(new_msg); diff --git a/phlex/core/declared_transform.hpp b/phlex/core/declared_transform.hpp index 125024cd..7db3ccd3 100644 --- a/phlex/core/declared_transform.hpp +++ b/phlex/core/declared_transform.hpp @@ -104,8 +104,8 @@ namespace phlex::experimental { ++product_count_[store->id()->layer_hash()]; products new_products; new_products.add_all(output_, std::move(result)); - a->second = - store->make_continuation(this->full_name(), std::move(new_products)); + a->second = std::make_shared( + store->id(), this->full_name(), stage::process, std::move(new_products)); message const new_msg{a->second, msg.eom, message_id}; stay_in_graph.try_put(new_msg); diff --git a/phlex/core/declared_unfold.cpp b/phlex/core/declared_unfold.cpp index 38cea243..35d0f949 100644 --- a/phlex/core/declared_unfold.cpp +++ b/phlex/core/declared_unfold.cpp @@ -18,9 +18,10 @@ namespace phlex::experimental { product_store_const_ptr generator::make_child(std::size_t const i, products new_products) { - auto child = parent_->make_child(i, child_layer_name_, node_name_, std::move(new_products)); - ++child_counts_[child->id()->layer_hash()]; - return child; + auto child_index = parent_->id()->make_child(i, child_layer_name_); + ++child_counts_[child_index->layer_hash()]; + return std::make_shared( + child_index, node_name_, stage::process, std::move(new_products)); } product_store_const_ptr generator::flush_store() const diff --git a/phlex/core/framework_graph.cpp b/phlex/core/framework_graph.cpp index d9fdaab5..9913a767 100644 --- a/phlex/core/framework_graph.cpp +++ b/phlex/core/framework_graph.cpp @@ -52,7 +52,7 @@ namespace phlex::experimental { return {}; } auto index = *item; - auto store = std::make_shared(nullptr, index, "Source"); + auto store = std::make_shared(index, "Source"); return sender_.make_message(accept(std::move(store))); }}, multiplexer_{graph_} diff --git a/phlex/model/product_store.cpp b/phlex/model/product_store.cpp index 42934d88..74f44355 100644 --- a/phlex/model/product_store.cpp +++ b/phlex/model/product_store.cpp @@ -6,12 +6,10 @@ namespace phlex::experimental { - product_store::product_store(product_store_const_ptr parent, - data_cell_index_ptr id, + product_store::product_store(data_cell_index_ptr id, std::string source, stage processing_stage, products new_products) : - parent_{std::move(parent)}, products_{std::move(new_products)}, id_{std::move(id)}, source_{std::move(source)}, @@ -19,99 +17,20 @@ namespace phlex::experimental { { } - product_store::product_store(product_store_const_ptr parent, - std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - products new_products) : - parent_{parent}, - products_{std::move(new_products)}, - id_{parent->id()->make_child(data_cell_number, child_layer_name)}, - source_{std::move(source)}, - stage_{stage::process} - { - } - - product_store::product_store(product_store_const_ptr parent, - std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - stage processing_stage) : - parent_{parent}, - id_{parent->id()->make_child(data_cell_number, child_layer_name)}, - source_{std::move(source)}, - stage_{processing_stage} - { - } - product_store::~product_store() = default; product_store_ptr product_store::base(std::string base_name) { - return product_store_ptr{ - new product_store{nullptr, data_cell_index::base_ptr(), std::move(base_name)}}; - } - - product_store_const_ptr product_store::parent(std::string const& layer_name) const noexcept - { - auto store = parent_; - while (store != nullptr) { - if (store->layer_name() == layer_name) { - return store; - } - store = store->parent_; - } - return nullptr; - } - - product_store_const_ptr product_store::store_for_product(std::string const& product_name) const - { - auto store = shared_from_this(); - while (store != nullptr) { - if (store->contains_product(product_name)) { - return store; - } - store = store->parent_; - } - return nullptr; + return product_store_ptr{new product_store{data_cell_index::base_ptr(), std::move(base_name)}}; } product_store_ptr product_store::make_flush() const { - return product_store_ptr{new product_store{parent_, id_, "[inserted]", stage::flush}}; - } - - product_store_ptr product_store::make_continuation(std::string source, - products new_products) const - { - return product_store_ptr{ - new product_store{parent_, id_, std::move(source), stage::process, std::move(new_products)}}; - } - - product_store_ptr product_store::make_child(std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - products new_products) - { - return product_store_ptr{new product_store{shared_from_this(), - data_cell_number, - child_layer_name, - std::move(source), - std::move(new_products)}}; - } - - product_store_ptr product_store::make_child(std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - stage processing_stage) - { - return product_store_ptr{new product_store{ - shared_from_this(), data_cell_number, child_layer_name, std::move(source), processing_stage}}; + return product_store_ptr{new product_store{id_, "[inserted]", stage::flush}}; } std::string const& product_store::layer_name() const noexcept { return id_->layer_name(); } std::string const& product_store::source() const noexcept { return source_; } - product_store_const_ptr product_store::parent() const noexcept { return parent_; } data_cell_index_ptr const& product_store::id() const noexcept { return id_; } bool product_store::is_flush() const noexcept { return stage_ == stage::flush; } diff --git a/phlex/model/product_store.hpp b/phlex/model/product_store.hpp index beec4a2c..3af653f2 100644 --- a/phlex/model/product_store.hpp +++ b/phlex/model/product_store.hpp @@ -7,26 +7,22 @@ #include "phlex/model/products.hpp" #include -#include #include #include +#include #include namespace phlex::experimental { class product_store : public std::enable_shared_from_this { public: - - explicit product_store(product_store_const_ptr parent, - data_cell_index_ptr id, - std::string source, + explicit product_store(data_cell_index_ptr id, + std::string source = "Source", stage processing_stage = stage::process, products new_products = {}); ~product_store(); static product_store_ptr base(std::string base_name = "Source"); - product_store_const_ptr store_for_product(std::string const& product_name) const; - auto begin() const noexcept { return products_.begin(); } auto end() const noexcept { return products_.end(); } auto size() const noexcept { return products_.size(); } @@ -34,18 +30,7 @@ namespace phlex::experimental { std::string const& layer_name() const noexcept; std::string const& source() const noexcept; - product_store_const_ptr parent(std::string const& layer_name) const noexcept; - product_store_const_ptr parent() const noexcept; product_store_ptr make_flush() const; - product_store_ptr make_continuation(std::string source, products new_products = {}) const; - product_store_ptr make_child(std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - products new_products); - product_store_ptr make_child(std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source = {}, - stage st = stage::process); data_cell_index_ptr const& id() const noexcept; bool is_flush() const noexcept; @@ -66,18 +51,6 @@ namespace phlex::experimental { void add_product(std::string const& key, std::unique_ptr>&& t); private: - explicit product_store(product_store_const_ptr parent, - std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - products new_products); - explicit product_store(product_store_const_ptr parent, - std::size_t data_cell_number, - std::string const& child_layer_name, - std::string source, - stage processing_stage); - - product_store_const_ptr parent_{nullptr}; products products_{}; data_cell_index_ptr id_; std::string diff --git a/test/data_cell_counting.cpp b/test/data_cell_counting.cpp index 23abd5b4..0e7e045b 100644 --- a/test/data_cell_counting.cpp +++ b/test/data_cell_counting.cpp @@ -1,7 +1,6 @@ #include "phlex/model/data_cell_counter.hpp" #include "phlex/model/data_cell_index.hpp" #include "phlex/model/data_layer_hierarchy.hpp" -#include "phlex/model/product_store.hpp" #include "phlex/utilities/hashing.hpp" #include "catch2/catch_test_macros.hpp" @@ -54,41 +53,41 @@ TEST_CASE("Counter multiple layers deep", "[data model]") auto const subrun_hash_value = hash(run_hash_value, "subrun"); auto const event_hash_value = hash(subrun_hash_value, "event"); - auto job_store = product_store::base(); - counters.update(job_store->id()); + auto job_index = data_cell_index::base_ptr(); + counters.update(job_index); for (std::size_t i = 0; i != nruns; ++i) { - auto run_store = job_store->make_child(i, "run"); - counters.update(run_store->id()); + auto run_index = job_index->make_child(i, "run"); + counters.update(run_index); for (std::size_t j = 0; j != nsubruns_per_run; ++j) { - auto subrun_store = run_store->make_child(j, "subrun"); - counters.update(subrun_store->id()); + auto subrun_index = run_index->make_child(j, "subrun"); + counters.update(subrun_index); for (std::size_t k = 0; k != nevents_per_subrun; ++k) { - auto event_store = subrun_store->make_child(k, "event"); - counters.update(event_store->id()); + auto event_index = subrun_index->make_child(k, "event"); + counters.update(event_index); ++processed_events; - h.increment_count(event_store->id()); - auto results = counters.extract(event_store->id()); + h.increment_count(event_index); + auto results = counters.extract(event_index); CHECK(results.empty()); check_all_processed(); } - h.increment_count(subrun_store->id()); - auto results = counters.extract(subrun_store->id()); + h.increment_count(subrun_index); + auto results = counters.extract(subrun_index); ++processed_subruns; CHECK(results.count_for(event_hash_value)); check_all_processed(); } - h.increment_count(run_store->id()); - auto results = counters.extract(run_store->id()); + h.increment_count(run_index); + auto results = counters.extract(run_index); ++processed_runs; CHECK(results.count_for(event_hash_value) == nevents_per_subrun * nsubruns_per_run); CHECK(results.count_for(subrun_hash_value) == nsubruns_per_run); check_all_processed(); } - h.increment_count(job_store->id()); - auto results = counters.extract(job_store->id()); + h.increment_count(job_index); + auto results = counters.extract(job_index); ++processed_jobs; CHECK(results.count_for(event_hash_value) == nevents_per_subrun * nsubruns_per_run * nruns); diff --git a/test/product_store.cpp b/test/product_store.cpp index 35f103e7..bea0c368 100644 --- a/test/product_store.cpp +++ b/test/product_store.cpp @@ -49,7 +49,7 @@ TEST_CASE("Product store derivation", "[data model]") } auto root = product_store::base(); - auto trunk = root->make_child(1, "trunk"); + auto trunk = std::make_shared(root->id()->make_child(1, "trunk")); SECTION("Compare different generations") { CHECK(trunk == more_derived(root, trunk)); @@ -57,15 +57,15 @@ TEST_CASE("Product store derivation", "[data model]") } SECTION("Compare siblings (right is always favored)") { - auto bole = root->make_child(2, "bole"); + auto bole = std::make_shared(root->id()->make_child(2, "bole")); CHECK(bole == more_derived(trunk, bole)); CHECK(trunk == more_derived(bole, trunk)); } - auto limb = trunk->make_child(2, "limb"); - auto branch = limb->make_child(3, "branch"); - auto twig = branch->make_child(4, "twig"); - auto leaf = twig->make_child(5, "leaf"); + auto limb = std::make_shared(trunk->id()->make_child(2, "limb")); + auto branch = std::make_shared(limb->id()->make_child(3, "branch")); + auto twig = std::make_shared(branch->id()->make_child(4, "twig")); + auto leaf = std::make_shared(twig->id()->make_child(5, "leaf")); auto order_a = std::make_tuple(root, trunk, limb, branch, twig, leaf); auto order_b = std::make_tuple(leaf, twig, branch, limb, trunk, root); From 141496c576d7f2e0db81faada2aef9ae8207e51f Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 09:47:53 -0600 Subject: [PATCH 27/49] Apply clang-format --- phlex/core/graph_proxy.hpp | 2 +- test/allowed_families.cpp | 5 +++-- test/benchmarks/benchmarks_provider.cpp | 9 ++++----- test/class_registration.cpp | 11 +++++------ test/demo-giantdata/unfold_transform_fold.cpp | 16 ++++++++-------- test/demo-giantdata/user_algorithms.cpp | 3 +-- test/demo-giantdata/waveform_generator.cpp | 9 ++------- test/demo-giantdata/waveform_generator_input.cpp | 4 +--- test/demo-giantdata/waveforms.cpp | 4 +--- test/function_registration.cpp | 5 ++--- test/max-parallelism/check_parallelism.cpp | 4 +--- test/memory-checks/many_events.cpp | 3 ++- test/mock-workflow/id_provider.cpp | 9 ++++----- test/plugins/module.cpp | 6 ++++-- 14 files changed, 39 insertions(+), 51 deletions(-) diff --git a/phlex/core/graph_proxy.hpp b/phlex/core/graph_proxy.hpp index 34665171..acae2bf3 100644 --- a/phlex/core/graph_proxy.hpp +++ b/phlex/core/graph_proxy.hpp @@ -66,7 +66,7 @@ namespace phlex::experimental { { return create_glue().predicate(std::move(name), std::move(f), c); } - + auto provide(std::string name, is_provider_like auto f, concurrency c = concurrency::serial) { return create_glue().provide(std::move(name), std::move(f), c); diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index f3e3f83e..f0239a16 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -21,7 +21,7 @@ namespace { auto subrun_index = run_index->make_child(0, "subrun"); driver.yield(subrun_index); - + auto event_index = subrun_index->make_child(0, "event"); driver.yield(event_index); } @@ -58,7 +58,8 @@ TEST_CASE("Testing families", "[data model]") framework_graph g{cells_to_process, 2}; // Wire up providers for each level - g.provide("run_id_provider", provide_run_id, concurrency::unlimited).output_product("id"_in("run")); + g.provide("run_id_provider", provide_run_id, concurrency::unlimited) + .output_product("id"_in("run")); g.provide("subrun_id_provider", provide_subrun_id, concurrency::unlimited) .output_product("id"_in("subrun")); g.provide("event_id_provider", provide_event_id, concurrency::unlimited) diff --git a/test/benchmarks/benchmarks_provider.cpp b/test/benchmarks/benchmarks_provider.cpp index 295e01a4..92aefc5e 100644 --- a/test/benchmarks/benchmarks_provider.cpp +++ b/test/benchmarks/benchmarks_provider.cpp @@ -1,9 +1,8 @@ #include "phlex/module.hpp" -PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { +PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) +{ using namespace phlex::experimental; - m.provide("provide_id", [](data_cell_index const& id) { - return id; - }) - .output_product("id"_in("event")); + m.provide("provide_id", [](data_cell_index const& id) { return id; }) + .output_product("id"_in("event")); } diff --git a/test/class_registration.cpp b/test/class_registration.cpp index a97900e8..948712f9 100644 --- a/test/class_registration.cpp +++ b/test/class_registration.cpp @@ -14,9 +14,9 @@ using namespace phlex::experimental; namespace { // Provider functions int provide_number(data_cell_index const&) { return 3; } - + double provide_temperature(data_cell_index const&) { return 98.5; } - + std::string provide_name(data_cell_index const&) { return "John"; } } @@ -63,15 +63,14 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const oproduct_names{"onumber"s, "otemperature"s, "oname"s}; framework_graph g{data_cell_index::base_ptr()}; - + // Register providers for the input products g.provide("provide_number", provide_number, concurrency::unlimited) .output_product("number"_in("job")); g.provide("provide_temperature", provide_temperature, concurrency::unlimited) .output_product("temperature"_in("job")); - g.provide("provide_name", provide_name, concurrency::unlimited) - .output_product("name"_in("job")); - + g.provide("provide_name", provide_name, concurrency::unlimited).output_product("name"_in("job")); + auto glueball = g.make(); SECTION("No framework") { diff --git a/test/demo-giantdata/unfold_transform_fold.cpp b/test/demo-giantdata/unfold_transform_fold.cpp index 0262a75d..7f45e7a2 100644 --- a/test/demo-giantdata/unfold_transform_fold.cpp +++ b/test/demo-giantdata/unfold_transform_fold.cpp @@ -87,14 +87,14 @@ int main(int argc, char* argv[]) // to the constructor of the WaveformGenerator, so we will use the default value. auto const chunksize = 256LL; // this could be read from a configuration file - g.provide("provide_wgen", [wires_per_spill](data_cell_index const& spill_index) { - return - demo::WGI(wires_per_spill, - spill_index.parent()->parent()->number(), // ugh - spill_index.parent()->number(), - spill_index.number()); - }) - .output_product("wget"_in("spill")); + g.provide("provide_wgen", + [wires_per_spill](data_cell_index const& spill_index) { + return demo::WGI(wires_per_spill, + spill_index.parent()->parent()->number(), // ugh + spill_index.parent()->number(), + spill_index.number()); + }) + .output_product("wget"_in("spill")); g.unfold( "WaveformGenerator", diff --git a/test/demo-giantdata/user_algorithms.cpp b/test/demo-giantdata/user_algorithms.cpp index 15bd1c62..71e136e4 100644 --- a/test/demo-giantdata/user_algorithms.cpp +++ b/test/demo-giantdata/user_algorithms.cpp @@ -16,8 +16,7 @@ demo::Waveforms demo::clampWaveforms(demo::Waveforms const& input) } // This is the fold operator that will accumulate a SummedClampedWaveforms object. -void demo::accumulateSCW(demo::SummedClampedWaveforms& accumulator, - demo::Waveforms const& wf) +void demo::accumulateSCW(demo::SummedClampedWaveforms& accumulator, demo::Waveforms const& wf) { // This is the fold operator that will accumulate a SummedClampedWaveforms object. accumulator.size += wf.size(); diff --git a/test/demo-giantdata/waveform_generator.cpp b/test/demo-giantdata/waveform_generator.cpp index 53e6d1bc..101b89a2 100644 --- a/test/demo-giantdata/waveform_generator.cpp +++ b/test/demo-giantdata/waveform_generator.cpp @@ -5,14 +5,9 @@ #include -demo::WaveformGenerator::WaveformGenerator(WGI const& wgi) : - maxsize_{wgi.size} -{ -} +demo::WaveformGenerator::WaveformGenerator(WGI const& wgi) : maxsize_{wgi.size} {} -demo::WaveformGenerator::~WaveformGenerator() -{ -} +demo::WaveformGenerator::~WaveformGenerator() {} std::size_t demo::WaveformGenerator::initial_value() const { return 0; } diff --git a/test/demo-giantdata/waveform_generator_input.cpp b/test/demo-giantdata/waveform_generator_input.cpp index 990a8d47..2ee458a7 100644 --- a/test/demo-giantdata/waveform_generator_input.cpp +++ b/test/demo-giantdata/waveform_generator_input.cpp @@ -34,6 +34,4 @@ demo::WaveformGeneratorInput& demo::WaveformGeneratorInput::operator=( return *this; } -demo::WaveformGeneratorInput::~WaveformGeneratorInput() -{ -} +demo::WaveformGeneratorInput::~WaveformGeneratorInput() {} diff --git a/test/demo-giantdata/waveforms.cpp b/test/demo-giantdata/waveforms.cpp index 62c8cc05..18c46ec0 100644 --- a/test/demo-giantdata/waveforms.cpp +++ b/test/demo-giantdata/waveforms.cpp @@ -34,6 +34,4 @@ demo::Waveforms& demo::Waveforms::operator=(Waveforms&& other) return *this; } -demo::Waveforms::~Waveforms() -{ -}; +demo::Waveforms::~Waveforms() {}; diff --git a/test/function_registration.cpp b/test/function_registration.cpp index ed3898a7..67a1364b 100644 --- a/test/function_registration.cpp +++ b/test/function_registration.cpp @@ -58,15 +58,14 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const oproduct_names = {"onumber"s, "otemperature"s, "oname"s}; std::array const result{"result"s}; - framework_graph g {data_cell_index::base_ptr()}; + framework_graph g{data_cell_index::base_ptr()}; // Register providers g.provide("provide_number", provide_number, concurrency::unlimited) .output_product("number"_in("job")); g.provide("provide_temperature", provide_temperature, concurrency::unlimited) .output_product("temperature"_in("job")); - g.provide("provide_name", provide_name, concurrency::unlimited) - .output_product("name"_in("job")); + g.provide("provide_name", provide_name, concurrency::unlimited).output_product("name"_in("job")); SECTION("No framework") { diff --git a/test/max-parallelism/check_parallelism.cpp b/test/max-parallelism/check_parallelism.cpp index 739212a3..338e77ca 100644 --- a/test/max-parallelism/check_parallelism.cpp +++ b/test/max-parallelism/check_parallelism.cpp @@ -32,9 +32,7 @@ PHLEX_EXPERIMENTAL_REGISTER_SOURCE(send_parallelism) PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m, config) { m.provide("provide_max_parallelism", - [](data_cell_index const&) { - return max_allowed_parallelism::active_value(); - }) + [](data_cell_index const&) { return max_allowed_parallelism::active_value(); }) .output_product("max_parallelism"_in("job")); m.observe("verify_expected", diff --git a/test/memory-checks/many_events.cpp b/test/memory-checks/many_events.cpp index 2bc5c6fa..636f12a5 100644 --- a/test/memory-checks/many_events.cpp +++ b/test/memory-checks/many_events.cpp @@ -24,7 +24,8 @@ int main() }; framework_graph g{cells_to_process}; - g.provide("provide_number", [](data_cell_index const& id)->unsigned { return id.number(); }).output_product("number"_in("event")); + g.provide("provide_number", [](data_cell_index const& id) -> unsigned { return id.number(); }) + .output_product("number"_in("event")); g.transform("pass_on", pass_on, concurrency::unlimited) .input_family("number"_in("event")) .output_products("different"); diff --git a/test/mock-workflow/id_provider.cpp b/test/mock-workflow/id_provider.cpp index 295e01a4..92aefc5e 100644 --- a/test/mock-workflow/id_provider.cpp +++ b/test/mock-workflow/id_provider.cpp @@ -1,9 +1,8 @@ #include "phlex/module.hpp" -PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { +PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) +{ using namespace phlex::experimental; - m.provide("provide_id", [](data_cell_index const& id) { - return id; - }) - .output_product("id"_in("event")); + m.provide("provide_id", [](data_cell_index const& id) { return id; }) + .output_product("id"_in("event")); } diff --git a/test/plugins/module.cpp b/test/plugins/module.cpp index 61969554..23001795 100644 --- a/test/plugins/module.cpp +++ b/test/plugins/module.cpp @@ -9,8 +9,10 @@ using namespace phlex::experimental; PHLEX_EXPERIMENTAL_REGISTER_ALGORITHMS(m) { - m.provide("provide_i", [](data_cell_index const& id)->int{ return id.number(); }).output_product("i"_in("event")); - m.provide("provide_j", [](data_cell_index const& id)->int{ return -id.number(); }).output_product("j"_in("event")); + m.provide("provide_i", [](data_cell_index const& id) -> int { return id.number(); }) + .output_product("i"_in("event")); + m.provide("provide_j", [](data_cell_index const& id) -> int { return -id.number(); }) + .output_product("j"_in("event")); m.transform("add", test::add, concurrency::unlimited) .input_family("i"_in("event"), "j"_in("event")) .output_products("sum"); From 0bfb7bc8c2aaf2a308d2289053959fb9599a6ed4 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Tue, 9 Dec 2025 13:15:03 -0600 Subject: [PATCH 28/49] added algorithm header --- test/demo-giantdata/user_algorithms.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/demo-giantdata/user_algorithms.cpp b/test/demo-giantdata/user_algorithms.cpp index 71e136e4..f4278562 100644 --- a/test/demo-giantdata/user_algorithms.cpp +++ b/test/demo-giantdata/user_algorithms.cpp @@ -1,3 +1,5 @@ +#include + #include "user_algorithms.hpp" #include "summed_clamped_waveforms.hpp" #include "waveforms.hpp" From 6904f36ad386bc720fa134906a3f10abc6db0b70 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 12:14:10 -0600 Subject: [PATCH 29/49] Adjust driver to accommodate API change --- test/provider_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 8e0ced0b..00c41363 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -34,12 +34,12 @@ TEST_CASE("provider_test") spdlog::flush_on(spdlog::level::trace); auto levels_to_process = [](framework_driver& driver) { - auto job_store = product_store::base(); - driver.yield(job_store); + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); for (unsigned int i : std::views::iota(1u, max_events + 1)) { - auto store = job_store->make_child(i, "spill", "Source"); - driver.yield(store); + auto index = job_index->make_child(i, "spill"); + driver.yield(index); } }; From 96cb5d45d703ccb3a140041a4070eb93efbd8da6 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Tue, 9 Dec 2025 13:15:58 -0600 Subject: [PATCH 30/49] updated multiplexer to work with providers --- phlex/core/edge_maker.hpp | 3 ++- phlex/core/multiplexer.cpp | 16 +++++++++------- test/allowed_families.cpp | 36 +++++++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index ad587650..87a12d9c 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -115,7 +115,8 @@ namespace phlex::experimental { // FIXME: The check should probably be more robust. Right now, the product_specification // buried in the p->input()[0] call does not have its type set, which prevents us from // doing a simpler comparison (e.g., port.product_label == p->input()[0]). - if (port.product_label.name.full() == p->input()[0].name.full()) { + if (port.product_label.name.full() == p->input()[0].name.full() && + port.product_label.layer == p->input()[0].layer) { auto& provider = *p; assert(provider.ports().size() == 1); auto it = provider_ports.find(provider.full_name()); diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index a2aa7f93..f7f4db26 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -32,14 +32,16 @@ namespace { std::vector result; result.reserve(ports.size()); for (auto const& [product_label, port] : ports) { - // Look for the product_label that matches our store's layer_name. - if (auto const& allowed_layer = product_label.layer; not allowed_layer.empty()) { - if (store->layer_name() != allowed_layer) { - // This store level does not match the required layer - continue; + // Look for the product_label that matches our store's level_name. + if( store->id()->layer_name() == product_label.layer) { + // This store layer does not match the required layer + result.push_back({port, store}); + } + else if (auto index = store->id()->parent(product_label.layer)){ + // This store layer does not match the required layer + result.push_back({port, std::make_shared( + index, store->source())}); } - } - result.push_back({port, store}); } assert(result.size() <= 1); return result; diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index f0239a16..24f5e084 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -14,24 +14,37 @@ namespace { void cells_to_process(framework_driver& driver) { auto job_index = data_cell_index::base_ptr(); + spdlog::info("Yielding job index with depth {}", job_index->depth()); driver.yield(job_index); auto run_index = job_index->make_child(0, "run"); + spdlog::info("Yielding run index with depth {}", run_index->depth()); driver.yield(run_index); auto subrun_index = run_index->make_child(0, "subrun"); + spdlog::info("Yielding subrun index with depth {}", subrun_index->depth()); driver.yield(subrun_index); auto event_index = subrun_index->make_child(0, "event"); + spdlog::info("Yielding event index with depth {}", event_index->depth()); driver.yield(event_index); } // Provider functions that return data_cell_index for each level - data_cell_index provide_run_id(data_cell_index const& index) { return index; } + data_cell_index provide_run_id(data_cell_index const& index) { + spdlog::info("Providing run id with depth {}", index.depth()); + return index; + } - data_cell_index provide_subrun_id(data_cell_index const& index) { return index; } + data_cell_index provide_subrun_id(data_cell_index const& index) { + spdlog::info("Providing subrun id with depth {}", index.depth()); + return index; + } - data_cell_index provide_event_id(data_cell_index const& index) { return index; } + data_cell_index provide_event_id(data_cell_index const& index) { + spdlog::info("Providing event id with depth {}", index.depth()); + return index; + } void check_two_ids(data_cell_index const& parent_id, data_cell_index const& id) { @@ -39,6 +52,23 @@ namespace { CHECK(parent_id.hash() == id.parent()->hash()); } +/************* ✨ Windsurf Command ⭐ *************/ +/** + * @brief Checks the relationships between three data_cell_index objects. + * + * @param grandparent_id The data_cell_index object at depth 1. + * @param parent_id The data_cell_index object at depth 2, which is the parent of id. + * @param id The data_cell_index object at depth 3. + * + * This function checks that the relationships between the three objects are correct. + * Specifically, it checks that: + * - id is at depth 3 + * - parent_id is at depth 2 + * - grandparent_id is at depth 1 + * - grandparent_id is the grandparent of id + * - parent_id is the parent of id + */ +/******* f1db79b2-2a69-493d-ae03-20d77cee24b3 *******/ void check_three_ids(data_cell_index const& grandparent_id, data_cell_index const& parent_id, data_cell_index const& id) From 3be11b34a85596246c4f5d574bf1a4fa94eff69e Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Tue, 9 Dec 2025 13:19:00 -0600 Subject: [PATCH 31/49] removed fancy comments --- test/allowed_families.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index 24f5e084..5b81b705 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -52,23 +52,6 @@ namespace { CHECK(parent_id.hash() == id.parent()->hash()); } -/************* ✨ Windsurf Command ⭐ *************/ -/** - * @brief Checks the relationships between three data_cell_index objects. - * - * @param grandparent_id The data_cell_index object at depth 1. - * @param parent_id The data_cell_index object at depth 2, which is the parent of id. - * @param id The data_cell_index object at depth 3. - * - * This function checks that the relationships between the three objects are correct. - * Specifically, it checks that: - * - id is at depth 3 - * - parent_id is at depth 2 - * - grandparent_id is at depth 1 - * - grandparent_id is the grandparent of id - * - parent_id is the parent of id - */ -/******* f1db79b2-2a69-493d-ae03-20d77cee24b3 *******/ void check_three_ids(data_cell_index const& grandparent_id, data_cell_index const& parent_id, data_cell_index const& id) From 2c9ebed8933296bb5ff8fa55d3f3b6319f596cf6 Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Tue, 9 Dec 2025 13:22:56 -0600 Subject: [PATCH 32/49] added execution checks for providers --- test/allowed_families.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index 5b81b705..0f72003d 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -87,4 +87,7 @@ TEST_CASE("Testing families", "[data model]") CHECK(g.execution_counts("se") == 1ull); CHECK(g.execution_counts("rs") == 1ull); CHECK(g.execution_counts("rse") == 1ull); + CHECK(g.execution_counts("run_id_provider") == 1ull); + CHECK(g.execution_counts("subrun_id_provider") == 1ull); + CHECK(g.execution_counts("event_id_provider") == 1ull); } From 32ab6a48a45b930a99e8da75be67c8fa9bcad018 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 13:42:45 -0600 Subject: [PATCH 33/49] Apply clang-format and update code comments --- phlex/core/multiplexer.cpp | 18 ++++++++---------- test/allowed_families.cpp | 17 ++++++++++------- test/demo-giantdata/user_algorithms.cpp | 2 +- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index f7f4db26..4ea7cd71 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -32,16 +32,14 @@ namespace { std::vector result; result.reserve(ports.size()); for (auto const& [product_label, port] : ports) { - // Look for the product_label that matches our store's level_name. - if( store->id()->layer_name() == product_label.layer) { - // This store layer does not match the required layer - result.push_back({port, store}); - } - else if (auto index = store->id()->parent(product_label.layer)){ - // This store layer does not match the required layer - result.push_back({port, std::make_shared( - index, store->source())}); - } + if (store->id()->layer_name() == product_label.layer) { + // This store's layer matches what is expected by the port + result.push_back({port, store}); + } else if (auto index = store->id()->parent(product_label.layer)) { + // This store has a parent layer that matches what is expected by the port + result.push_back( + {port, std::make_shared(index, store->source())}); + } } assert(result.size() <= 1); return result; diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index 0f72003d..00ead597 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -31,19 +31,22 @@ namespace { } // Provider functions that return data_cell_index for each level - data_cell_index provide_run_id(data_cell_index const& index) { + data_cell_index provide_run_id(data_cell_index const& index) + { spdlog::info("Providing run id with depth {}", index.depth()); - return index; + return index; } - data_cell_index provide_subrun_id(data_cell_index const& index) { - spdlog::info("Providing subrun id with depth {}", index.depth()); - return index; + data_cell_index provide_subrun_id(data_cell_index const& index) + { + spdlog::info("Providing subrun id with depth {}", index.depth()); + return index; } - data_cell_index provide_event_id(data_cell_index const& index) { + data_cell_index provide_event_id(data_cell_index const& index) + { spdlog::info("Providing event id with depth {}", index.depth()); - return index; + return index; } void check_two_ids(data_cell_index const& parent_id, data_cell_index const& id) diff --git a/test/demo-giantdata/user_algorithms.cpp b/test/demo-giantdata/user_algorithms.cpp index f4278562..56a019e4 100644 --- a/test/demo-giantdata/user_algorithms.cpp +++ b/test/demo-giantdata/user_algorithms.cpp @@ -1,7 +1,7 @@ #include -#include "user_algorithms.hpp" #include "summed_clamped_waveforms.hpp" +#include "user_algorithms.hpp" #include "waveforms.hpp" // This function is used to transform an input Waveforms object into an From 8efa8f046b5a13c3dc1d801d7673d348e9dd25ed Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 14:12:42 -0600 Subject: [PATCH 34/49] Move making of provider edges to dedicated function --- phlex/core/CMakeLists.txt | 1 + phlex/core/edge_maker.cpp | 52 ++++++++++++++++++++++++++++++++++++++ phlex/core/edge_maker.hpp | 52 +++++--------------------------------- phlex/core/multiplexer.cpp | 9 ++++++- 4 files changed, 67 insertions(+), 47 deletions(-) create mode 100644 phlex/core/edge_maker.cpp diff --git a/phlex/core/CMakeLists.txt b/phlex/core/CMakeLists.txt index 5cc786ad..20c0443a 100644 --- a/phlex/core/CMakeLists.txt +++ b/phlex/core/CMakeLists.txt @@ -16,6 +16,7 @@ cet_make_library( detail/maybe_predicates.cpp detail/filter_impl.cpp edge_creation_policy.cpp + edge_maker.cpp end_of_message.cpp filter.cpp framework_graph.cpp diff --git a/phlex/core/edge_maker.cpp b/phlex/core/edge_maker.cpp new file mode 100644 index 00000000..b6bca713 --- /dev/null +++ b/phlex/core/edge_maker.cpp @@ -0,0 +1,52 @@ +#include "phlex/core/edge_maker.hpp" + +#include "spdlog/spdlog.h" +#include "tbb/flow_graph.h" + +#include + +namespace phlex::experimental { + multiplexer::head_ports_t make_provider_edges(multiplexer::head_ports_t head_ports, + declared_providers& providers) + { + assert(!head_ports.empty()); + + multiplexer::head_ports_t result; + for (auto const& [node_name, ports] : head_ports) { + for (auto const& port : ports) { + // Find the provider that has the right product name (hidden in the + // output port) and the right family (hidden in the input port). + bool found_match = false; + for (auto const& [_, p] : providers) { + auto& provider = *p; + // FIXME: The check should probably be more robust. Right now, the + // product_specification buried in the provider.input()[0] call does not + // have its type set, which prevents us from doing a simpler comparison + // (e.g., port.product_label == provider.input()[0]). + if (port.product_label.name.full() == provider.input()[0].name.full() && + port.product_label.layer == provider.input()[0].layer) { + assert(provider.ports().size() == 1); + auto it = result.find(provider.full_name()); + if (it == result.cend()) { + multiplexer::named_input_ports_t provider_input_ports; + provider_input_ports.emplace_back(port.product_label, provider.ports()[0]); + result.try_emplace(provider.full_name(), std::move(provider_input_ports)); + } + spdlog::info("Connecting provider {} to node {} (product: {})", + provider.full_name(), + node_name, + port.product_label.to_string()); + make_edge(provider.sender(), *(port.port)); + found_match = true; + break; + } + } + if (!found_match) { + throw std::runtime_error("No provider found for product: "s + + port.product_label.to_string()); + } + } + } + return result; + } +} diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index 87a12d9c..6a0417f6 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -2,6 +2,7 @@ #define PHLEX_CORE_EDGE_MAKER_HPP #include "phlex/core/declared_output.hpp" +#include "phlex/core/declared_provider.hpp" #include "phlex/core/edge_creation_policy.hpp" #include "phlex/core/filter.hpp" #include "phlex/core/multiplexer.hpp" @@ -23,6 +24,9 @@ namespace phlex::experimental { using product_name_t = std::string; + multiplexer::head_ports_t make_provider_edges(multiplexer::head_ports_t head_ports, + declared_providers& providers); + class edge_maker { public: template @@ -100,52 +104,8 @@ namespace phlex::experimental { // figure out what provider nodes are needed. // For now, we take as input a mapping of declared_providers. - // wire up the head_ports to the given providers. - // If any head_port does not have a matching provider, that is - // an error. - multiplexer::head_ports_t provider_ports; - assert(!head_ports.empty()); - - for (auto const& [node_name, ports] : head_ports) { - for (auto const& port : ports) { - // Find the provider that has the right product name (hidden in the - // output port) and the right family (hidden in the input port). - bool found_match = false; - for (auto const& [_, p] : providers) { - // FIXME: The check should probably be more robust. Right now, the product_specification - // buried in the p->input()[0] call does not have its type set, which prevents us from - // doing a simpler comparison (e.g., port.product_label == p->input()[0]). - if (port.product_label.name.full() == p->input()[0].name.full() && - port.product_label.layer == p->input()[0].layer) { - auto& provider = *p; - assert(provider.ports().size() == 1); - auto it = provider_ports.find(provider.full_name()); - if (it == provider_ports.cend()) { - multiplexer::named_input_ports_t provider_input_ports; - provider_input_ports.emplace_back(port.product_label, provider.ports()[0]); - provider_ports.try_emplace(provider.full_name(), std::move(provider_input_ports)); - } - spdlog::info("Connecting provider {} to node {} (product: {})", - provider.full_name(), - node_name, - port.product_label.to_string()); - make_edge(provider.sender(), *(port.port)); - found_match = true; - break; - } - } - if (!found_match) { - throw std::runtime_error("No provider found for product: "s + - port.product_label.to_string()); - } - } - } - - // We must have at least one provider port, or there can be no data to - // process. - assert(!provider_ports.empty()); - // Pass the providers to the multiplexer. - multi.finalize(std::move(provider_ports)); + auto provider_input_ports = make_provider_edges(std::move(head_ports), providers); + multi.finalize(std::move(provider_input_ports)); } } diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index 4ea7cd71..f3d973d4 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -53,7 +53,14 @@ namespace phlex::experimental { { } - void multiplexer::finalize(head_ports_t head_ports) { head_ports_ = std::move(head_ports); } + void multiplexer::finalize(head_ports_t head_ports) + { + // We must have at least one provider port, or there can be no data to + // process. + assert(!head_ports.empty()); + + head_ports_ = std::move(head_ports); + } tbb::flow::continue_msg multiplexer::multiplex(message const& msg) { From 3e164c765840ee7e0b00dfbb1d40f085d7eadad1 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 14:18:26 -0600 Subject: [PATCH 35/49] Reduce severity of (or remove) logged messages --- phlex/core/declared_provider.hpp | 2 +- phlex/core/edge_maker.cpp | 8 ++++---- test/allowed_families.cpp | 22 +++------------------- 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index 1c624c37..f921ee53 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -83,7 +83,7 @@ namespace phlex::experimental { to_output.try_put(new_msg); }} { - spdlog::info( + spdlog::debug( "Created provider node {} making output {}", this->full_name(), output.to_string()); } diff --git a/phlex/core/edge_maker.cpp b/phlex/core/edge_maker.cpp index b6bca713..944f594b 100644 --- a/phlex/core/edge_maker.cpp +++ b/phlex/core/edge_maker.cpp @@ -32,10 +32,10 @@ namespace phlex::experimental { provider_input_ports.emplace_back(port.product_label, provider.ports()[0]); result.try_emplace(provider.full_name(), std::move(provider_input_ports)); } - spdlog::info("Connecting provider {} to node {} (product: {})", - provider.full_name(), - node_name, - port.product_label.to_string()); + spdlog::debug("Connecting provider {} to node {} (product: {})", + provider.full_name(), + node_name, + port.product_label.to_string()); make_edge(provider.sender(), *(port.port)); found_match = true; break; diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index 00ead597..a695ee34 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -14,40 +14,24 @@ namespace { void cells_to_process(framework_driver& driver) { auto job_index = data_cell_index::base_ptr(); - spdlog::info("Yielding job index with depth {}", job_index->depth()); driver.yield(job_index); auto run_index = job_index->make_child(0, "run"); - spdlog::info("Yielding run index with depth {}", run_index->depth()); driver.yield(run_index); auto subrun_index = run_index->make_child(0, "subrun"); - spdlog::info("Yielding subrun index with depth {}", subrun_index->depth()); driver.yield(subrun_index); auto event_index = subrun_index->make_child(0, "event"); - spdlog::info("Yielding event index with depth {}", event_index->depth()); driver.yield(event_index); } // Provider functions that return data_cell_index for each level - data_cell_index provide_run_id(data_cell_index const& index) - { - spdlog::info("Providing run id with depth {}", index.depth()); - return index; - } + data_cell_index provide_run_id(data_cell_index const& index) { return index; } - data_cell_index provide_subrun_id(data_cell_index const& index) - { - spdlog::info("Providing subrun id with depth {}", index.depth()); - return index; - } + data_cell_index provide_subrun_id(data_cell_index const& index) { return index; } - data_cell_index provide_event_id(data_cell_index const& index) - { - spdlog::info("Providing event id with depth {}", index.depth()); - return index; - } + data_cell_index provide_event_id(data_cell_index const& index) { return index; } void check_two_ids(data_cell_index const& parent_id, data_cell_index const& id) { From 611de81660eb3e43dfa6e4529fb7fd10e28e08f2 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 15:43:15 -0600 Subject: [PATCH 36/49] Relax check on execution counts for now --- test/allowed_families.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index a695ee34..7e35ee58 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -74,7 +74,9 @@ TEST_CASE("Testing families", "[data model]") CHECK(g.execution_counts("se") == 1ull); CHECK(g.execution_counts("rs") == 1ull); CHECK(g.execution_counts("rse") == 1ull); - CHECK(g.execution_counts("run_id_provider") == 1ull); - CHECK(g.execution_counts("subrun_id_provider") == 1ull); + + // FIXME: Need to improve the synchronization to supply strict equality + CHECK(g.execution_counts("run_id_provider") >= 1ull); + CHECK(g.execution_counts("subrun_id_provider") >= 1ull); CHECK(g.execution_counts("event_id_provider") == 1ull); } From bd989141fcdafc5ed09e57bc29b6815dee71ba6a Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 16:00:11 -0600 Subject: [PATCH 37/49] Simplify multiplexing --- phlex/core/multiplexer.cpp | 58 ++++++++++++-------------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index f3d973d4..70e8333d 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -6,43 +6,28 @@ #include "spdlog/spdlog.h" #include +#include #include #include using namespace std::chrono; +using namespace phlex::experimental; namespace { - - struct sender_slot { - tbb::flow::receiver* port; - phlex::experimental::product_store_const_ptr store; - void operator()(phlex::experimental::end_of_message_ptr eom, std::size_t message_id) const - { - port->try_put({store, eom, message_id}); + product_store_const_ptr store_for(product_store_const_ptr store, + std::string const& port_product_layer) + { + if (store->id()->layer_name() == port_product_layer) { + // This store's layer matches what is expected by the port + return store; } - }; - // FIXME: this function should return optional to indicate - // because it can never return more than one slot, but can return no - // sender_slot. - std::vector senders_for( - phlex::experimental::product_store_const_ptr store, - phlex::experimental::multiplexer::named_input_ports_t const& ports) - { - std::vector result; - result.reserve(ports.size()); - for (auto const& [product_label, port] : ports) { - if (store->id()->layer_name() == product_label.layer) { - // This store's layer matches what is expected by the port - result.push_back({port, store}); - } else if (auto index = store->id()->parent(product_label.layer)) { - // This store has a parent layer that matches what is expected by the port - result.push_back( - {port, std::make_shared(index, store->source())}); - } + if (auto index = store->id()->parent(port_product_layer)) { + // This store has a parent layer that matches what is expected by the port + return std::make_shared(index, store->source()); } - assert(result.size() <= 1); - return result; + + return nullptr; } } @@ -55,8 +40,7 @@ namespace phlex::experimental { void multiplexer::finalize(head_ports_t head_ports) { - // We must have at least one provider port, or there can be no data to - // process. + // We must have at least one provider port, or there can be no data to process. assert(!head_ports.empty()); head_ports_ = std::move(head_ports); @@ -82,16 +66,10 @@ namespace phlex::experimental { } for (auto const& ports : head_ports_ | std::views::values) { - // FIXME: Should make sure that the received store has the same layer name as the most - // derived store required by the algorithm. - auto const senders = senders_for(store, ports); - if (size(senders) != size(ports)) { - // Not enough stores to ports of the node - continue; - } - - for (auto const& sender : senders) { - sender(eom, message_id); + assert(ports.size() == 1ull); + auto const& [product_label, port] = ports[0]; + if (auto const store_to_send = store_for(store, product_label.layer)) { + port->try_put({store_to_send, eom, message_id}); } } From 94dfb00c85dc1c0faa66353ff0a825c41f3bf426 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 17:11:29 -0600 Subject: [PATCH 38/49] Further simplifications to multiplexing --- phlex/core/edge_maker.cpp | 10 ++++------ phlex/core/edge_maker.hpp | 4 ++-- phlex/core/multiplexer.cpp | 22 ++++++++++------------ phlex/core/multiplexer.hpp | 7 +++---- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/phlex/core/edge_maker.cpp b/phlex/core/edge_maker.cpp index 944f594b..cb99fc11 100644 --- a/phlex/core/edge_maker.cpp +++ b/phlex/core/edge_maker.cpp @@ -6,12 +6,12 @@ #include namespace phlex::experimental { - multiplexer::head_ports_t make_provider_edges(multiplexer::head_ports_t head_ports, - declared_providers& providers) + multiplexer::input_ports_t make_provider_edges(multiplexer::head_ports_t head_ports, + declared_providers& providers) { assert(!head_ports.empty()); - multiplexer::head_ports_t result; + multiplexer::input_ports_t result; for (auto const& [node_name, ports] : head_ports) { for (auto const& port : ports) { // Find the provider that has the right product name (hidden in the @@ -28,9 +28,7 @@ namespace phlex::experimental { assert(provider.ports().size() == 1); auto it = result.find(provider.full_name()); if (it == result.cend()) { - multiplexer::named_input_ports_t provider_input_ports; - provider_input_ports.emplace_back(port.product_label, provider.ports()[0]); - result.try_emplace(provider.full_name(), std::move(provider_input_ports)); + result.try_emplace(provider.full_name(), port.product_label, provider.ports()[0]); } spdlog::debug("Connecting provider {} to node {} (product: {})", provider.full_name(), diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index 6a0417f6..4b22c0ea 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -24,8 +24,8 @@ namespace phlex::experimental { using product_name_t = std::string; - multiplexer::head_ports_t make_provider_edges(multiplexer::head_ports_t head_ports, - declared_providers& providers); + multiplexer::input_ports_t make_provider_edges(multiplexer::head_ports_t head_ports, + declared_providers& providers); class edge_maker { public: diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index 70e8333d..fda190ef 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -38,38 +38,36 @@ namespace phlex::experimental { { } - void multiplexer::finalize(head_ports_t head_ports) + void multiplexer::finalize(input_ports_t provider_input_ports) { // We must have at least one provider port, or there can be no data to process. - assert(!head_ports.empty()); - - head_ports_ = std::move(head_ports); + assert(!provider_input_ports.empty()); + provider_input_ports_ = std::move(provider_input_ports); } tbb::flow::continue_msg multiplexer::multiplex(message const& msg) { ++received_messages_; - auto const& [store, eom, message_id] = std::tie(msg.store, msg.eom, msg.id); + auto const& [store, eom, message_id, _] = msg; if (debug_) { spdlog::debug("Multiplexing {} with ID {} (is flush: {})", store->id()->to_string(), message_id, store->is_flush()); } - auto start_time = steady_clock::now(); if (store->is_flush()) { - for (auto const& head_port : head_ports_ | std::views::values | std::views::join) { + for (auto const& head_port : provider_input_ports_ | std::views::values) { head_port.port->try_put(msg); } return {}; } - for (auto const& ports : head_ports_ | std::views::values) { - assert(ports.size() == 1ull); - auto const& [product_label, port] = ports[0]; - if (auto const store_to_send = store_for(store, product_label.layer)) { - port->try_put({store_to_send, eom, message_id}); + auto start_time = steady_clock::now(); + + for (auto const& [product_label, port] : provider_input_ports_ | std::views::values) { + if (auto store_to_send = store_for(store, product_label.layer)) { + port->try_put({std::move(store_to_send), eom, message_id}); } } diff --git a/phlex/core/multiplexer.hpp b/phlex/core/multiplexer.hpp index 9717e198..69241ccf 100644 --- a/phlex/core/multiplexer.hpp +++ b/phlex/core/multiplexer.hpp @@ -29,16 +29,15 @@ namespace phlex::experimental { using named_input_ports_t = std::vector; // map of node name to its input ports using head_ports_t = std::map; + using input_ports_t = std::map; explicit multiplexer(tbb::flow::graph& g, bool debug = false); tbb::flow::continue_msg multiplex(message const& msg); - void finalize(head_ports_t head_ports); - - head_ports_t const& downstream_ports() const noexcept { return head_ports_; } + void finalize(input_ports_t provider_input_ports); private: - head_ports_t head_ports_; + input_ports_t provider_input_ports_; bool debug_; std::atomic received_messages_{}; std::chrono::duration execution_time_{}; From 0ece841e70e9d2d127ec9b82c5da5d76aa92d398 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Tue, 9 Dec 2025 17:14:19 -0600 Subject: [PATCH 39/49] Another tweak --- phlex/core/multiplexer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phlex/core/multiplexer.cpp b/phlex/core/multiplexer.cpp index fda190ef..10f983f9 100644 --- a/phlex/core/multiplexer.cpp +++ b/phlex/core/multiplexer.cpp @@ -57,8 +57,8 @@ namespace phlex::experimental { } if (store->is_flush()) { - for (auto const& head_port : provider_input_ports_ | std::views::values) { - head_port.port->try_put(msg); + for (auto const& [_, port] : provider_input_ports_ | std::views::values) { + port->try_put(msg); } return {}; } From e820868fc768bde203c57c1fd7566c1c06877012 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 07:57:50 -0600 Subject: [PATCH 40/49] Reduce duplication --- test/allowed_families.cpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index 7e35ee58..a4b95182 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -26,12 +26,7 @@ namespace { driver.yield(event_index); } - // Provider functions that return data_cell_index for each level - data_cell_index provide_run_id(data_cell_index const& index) { return index; } - - data_cell_index provide_subrun_id(data_cell_index const& index) { return index; } - - data_cell_index provide_event_id(data_cell_index const& index) { return index; } + data_cell_index provide_index(data_cell_index const& index) { return index; } void check_two_ids(data_cell_index const& parent_id, data_cell_index const& id) { @@ -58,11 +53,11 @@ TEST_CASE("Testing families", "[data model]") framework_graph g{cells_to_process, 2}; // Wire up providers for each level - g.provide("run_id_provider", provide_run_id, concurrency::unlimited) + g.provide("run_id_provider", provide_index, concurrency::unlimited) .output_product("id"_in("run")); - g.provide("subrun_id_provider", provide_subrun_id, concurrency::unlimited) + g.provide("subrun_id_provider", provide_index, concurrency::unlimited) .output_product("id"_in("subrun")); - g.provide("event_id_provider", provide_event_id, concurrency::unlimited) + g.provide("event_id_provider", provide_index, concurrency::unlimited) .output_product("id"_in("event")); g.observe("se", check_two_ids).input_family("id"_in("subrun"), "id"_in("event")); From 74484568870053c23f4891033129e1c0116e0b9c Mon Sep 17 00:00:00 2001 From: Saba Sehrish Date: Wed, 10 Dec 2025 09:05:48 -0600 Subject: [PATCH 41/49] Add caching to providers to prevent unnecessary re-execution Also includes slight adjustment of product_store constructor arguments --- phlex/core/declared_provider.cpp | 14 ++++++++ phlex/core/declared_provider.hpp | 57 +++++++++++++++++++++++-------- phlex/core/declared_transform.hpp | 2 +- phlex/core/declared_unfold.cpp | 3 +- phlex/model/product_store.cpp | 6 ++-- phlex/model/product_store.hpp | 4 +-- test/allowed_families.cpp | 5 ++- 7 files changed, 66 insertions(+), 25 deletions(-) diff --git a/phlex/core/declared_provider.cpp b/phlex/core/declared_provider.cpp index 74767691..15aec3f3 100644 --- a/phlex/core/declared_provider.cpp +++ b/phlex/core/declared_provider.cpp @@ -7,4 +7,18 @@ namespace phlex::experimental { } declared_provider::~declared_provider() = default; + + void declared_provider::report_cached_stores(stores_t const& stores) const + { + if (stores.size() > 0ull) { + spdlog::warn("Provider {} has {} cached stores.", full_name(), stores.size()); + } + for (auto const& [hash, store] : stores) { + if (not store) { + spdlog::warn("Store with hash {} is null!", hash); + continue; + } + spdlog::debug(" => ID: {} (hash: {})", store->id()->to_string(), hash); + } + } } diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index f921ee53..415f256d 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -5,6 +5,7 @@ #include "phlex/core/fwd.hpp" #include "phlex/core/message.hpp" #include "phlex/core/products_consumer.hpp" +#include "phlex/core/store_counters.hpp" #include "phlex/metaprogramming/type_deduction.hpp" #include "phlex/model/algorithm_name.hpp" #include "phlex/model/data_cell_index.hpp" @@ -35,6 +36,12 @@ namespace phlex::experimental { virtual tbb::flow::sender& to_output() = 0; virtual product_specifications const& output() const = 0; virtual std::size_t num_calls() const = 0; + + protected: + using stores_t = tbb::concurrent_hash_map; + using const_accessor = stores_t::const_accessor; + + void report_cached_stores(stores_t const& stores) const; }; using declared_provider_ptr = std::unique_ptr; @@ -43,7 +50,7 @@ namespace phlex::experimental { // ===================================================================================== template - class provider_node : public declared_provider { + class provider_node : public declared_provider, private detect_flush_flag { using function_t = typename AlgorithmBits::bound_type; static constexpr auto M = number_output_objects; @@ -65,28 +72,49 @@ namespace phlex::experimental { auto& [stay_in_graph, to_output] = output; if (msg.store->is_flush()) { + flag_for(msg.store->id()->hash()).flush_received(msg.original_id); stay_in_graph.try_put(msg); - return; + } else { + // Check cache first + auto index_hash = msg.store->id()->hash(); + if (const_accessor ca; cache_.find(ca, index_hash)) { + // Cache hit - reuse the cached store + message const new_msg{ca->second, msg.eom, msg.id}; + stay_in_graph.try_put(new_msg); + to_output.try_put(new_msg); + return; + } + + // Cache miss - compute the result + auto result = std::invoke(ft, *msg.store->id()); + ++calls_; + + products new_products; + // Add all adds all products; we should only have one. Fix this later. + new_products.add_all(output_, std::move(result)); + auto store = std::make_shared( + msg.store->id(), this->full_name(), std::move(new_products)); + + // Store in cache + cache_.emplace(index_hash, store); + + message const new_msg{store, msg.eom, msg.id}; + stay_in_graph.try_put(new_msg); + to_output.try_put(new_msg); + flag_for(msg.store->id()->hash()).mark_as_processed(); } - auto result = std::invoke(ft, *msg.store->id()); - ++calls_; - - products new_products; - // Add all adds all products; we should only have one. Fix this later. - new_products.add_all(output_, std::move(result)); - auto store = std::make_shared( - msg.store->id(), this->full_name(), stage::process, std::move(new_products)); - - message const new_msg{store, msg.eom, msg.id}; - stay_in_graph.try_put(new_msg); - to_output.try_put(new_msg); + if (done_with(msg.store)) { + cache_.erase(msg.store->id()->hash()); + } }} { spdlog::debug( "Created provider node {} making output {}", this->full_name(), output.to_string()); } + ~provider_node() { report_cached_stores(cache_); } + tbb::flow::receiver& receiver() { return provider_; } tbb::flow::receiver& port_for(product_query const&) override { return provider_; } @@ -101,6 +129,7 @@ namespace phlex::experimental { product_specifications output_; tbb::flow::multifunction_node> provider_; std::atomic calls_; + stores_t cache_; }; } diff --git a/phlex/core/declared_transform.hpp b/phlex/core/declared_transform.hpp index 7db3ccd3..f6a58493 100644 --- a/phlex/core/declared_transform.hpp +++ b/phlex/core/declared_transform.hpp @@ -105,7 +105,7 @@ namespace phlex::experimental { products new_products; new_products.add_all(output_, std::move(result)); a->second = std::make_shared( - store->id(), this->full_name(), stage::process, std::move(new_products)); + store->id(), this->full_name(), std::move(new_products)); message const new_msg{a->second, msg.eom, message_id}; stay_in_graph.try_put(new_msg); diff --git a/phlex/core/declared_unfold.cpp b/phlex/core/declared_unfold.cpp index 35d0f949..b291b89e 100644 --- a/phlex/core/declared_unfold.cpp +++ b/phlex/core/declared_unfold.cpp @@ -20,8 +20,7 @@ namespace phlex::experimental { { auto child_index = parent_->id()->make_child(i, child_layer_name_); ++child_counts_[child_index->layer_hash()]; - return std::make_shared( - child_index, node_name_, stage::process, std::move(new_products)); + return std::make_shared(child_index, node_name_, std::move(new_products)); } product_store_const_ptr generator::flush_store() const diff --git a/phlex/model/product_store.cpp b/phlex/model/product_store.cpp index 74f44355..2e6fc522 100644 --- a/phlex/model/product_store.cpp +++ b/phlex/model/product_store.cpp @@ -8,8 +8,8 @@ namespace phlex::experimental { product_store::product_store(data_cell_index_ptr id, std::string source, - stage processing_stage, - products new_products) : + products new_products, + stage processing_stage) : products_{std::move(new_products)}, id_{std::move(id)}, source_{std::move(source)}, @@ -26,7 +26,7 @@ namespace phlex::experimental { product_store_ptr product_store::make_flush() const { - return product_store_ptr{new product_store{id_, "[inserted]", stage::flush}}; + return product_store_ptr{new product_store{id_, "[inserted]", {}, stage::flush}}; } std::string const& product_store::layer_name() const noexcept { return id_->layer_name(); } diff --git a/phlex/model/product_store.hpp b/phlex/model/product_store.hpp index 3af653f2..967e8430 100644 --- a/phlex/model/product_store.hpp +++ b/phlex/model/product_store.hpp @@ -18,8 +18,8 @@ namespace phlex::experimental { public: explicit product_store(data_cell_index_ptr id, std::string source = "Source", - stage processing_stage = stage::process, - products new_products = {}); + products new_products = {}, + stage processing_stage = stage::process); ~product_store(); static product_store_ptr base(std::string base_name = "Source"); diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index a4b95182..dbfcee50 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -70,8 +70,7 @@ TEST_CASE("Testing families", "[data model]") CHECK(g.execution_counts("rs") == 1ull); CHECK(g.execution_counts("rse") == 1ull); - // FIXME: Need to improve the synchronization to supply strict equality - CHECK(g.execution_counts("run_id_provider") >= 1ull); - CHECK(g.execution_counts("subrun_id_provider") >= 1ull); + CHECK(g.execution_counts("run_id_provider") == 1ull); + CHECK(g.execution_counts("subrun_id_provider") == 1ull); CHECK(g.execution_counts("event_id_provider") == 1ull); } From f8d019606b3305b13102868a469b546454c33835 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 13:28:32 -0600 Subject: [PATCH 42/49] Allow head ports to be empty for driver-only jobs --- phlex/core/edge_maker.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/phlex/core/edge_maker.hpp b/phlex/core/edge_maker.hpp index 4b22c0ea..f837231f 100644 --- a/phlex/core/edge_maker.hpp +++ b/phlex/core/edge_maker.hpp @@ -104,6 +104,11 @@ namespace phlex::experimental { // figure out what provider nodes are needed. // For now, we take as input a mapping of declared_providers. + if (head_ports.empty()) { + // This can happen for jobs that only execute the driver, which is helpful for debugging + return; + } + auto provider_input_ports = make_provider_edges(std::move(head_ports), providers); multi.finalize(std::move(provider_input_ports)); } From 6242bd8aba1b5d91058145c550d3fbbf125a6bb8 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 14:58:09 -0600 Subject: [PATCH 43/49] Simplify declared_provider class --- phlex/core/declared_provider.cpp | 11 ++++++++-- phlex/core/declared_provider.hpp | 35 ++++++++++++++------------------ phlex/core/edge_maker.cpp | 13 ++++++------ 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/phlex/core/declared_provider.cpp b/phlex/core/declared_provider.cpp index 15aec3f3..bb94ee5d 100644 --- a/phlex/core/declared_provider.cpp +++ b/phlex/core/declared_provider.cpp @@ -1,13 +1,20 @@ #include "phlex/core/declared_provider.hpp" namespace phlex::experimental { - declared_provider::declared_provider(algorithm_name name, product_queries output_products) : - products_consumer{std::move(name), {}, std::move(output_products)} + declared_provider::declared_provider(algorithm_name name, product_query output_product) : + name_{std::move(name)}, output_product_{std::move(output_product)} { } declared_provider::~declared_provider() = default; + std::string declared_provider::full_name() const { return name_.full(); } + + product_query const& declared_provider::output_product() const noexcept + { + return output_product_; + } + void declared_provider::report_cached_stores(stores_t const& stores) const { if (stores.size() > 0ull) { diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index 415f256d..2ea46568 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -4,7 +4,6 @@ #include "phlex/core/concepts.hpp" #include "phlex/core/fwd.hpp" #include "phlex/core/message.hpp" -#include "phlex/core/products_consumer.hpp" #include "phlex/core/store_counters.hpp" #include "phlex/metaprogramming/type_deduction.hpp" #include "phlex/model/algorithm_name.hpp" @@ -27,14 +26,16 @@ namespace phlex::experimental { - class declared_provider : public products_consumer { + class declared_provider { public: - declared_provider(algorithm_name name, product_queries output_products); + declared_provider(algorithm_name name, product_query output_product); virtual ~declared_provider(); + std::string full_name() const; + product_query const& output_product() const noexcept; + + virtual tbb::flow::receiver* input_port() = 0; virtual tbb::flow::sender& sender() = 0; - virtual tbb::flow::sender& to_output() = 0; - virtual product_specifications const& output() const = 0; virtual std::size_t num_calls() const = 0; protected: @@ -42,6 +43,10 @@ namespace phlex::experimental { using const_accessor = stores_t::const_accessor; void report_cached_stores(stores_t const& stores) const; + + private: + algorithm_name name_; + product_query output_product_; }; using declared_provider_ptr = std::unique_ptr; @@ -52,20 +57,16 @@ namespace phlex::experimental { template class provider_node : public declared_provider, private detect_flush_flag { using function_t = typename AlgorithmBits::bound_type; - static constexpr auto M = number_output_objects; public: using node_ptr_type = declared_provider_ptr; - // We are setting number_output_products to 0 because that will allow the - // registration mechanism to work, for now. - static constexpr auto number_output_products = 0; provider_node(algorithm_name name, std::size_t concurrency, tbb::flow::graph& g, AlgorithmBits alg, product_query output) : - declared_provider{std::move(name), {output}}, + declared_provider{std::move(name), output}, output_{output.name}, provider_{ g, concurrency, [this, ft = alg.release_algorithm()](message const& msg, auto& output) { @@ -90,8 +91,7 @@ namespace phlex::experimental { ++calls_; products new_products; - // Add all adds all products; we should only have one. Fix this later. - new_products.add_all(output_, std::move(result)); + new_products.add(output_.name(), std::move(result)); auto store = std::make_shared( msg.store->id(), this->full_name(), std::move(new_products)); @@ -115,18 +115,13 @@ namespace phlex::experimental { ~provider_node() { report_cached_stores(cache_); } - tbb::flow::receiver& receiver() { return provider_; } - - tbb::flow::receiver& port_for(product_query const&) override { return provider_; } - std::vector*> ports() override { return {&provider_}; } - private: + tbb::flow::receiver* input_port() override { return &provider_; } tbb::flow::sender& sender() override { return output_port<0>(provider_); } - tbb::flow::sender& to_output() override { return output_port<1>(provider_); } - product_specifications const& output() const override { return output_; } std::size_t num_calls() const final { return calls_.load(); } - product_specifications output_; + + product_specification output_; tbb::flow::multifunction_node> provider_; std::atomic calls_; stores_t cache_; diff --git a/phlex/core/edge_maker.cpp b/phlex/core/edge_maker.cpp index cb99fc11..7dc44796 100644 --- a/phlex/core/edge_maker.cpp +++ b/phlex/core/edge_maker.cpp @@ -20,15 +20,14 @@ namespace phlex::experimental { for (auto const& [_, p] : providers) { auto& provider = *p; // FIXME: The check should probably be more robust. Right now, the - // product_specification buried in the provider.input()[0] call does not - // have its type set, which prevents us from doing a simpler comparison - // (e.g., port.product_label == provider.input()[0]). - if (port.product_label.name.full() == provider.input()[0].name.full() && - port.product_label.layer == provider.input()[0].layer) { - assert(provider.ports().size() == 1); + // product_specification buried in the provider.output_product() call + // does not have its type set, which prevents us from doing a simpler + // comparison (e.g., port.product_label == provider.output_product()). + if (port.product_label.name.full() == provider.output_product().name.full() && + port.product_label.layer == provider.output_product().layer) { auto it = result.find(provider.full_name()); if (it == result.cend()) { - result.try_emplace(provider.full_name(), port.product_label, provider.ports()[0]); + result.try_emplace(provider.full_name(), port.product_label, provider.input_port()); } spdlog::debug("Connecting provider {} to node {} (product: {})", provider.full_name(), From 589d19a462c95c2c2d2723822d1161ffac5dfe2f Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 15:43:07 -0600 Subject: [PATCH 44/49] Some cleanups; and a typo fix --- test/demo-giantdata/log_record.hpp | 1 - test/demo-giantdata/unfold_transform_fold.cpp | 2 +- test/provider_test.cpp | 34 ++----------------- 3 files changed, 4 insertions(+), 33 deletions(-) delete mode 100644 test/demo-giantdata/log_record.hpp diff --git a/test/demo-giantdata/log_record.hpp b/test/demo-giantdata/log_record.hpp deleted file mode 100644 index b48b814b..00000000 --- a/test/demo-giantdata/log_record.hpp +++ /dev/null @@ -1 +0,0 @@ -#error DO NOT INCLUDE THIS FILE \ No newline at end of file diff --git a/test/demo-giantdata/unfold_transform_fold.cpp b/test/demo-giantdata/unfold_transform_fold.cpp index 7f45e7a2..ce1949ff 100644 --- a/test/demo-giantdata/unfold_transform_fold.cpp +++ b/test/demo-giantdata/unfold_transform_fold.cpp @@ -94,7 +94,7 @@ int main(int argc, char* argv[]) spill_index.parent()->number(), spill_index.number()); }) - .output_product("wget"_in("spill")); + .output_product("wgen"_in("spill")); g.unfold( "WaveformGenerator", diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 00c41363..c9d55c3f 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -52,35 +52,7 @@ TEST_CASE("provider_test") .input_family("happy_vertices"_in("spill")) .output_products("vertex_data"); - spdlog::info("graph: is about to be executed"); - try { - g.execute(); - CHECK(g.execution_counts("passer") == max_events); - CHECK(g.execution_counts("my_name_here") == max_events); - } catch (std::exception const& e) { - spdlog::error("Exception during graph execution: {}", e.what()); - throw; - } catch (...) { - spdlog::error("Unknown exception during graph execution"); - throw; - } + g.execute(); + CHECK(g.execution_counts("passer") == max_events); + CHECK(g.execution_counts("my_name_here") == max_events); } - -/* - -Planned development flow: - -[x] Get initial test working. -[x] Make the data product be a simple struct not a fundamental type. -[x] Introduce stub `provider` that takes product_store_ptr as input and returns a product_store_ptr. - Wire the provider into the graph. -[ ] Modify the `provider` to take data_cell_index as input. - The `multiplexer` will then need to emit data_cell_index, stripped from the `product_store` it currently returns. -[ ] Modify the `multiplexer` to accept data_cell_index as input. - The Input will then need to emit data_cell_index, stripped from the `product_store` it currently returns. -[ ] Modify `Input` to accept data_cell_index is input. - This will mean `framework_driver` will have to emit data_cell_index instead of product_store. - -Then we're done. - -*/ From 40cccf4aba9d45e2be920fb46282052b94b68438 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 16:01:14 -0600 Subject: [PATCH 45/49] Remove trailing whitespace Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- test/benchmarks/benchmark-01.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmarks/benchmark-01.jsonnet b/test/benchmarks/benchmark-01.jsonnet index 62433154..8fc7f65f 100644 --- a/test/benchmarks/benchmark-01.jsonnet +++ b/test/benchmarks/benchmark-01.jsonnet @@ -10,5 +10,5 @@ provider: { plugin: 'benchmarks_provider' } - }, + }, } From 46e0bdd03c85fae89ded4ca6c214900726d2abaa Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 16:48:07 -0600 Subject: [PATCH 46/49] Some renaming --- phlex/core/declared_provider.hpp | 2 +- phlex/core/detail/filter_impl.cpp | 2 +- phlex/core/edge_creation_policy.cpp | 2 +- phlex/core/input_arguments.hpp | 4 ++-- phlex/core/message.cpp | 2 +- phlex/core/product_query.cpp | 14 +++++++------- phlex/core/product_query.hpp | 18 +++++++++--------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/phlex/core/declared_provider.hpp b/phlex/core/declared_provider.hpp index 2ea46568..c15c9e18 100644 --- a/phlex/core/declared_provider.hpp +++ b/phlex/core/declared_provider.hpp @@ -67,7 +67,7 @@ namespace phlex::experimental { AlgorithmBits alg, product_query output) : declared_provider{std::move(name), output}, - output_{output.name}, + output_{output.spec}, provider_{ g, concurrency, [this, ft = alg.release_algorithm()](message const& msg, auto& output) { auto& [stay_in_graph, to_output] = output; diff --git a/phlex/core/detail/filter_impl.cpp b/phlex/core/detail/filter_impl.cpp index 459e78e4..001afe7b 100644 --- a/phlex/core/detail/filter_impl.cpp +++ b/phlex/core/detail/filter_impl.cpp @@ -78,7 +78,7 @@ namespace phlex::experimental { // Fill slots in the order of the input arguments to the downstream node. for (std::size_t i = 0; i != nargs_; ++i) { - if (elem[i] or not store->contains_product((*product_names_)[i].name.full())) + if (elem[i] or not store->contains_product((*product_names_)[i].spec.full())) continue; elem[i] = store; } diff --git a/phlex/core/edge_creation_policy.cpp b/phlex/core/edge_creation_policy.cpp index bc1922b8..136d0b57 100644 --- a/phlex/core/edge_creation_policy.cpp +++ b/phlex/core/edge_creation_policy.cpp @@ -9,7 +9,7 @@ namespace phlex::experimental { edge_creation_policy::named_output_port const* edge_creation_policy::find_producer( product_query const& query) const { - auto const& spec = query.name; + auto const& spec = query.spec; auto [b, e] = producers_.equal_range(spec.name()); if (b == e) { spdlog::debug( diff --git a/phlex/core/input_arguments.hpp b/phlex/core/input_arguments.hpp index 9e4b7038..c31ea791 100644 --- a/phlex/core/input_arguments.hpp +++ b/phlex/core/input_arguments.hpp @@ -15,11 +15,11 @@ namespace phlex::experimental { template struct retriever { using handle_arg_t = detail::handle_value_type; - product_query label; + product_query query; auto retrieve(auto const& messages) const { return std::get(messages).store->template get_handle( - label.name.name()); + query.spec.name()); } }; diff --git a/phlex/core/message.cpp b/phlex/core/message.cpp index 3e493843..eaa2d8c6 100644 --- a/phlex/core/message.cpp +++ b/phlex/core/message.cpp @@ -26,7 +26,7 @@ namespace phlex::experimental { auto const [b, e] = std::tuple{cbegin(product_labels), cend(product_labels)}; auto it = std::find(b, e, product_label); if (it == e) { - throw std::runtime_error("Algorithm does not accept product '" + product_label.name.name() + + throw std::runtime_error("Algorithm does not accept product '" + product_label.spec.name() + "'."); } return std::distance(b, it); diff --git a/phlex/core/product_query.cpp b/phlex/core/product_query.cpp index 08162e53..14fad88e 100644 --- a/phlex/core/product_query.cpp +++ b/phlex/core/product_query.cpp @@ -12,15 +12,15 @@ namespace phlex::experimental { if (data_layer.empty()) { throw std::runtime_error("Cannot specify the empty string as a data layer."); } - return {std::move(name), std::move(data_layer)}; + return {std::move(spec), std::move(data_layer)}; } std::string product_query::to_string() const { if (layer.empty()) { - return name.full(); + return spec.full(); } - return fmt::format("{} ϵ {}", name.full(), layer); + return fmt::format("{} ϵ {}", spec.full(), layer); } product_tag operator""_in(char const* product_name, std::size_t length) @@ -33,19 +33,19 @@ namespace phlex::experimental { bool operator==(product_query const& a, product_query const& b) { - return std::tie(a.name, a.layer) == std::tie(b.name, b.layer); + return std::tie(a.spec, a.layer) == std::tie(b.spec, b.layer); } bool operator!=(product_query const& a, product_query const& b) { return !(a == b); } bool operator<(product_query const& a, product_query const& b) { - return std::tie(a.name, a.layer) < std::tie(b.name, b.layer); + return std::tie(a.spec, a.layer) < std::tie(b.spec, b.layer); } - std::ostream& operator<<(std::ostream& os, product_query const& label) + std::ostream& operator<<(std::ostream& os, product_query const& query) { - os << label.to_string(); + os << query.to_string(); return os; } } diff --git a/phlex/core/product_query.hpp b/phlex/core/product_query.hpp index b781c334..81a79e4a 100644 --- a/phlex/core/product_query.hpp +++ b/phlex/core/product_query.hpp @@ -1,5 +1,5 @@ -#ifndef PHLEX_CORE_SPECIFIED_LABEL_HPP -#define PHLEX_CORE_SPECIFIED_LABEL_HPP +#ifndef PHLEX_CORE_PRODUCT_QUERY_HPP +#define PHLEX_CORE_PRODUCT_QUERY_HPP #include "phlex/model/product_specification.hpp" @@ -11,26 +11,26 @@ namespace phlex::experimental { struct product_query { - product_specification name; + product_specification spec; std::string layer; std::string to_string() const; }; struct product_tag { - product_specification name; + product_specification spec; product_query operator()(std::string layer) &&; }; using product_queries = std::vector; - inline auto& to_name(product_query const& label) { return label.name.name(); } - inline auto& to_layer(product_query& label) { return label.layer; } + inline auto& to_name(product_query const& query) { return query.spec.name(); } + inline auto& to_layer(product_query& query) { return query.layer; } product_tag operator""_in(char const* str, std::size_t); bool operator==(product_query const& a, product_query const& b); bool operator!=(product_query const& a, product_query const& b); bool operator<(product_query const& a, product_query const& b); - std::ostream& operator<<(std::ostream& os, product_query const& label); + std::ostream& operator<<(std::ostream& os, product_query const& query); namespace detail { // C is a container of product_queries @@ -46,7 +46,7 @@ namespace phlex::experimental { template void set_type(C& container) { - container.at(index_).name.set_type(make_type_id()); + container.at(index_).spec.set_type(make_type_id()); ++index_; } @@ -69,4 +69,4 @@ namespace phlex::experimental { } } -#endif // PHLEX_CORE_SPECIFIED_LABEL_HPP +#endif // PHLEX_CORE_PRODUCT_QUERY_HPP From 92c09c9e5ccac7b328b3ed1f2ee2ec2b85a3f828 Mon Sep 17 00:00:00 2001 From: Kyle Knoepfel Date: Wed, 10 Dec 2025 16:48:17 -0600 Subject: [PATCH 47/49] Set type of provided product --- phlex/core/edge_maker.cpp | 7 +------ phlex/core/registration_api.hpp | 7 +++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/phlex/core/edge_maker.cpp b/phlex/core/edge_maker.cpp index 7dc44796..9f9e3362 100644 --- a/phlex/core/edge_maker.cpp +++ b/phlex/core/edge_maker.cpp @@ -19,12 +19,7 @@ namespace phlex::experimental { bool found_match = false; for (auto const& [_, p] : providers) { auto& provider = *p; - // FIXME: The check should probably be more robust. Right now, the - // product_specification buried in the provider.output_product() call - // does not have its type set, which prevents us from doing a simpler - // comparison (e.g., port.product_label == provider.output_product()). - if (port.product_label.name.full() == provider.output_product().name.full() && - port.product_label.layer == provider.output_product().layer) { + if (port.product_label == provider.output_product()) { auto it = result.find(provider.full_name()); if (it == result.cend()) { result.try_emplace(provider.full_name(), port.product_label, provider.input_port()); diff --git a/phlex/core/registration_api.hpp b/phlex/core/registration_api.hpp index 14a23d15..da764d09 100644 --- a/phlex/core/registration_api.hpp +++ b/phlex/core/registration_api.hpp @@ -111,8 +111,6 @@ namespace phlex::experimental { template class provider_api { - using provider_type = provider_node; - public: provider_api(configuration const* config, std::string name, @@ -132,6 +130,11 @@ namespace phlex::experimental { auto output_product(product_query output) { + using return_type = return_type; + using provider_type = provider_node; + + output.spec.set_type(make_type_id()); + registrar_.set_creator( [this, output = std::move(output)](auto /* predicates */, auto /* output_products */) { return std::make_unique( From fb945c07a9892cd64d003a8b05749981d4ae2839 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Dec 2025 15:56:32 +0000 Subject: [PATCH 48/49] Initial plan From cf2a0dacf86d46fde75fd30163e65ad947e82a4d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Dec 2025 16:30:08 +0000 Subject: [PATCH 49/49] Add test coverage for provider flush message handling Co-authored-by: greenc-FNAL <2372949+greenc-FNAL@users.noreply.github.com> --- test/provider_test.cpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index c9d55c3f..3e7b2591 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -56,3 +56,39 @@ TEST_CASE("provider_test") CHECK(g.execution_counts("passer") == max_events); CHECK(g.execution_counts("my_name_here") == max_events); } + +TEST_CASE("provider_with_flush") +{ + // Test that providers handle flush messages correctly + // Flush messages should pass through providers without triggering execution + constexpr auto max_events{2u}; + spdlog::flush_on(spdlog::level::warn); + + auto levels_to_process = [](framework_driver& driver) { + auto job_index = data_cell_index::base_ptr(); + driver.yield(job_index); + + for (unsigned int i : std::views::iota(1u, max_events + 1)) { + auto spill_index = job_index->make_child(i, "spill"); + driver.yield(spill_index); + + // Send a flush after each spill - this tests the flush handling path + auto flush_store = product_store::base()->make_flush(); + driver.yield(flush_store->id()); + } + }; + + framework_graph g{levels_to_process}; + + g.provide("flush_aware_provider", give_me_vertices, concurrency::unlimited) + .output_product("flush_vertices"_in("spill")); + + g.transform("flush_consumer", pass_on, concurrency::unlimited) + .input_family("flush_vertices"_in("spill")) + .output_products("vertex_data"); + + g.execute(); + // Flush messages should not count as provider executions + CHECK(g.execution_counts("flush_consumer") == max_events); + CHECK(g.execution_counts("flush_aware_provider") == max_events); +}