From d0f19727e3d4218fee04e1843a2fe23a560860f4 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Fri, 26 Nov 2021 15:28:42 +0100 Subject: [PATCH] Expect ptree as input in InfrastructureGenerator --- .../QualityControl/InfrastructureGenerator.h | 81 ++++++++++--------- Framework/src/InfrastructureGenerator.cxx | 50 ++++++------ Framework/src/runAdvanced.cxx | 2 +- Framework/src/runBasic.cxx | 2 +- Framework/src/runQC.cxx | 11 ++- Framework/test/testCheckWorkflow.cxx | 2 +- .../test/testInfrastructureGenerator.cxx | 38 ++++++--- Framework/test/testWorkflow.cxx | 2 +- Modules/ITS/src/runITS.cxx | 2 +- Modules/PHOS/src/runQCPHOSRaw.cxx | 6 +- 10 files changed, 103 insertions(+), 93 deletions(-) diff --git a/Framework/include/QualityControl/InfrastructureGenerator.h b/Framework/include/QualityControl/InfrastructureGenerator.h index a5a1f1abc9..25341f1fda 100644 --- a/Framework/include/QualityControl/InfrastructureGenerator.h +++ b/Framework/include/QualityControl/InfrastructureGenerator.h @@ -19,6 +19,7 @@ #include #include +#include namespace o2::framework { @@ -54,9 +55,9 @@ class InfrastructureGenerator /// Generates a full QC infrastructure from a configuration file. This function is aimed to use for standalone setups /// and local development. It will create both local and remote QC tasks, and CheckRunners running associated Checks. /// - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") + /// \param configurationTree - full path to configuration file, preceded with the backend (e.g. "json://") /// \return generated standalone QC workflow - static framework::WorkflowSpec generateStandaloneInfrastructure(std::string configurationSource); + static framework::WorkflowSpec generateStandaloneInfrastructure(const boost::property_tree::ptree& configurationTree); /// \brief Generates a standalone QC infrastructure. /// @@ -64,18 +65,18 @@ class InfrastructureGenerator /// and local development. It will create both local and remote QC tasks, and CheckRunners running associated Checks. /// /// \param workflow - existing workflow where QC infrastructure should be placed - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") - static void generateStandaloneInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource); + /// \param configurationTree - full QC config ptree + static void generateStandaloneInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree); /// \brief Generates the local part of the QC infrastructure for a specified host. /// /// Generates the local part of the QC infrastructure for a specified host - taskRunners which are declared in the /// configuration to be 'local'. /// - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") + /// \param configurationTree - full QC config ptree /// \param targetHost - name of the machine /// \return generated local QC workflow - static framework::WorkflowSpec generateLocalInfrastructure(std::string configurationSource, std::string targetHost); + static framework::WorkflowSpec generateLocalInfrastructure(const boost::property_tree::ptree& configurationTree, std::string targetHost); /// \brief Generates the local part of the QC infrastructure for a specified host. /// @@ -83,19 +84,19 @@ class InfrastructureGenerator /// configuration to be 'local'. /// /// \param workflow - existing workflow where QC infrastructure should be placed - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") + /// \param configurationTree - full QC config ptree /// \param host - name of the machine /// \return generated local QC workflow - static void generateLocalInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string host); + static void generateLocalInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string host); /// \brief Generates the remote part of the QC infrastructure. /// /// Generates the remote part of the QC infrastructure - mergers and checkers for 'local' tasks and full QC chain for /// 'remote' tasks. /// - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") + /// \param configurationTree - full QC config ptree /// \return generated remote QC workflow - static o2::framework::WorkflowSpec generateRemoteInfrastructure(std::string configurationSource); + static o2::framework::WorkflowSpec generateRemoteInfrastructure(const boost::property_tree::ptree& configurationTree); /// \brief Generates the remote part of the QC infrastructure. /// @@ -103,44 +104,44 @@ class InfrastructureGenerator /// 'remote' tasks. /// /// \param workflow - existing workflow where QC infrastructure should be placed - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") - static void generateRemoteInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource); + /// \param configurationTree - full QC config ptree + static void generateRemoteInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree); /// \brief Generates the local batch part of the QC infrastructure. /// /// Generates the local batch part of the QC infrastructure - tasks and a file sink/merger. /// /// \param workflow - existing workflow where QC infrastructure should be placed - /// \param configurationSource - full path to configuration file, preceded with the backend (e.g. "json://") + /// \param configurationTree - full QC config ptree /// \param sinkFilePath - path to the output file - static void generateLocalBatchInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string sinkFilePath); + static void generateLocalBatchInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string sinkFilePath); /// \brief Generates the local batch part of the QC infrastructure. /// /// Generates the local batch part of the QC infrastructure - tasks and a file sink/merger. /// - /// \param configurationSource - full path to configuration file, preceded with the backend (f.e. "json://") + /// \param configurationTree - full QC config ptree /// \param sinkFilePath - path to the output file /// \return generated local QC workflow - static framework::WorkflowSpec generateLocalBatchInfrastructure(std::string configurationSource, std::string sinkFilePath); + static framework::WorkflowSpec generateLocalBatchInfrastructure(const boost::property_tree::ptree& configurationTree, std::string sinkFilePath); /// \brief Generates the remote batch part of the QC infrastructure. /// /// Generates the remote batch part of the QC infrastructure - file reader, check runners, aggregator runners. /// /// \param workflow - existing workflow where QC infrastructure should be placed - /// \param configurationSource - full path to configuration file, preceded with the backend (f.e. "json://") + /// \param configurationTree - full QC config ptree /// \param sourceFilePath - path to the input file - static void generateRemoteBatchInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string sourceFilePath); + static void generateRemoteBatchInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string sourceFilePath); /// \brief Generates the remote batch part of the QC infrastructure. /// /// Generates the remote batch part of the QC infrastructure - file reader, check runners, aggregator runners. /// - /// \param configurationSource - full path to configuration file, preceded with the backend (f.e. "json://") + /// \param configurationTree - full QC config ptree /// \param sourceFilePath - path to the input file /// \return generated remote batch QC workflow - static framework::WorkflowSpec generateRemoteBatchInfrastructure(std::string configurationSource, std::string sourceFilePath); + static framework::WorkflowSpec generateRemoteBatchInfrastructure(const boost::property_tree::ptree& configurationTree, std::string sourceFilePath); /// \brief Provides necessary customization of the QC infrastructure. /// @@ -211,54 +212,54 @@ class InfrastructureGenerator // exposing the class above as a main QC interface, syntactic sugar -inline framework::WorkflowSpec generateStandaloneInfrastructure(std::string configurationSource) +inline framework::WorkflowSpec generateStandaloneInfrastructure(const boost::property_tree::ptree& configurationTree) { - return core::InfrastructureGenerator::generateStandaloneInfrastructure(configurationSource); + return core::InfrastructureGenerator::generateStandaloneInfrastructure(configurationTree); } -inline void generateStandaloneInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource) +inline void generateStandaloneInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree) { - core::InfrastructureGenerator::generateStandaloneInfrastructure(workflow, configurationSource); + core::InfrastructureGenerator::generateStandaloneInfrastructure(workflow, configurationTree); } -inline framework::WorkflowSpec generateLocalInfrastructure(std::string configurationSource, std::string host) +inline framework::WorkflowSpec generateLocalInfrastructure(const boost::property_tree::ptree& configurationTree, std::string host) { - return core::InfrastructureGenerator::generateLocalInfrastructure(configurationSource, host); + return core::InfrastructureGenerator::generateLocalInfrastructure(configurationTree, host); } -inline void generateLocalInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string host) +inline void generateLocalInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string host) { - core::InfrastructureGenerator::generateLocalInfrastructure(workflow, configurationSource, host); + core::InfrastructureGenerator::generateLocalInfrastructure(workflow, configurationTree, host); } -inline framework::WorkflowSpec generateRemoteInfrastructure(std::string configurationSource) +inline framework::WorkflowSpec generateRemoteInfrastructure(const boost::property_tree::ptree& configurationTree) { - return core::InfrastructureGenerator::generateRemoteInfrastructure(configurationSource); + return core::InfrastructureGenerator::generateRemoteInfrastructure(configurationTree); } -inline framework::WorkflowSpec generateLocalBatchInfrastructure(std::string configurationSource, std::string sinkFilePath) +inline framework::WorkflowSpec generateLocalBatchInfrastructure(const boost::property_tree::ptree& configurationTree, std::string sinkFilePath) { - return core::InfrastructureGenerator::generateLocalBatchInfrastructure(configurationSource, std::move(sinkFilePath)); + return core::InfrastructureGenerator::generateLocalBatchInfrastructure(configurationTree, std::move(sinkFilePath)); } -inline void generateLocalBatchInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string sinkFilePath) +inline void generateLocalBatchInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string sinkFilePath) { - core::InfrastructureGenerator::generateLocalBatchInfrastructure(workflow, configurationSource, std::move(sinkFilePath)); + core::InfrastructureGenerator::generateLocalBatchInfrastructure(workflow, configurationTree, std::move(sinkFilePath)); } -inline framework::WorkflowSpec generateRemoteBatchInfrastructure(std::string configurationSource, std::string sourceFilePath) +inline framework::WorkflowSpec generateRemoteBatchInfrastructure(const boost::property_tree::ptree& configurationTree, std::string sourceFilePath) { - return core::InfrastructureGenerator::generateRemoteBatchInfrastructure(configurationSource, std::move(sourceFilePath)); + return core::InfrastructureGenerator::generateRemoteBatchInfrastructure(configurationTree, std::move(sourceFilePath)); } -inline void generateRemoteBatchInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string sourceFilePath) +inline void generateRemoteBatchInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string sourceFilePath) { - core::InfrastructureGenerator::generateRemoteBatchInfrastructure(workflow, configurationSource, std::move(sourceFilePath)); + core::InfrastructureGenerator::generateRemoteBatchInfrastructure(workflow, configurationTree, std::move(sourceFilePath)); } -inline void generateRemoteInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource) +inline void generateRemoteInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree) { - core::InfrastructureGenerator::generateRemoteInfrastructure(workflow, configurationSource); + core::InfrastructureGenerator::generateRemoteInfrastructure(workflow, configurationTree); } inline void customizeInfrastructure(std::vector& policies) diff --git a/Framework/src/InfrastructureGenerator.cxx b/Framework/src/InfrastructureGenerator.cxx index 9d2c91aa01..90b2fe011d 100644 --- a/Framework/src/InfrastructureGenerator.cxx +++ b/Framework/src/InfrastructureGenerator.cxx @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -71,12 +72,11 @@ struct DataSamplingPolicySpec { std::string remoteMachine; }; -framework::WorkflowSpec InfrastructureGenerator::generateStandaloneInfrastructure(std::string configurationSource) +framework::WorkflowSpec InfrastructureGenerator::generateStandaloneInfrastructure(const boost::property_tree::ptree& configurationTree) { printVersion(); - auto config = ConfigurationFactory::getConfiguration(configurationSource); - auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(config->getRecursive()); + auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(configurationTree); // todo: report the number of tasks/checks/etc once all are read there. WorkflowSpec workflow; @@ -97,18 +97,17 @@ framework::WorkflowSpec InfrastructureGenerator::generateStandaloneInfrastructur return workflow; } -void InfrastructureGenerator::generateStandaloneInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource) +void InfrastructureGenerator::generateStandaloneInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree) { - auto qcInfrastructure = InfrastructureGenerator::generateStandaloneInfrastructure(configurationSource); + auto qcInfrastructure = InfrastructureGenerator::generateStandaloneInfrastructure(configurationTree); workflow.insert(std::end(workflow), std::begin(qcInfrastructure), std::end(qcInfrastructure)); } -WorkflowSpec InfrastructureGenerator::generateLocalInfrastructure(std::string configurationSource, std::string targetHost) +WorkflowSpec InfrastructureGenerator::generateLocalInfrastructure(const boost::property_tree::ptree& configurationTree, std::string targetHost) { printVersion(); - auto config = ConfigurationFactory::getConfiguration(configurationSource); - auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(config->getRecursive()); + auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(configurationTree); WorkflowSpec workflow; std::set samplingPoliciesForRemoteTasks; @@ -158,7 +157,7 @@ WorkflowSpec InfrastructureGenerator::generateLocalInfrastructure(std::string co } if (!samplingPoliciesForRemoteTasks.empty()) { - auto dataSamplingTree = config->getRecursive("dataSamplingPolicies"); + auto dataSamplingTree = configurationTree.get_child("dataSamplingPolicies"); // Creating Data Sampling Policies proxies for (const auto& [policyName, control, remoteMachine] : samplingPoliciesForRemoteTasks) { std::string port = std::to_string(DataSampling::PortForPolicy(dataSamplingTree, policyName).value_or(defaultPolicyPort)); @@ -178,18 +177,17 @@ WorkflowSpec InfrastructureGenerator::generateLocalInfrastructure(std::string co return workflow; } -void InfrastructureGenerator::generateLocalInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string host) +void InfrastructureGenerator::generateLocalInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string host) { - auto qcInfrastructure = InfrastructureGenerator::generateLocalInfrastructure(configurationSource, host); + auto qcInfrastructure = InfrastructureGenerator::generateLocalInfrastructure(configurationTree, host); workflow.insert(std::end(workflow), std::begin(qcInfrastructure), std::end(qcInfrastructure)); } -o2::framework::WorkflowSpec InfrastructureGenerator::generateRemoteInfrastructure(std::string configurationSource) +o2::framework::WorkflowSpec InfrastructureGenerator::generateRemoteInfrastructure(const boost::property_tree::ptree& configurationTree) { printVersion(); - auto config = ConfigurationFactory::getConfiguration(configurationSource); - auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(config->getRecursive()); + auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(configurationTree); WorkflowSpec workflow; std::set samplingPoliciesForRemoteTasks; @@ -234,7 +232,7 @@ o2::framework::WorkflowSpec InfrastructureGenerator::generateRemoteInfrastructur } if (!samplingPoliciesForRemoteTasks.empty()) { - auto dataSamplingTree = config->getRecursive("dataSamplingPolicies"); + auto dataSamplingTree = configurationTree.get_child("dataSamplingPolicies"); // Creating Data Sampling Policies proxies for (const auto& [policyName, control, remoteMachine] : samplingPoliciesForRemoteTasks) { (void)remoteMachine; @@ -260,18 +258,17 @@ o2::framework::WorkflowSpec InfrastructureGenerator::generateRemoteInfrastructur return workflow; } -void InfrastructureGenerator::generateRemoteInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource) +void InfrastructureGenerator::generateRemoteInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree) { - auto qcInfrastructure = InfrastructureGenerator::generateRemoteInfrastructure(configurationSource); + auto qcInfrastructure = InfrastructureGenerator::generateRemoteInfrastructure(configurationTree); workflow.insert(std::end(workflow), std::begin(qcInfrastructure), std::end(qcInfrastructure)); } -framework::WorkflowSpec InfrastructureGenerator::generateLocalBatchInfrastructure(std::string configurationSource, std::string sinkFilePath) +framework::WorkflowSpec InfrastructureGenerator::generateLocalBatchInfrastructure(const boost::property_tree::ptree& configurationTree, std::string sinkFilePath) { printVersion(); - auto config = ConfigurationFactory::getConfiguration(configurationSource); - auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(config->getRecursive()); + auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(configurationTree); std::vector fileSinkInputs; WorkflowSpec workflow; @@ -301,18 +298,17 @@ framework::WorkflowSpec InfrastructureGenerator::generateLocalBatchInfrastructur return workflow; } -void InfrastructureGenerator::generateLocalBatchInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string sinkFilePath) +void InfrastructureGenerator::generateLocalBatchInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string sinkFilePath) { - auto qcInfrastructure = InfrastructureGenerator::generateLocalBatchInfrastructure(std::move(configurationSource), std::move(sinkFilePath)); + auto qcInfrastructure = InfrastructureGenerator::generateLocalBatchInfrastructure(std::move(configurationTree), std::move(sinkFilePath)); workflow.insert(std::end(workflow), std::begin(qcInfrastructure), std::end(qcInfrastructure)); } -framework::WorkflowSpec InfrastructureGenerator::generateRemoteBatchInfrastructure(std::string configurationSource, std::string sourceFilePath) +framework::WorkflowSpec InfrastructureGenerator::generateRemoteBatchInfrastructure(const boost::property_tree::ptree& configurationTree, std::string sourceFilePath) { printVersion(); - auto config = ConfigurationFactory::getConfiguration(configurationSource); - auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(config->getRecursive()); + auto infrastructureSpec = InfrastructureSpecReader::readInfrastructureSpec(configurationTree); WorkflowSpec workflow; @@ -335,9 +331,9 @@ framework::WorkflowSpec InfrastructureGenerator::generateRemoteBatchInfrastructu return workflow; } -void InfrastructureGenerator::generateRemoteBatchInfrastructure(framework::WorkflowSpec& workflow, std::string configurationSource, std::string sourceFilePath) +void InfrastructureGenerator::generateRemoteBatchInfrastructure(framework::WorkflowSpec& workflow, const boost::property_tree::ptree& configurationTree, std::string sourceFilePath) { - auto qcInfrastructure = InfrastructureGenerator::generateRemoteBatchInfrastructure(configurationSource, sourceFilePath); + auto qcInfrastructure = InfrastructureGenerator::generateRemoteBatchInfrastructure(configurationTree, sourceFilePath); workflow.insert(std::end(workflow), std::begin(qcInfrastructure), std::end(qcInfrastructure)); } diff --git a/Framework/src/runAdvanced.cxx b/Framework/src/runAdvanced.cxx index 291d995d7d..f51d7e2cdb 100644 --- a/Framework/src/runAdvanced.cxx +++ b/Framework/src/runAdvanced.cxx @@ -102,7 +102,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config) auto dataSamplingTree = configInterface->getRecursive("dataSamplingPolicies"); DataSampling::GenerateInfrastructure(specs, dataSamplingTree); // Generation of the remote QC topology (for the QC servers) - quality_control::generateStandaloneInfrastructure(specs, qcConfigurationSource); + quality_control::generateStandaloneInfrastructure(specs, configInterface->getRecursive()); } return specs; } diff --git a/Framework/src/runBasic.cxx b/Framework/src/runBasic.cxx index 9d5bd11917..4597f81d50 100644 --- a/Framework/src/runBasic.cxx +++ b/Framework/src/runBasic.cxx @@ -105,7 +105,7 @@ WorkflowSpec defineDataProcessing(const ConfigContext& config) DataSampling::GenerateInfrastructure(specs, dataSamplingTree); // Generation of the QC topology (one task, one checker in this case) - quality_control::generateStandaloneInfrastructure(specs, qcConfigurationSource); + quality_control::generateStandaloneInfrastructure(specs, configInterface->getRecursive()); // Finally the printer if (hasChecks(qcConfigurationSource)) { diff --git a/Framework/src/runQC.cxx b/Framework/src/runQC.cxx index 8c93969c08..5e84052865 100644 --- a/Framework/src/runQC.cxx +++ b/Framework/src/runQC.cxx @@ -138,7 +138,6 @@ WorkflowSpec defineDataProcessing(const ConfigContext& config) } auto qcConfigurationSource = config.options().get("config"); - try { // The online QC infrastructure is divided into two parts: // - local - QC tasks which are on the same machines as the main processing. We also put Data Sampling there. @@ -174,7 +173,7 @@ WorkflowSpec defineDataProcessing(const ConfigContext& config) } else { ILOG(Info, Support) << "Omitting Data Sampling" << ENDM; } - quality_control::generateStandaloneInfrastructure(specs, qcConfigurationSource); + quality_control::generateStandaloneInfrastructure(specs, configTree); break; } case WorkflowType::Local: { @@ -191,14 +190,14 @@ WorkflowSpec defineDataProcessing(const ConfigContext& config) } // Generation of the local QC topology (local QC tasks and their output proxies) - quality_control::generateLocalInfrastructure(specs, qcConfigurationSource, host); + quality_control::generateLocalInfrastructure(specs, configTree, host); break; } case WorkflowType::Remote: { ILOG(Info, Support) << "Creating a remote QC workflow." << ENDM; // Generation of the remote QC topology (task for QC servers, input proxies, mergers and all check runners) - quality_control::generateRemoteInfrastructure(specs, qcConfigurationSource); + quality_control::generateRemoteInfrastructure(specs, configTree); break; } case WorkflowType::LocalBatch: { @@ -212,13 +211,13 @@ WorkflowSpec defineDataProcessing(const ConfigContext& config) auto localBatchFilePath = config.options().get("local-batch"); // Generation of the local batch QC workflow (QC tasks and file sink) - quality_control::generateLocalBatchInfrastructure(specs, qcConfigurationSource, localBatchFilePath); + quality_control::generateLocalBatchInfrastructure(specs, configTree, localBatchFilePath); break; } case WorkflowType::RemoteBatch: { auto remoteBatchFilePath = config.options().get("remote-batch"); // Creating the remote batch QC topology (file reader, check runners, aggregator runners, postprocessing) - quality_control::generateRemoteBatchInfrastructure(specs, qcConfigurationSource, remoteBatchFilePath); + quality_control::generateRemoteBatchInfrastructure(specs, configTree, remoteBatchFilePath); break; } } diff --git a/Framework/test/testCheckWorkflow.cxx b/Framework/test/testCheckWorkflow.cxx index 99d36ecd23..7560a7a419 100644 --- a/Framework/test/testCheckWorkflow.cxx +++ b/Framework/test/testCheckWorkflow.cxx @@ -148,7 +148,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const&) DataSampling::GenerateInfrastructure(specs, dataSamplingTree); // Generation of the QC topology (one task, one checker in this case) - quality_control::generateStandaloneInfrastructure(specs, qcConfigurationSource); + quality_control::generateStandaloneInfrastructure(specs, configInterface->getRecursive()); Receiver receiver(qcConfigurationSource); // Finally the receiver diff --git a/Framework/test/testInfrastructureGenerator.cxx b/Framework/test/testInfrastructureGenerator.cxx index c08f2c0f3a..fc8b2d4bc7 100644 --- a/Framework/test/testInfrastructureGenerator.cxx +++ b/Framework/test/testInfrastructureGenerator.cxx @@ -24,16 +24,20 @@ #include "getTestDataDirectory.h" #include +#include using namespace o2::quality_control::core; using namespace o2::framework; +using namespace o2::configuration; BOOST_AUTO_TEST_CASE(qc_factory_local_test) { std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testSharedConfig.json"; std::cout << configFilePath << std::endl; + auto configInterface = ConfigurationFactory::getConfiguration(configFilePath); + auto configTree = configInterface->getRecursive(); { - auto workflow = InfrastructureGenerator::generateLocalInfrastructure(configFilePath, "o2flp1"); + auto workflow = InfrastructureGenerator::generateLocalInfrastructure(configTree, "o2flp1"); BOOST_REQUIRE_EQUAL(workflow.size(), 3); @@ -53,7 +57,7 @@ BOOST_AUTO_TEST_CASE(qc_factory_local_test) } { - auto workflow = InfrastructureGenerator::generateLocalInfrastructure(configFilePath, "o2flp2"); + auto workflow = InfrastructureGenerator::generateLocalInfrastructure(configTree, "o2flp2"); BOOST_REQUIRE_EQUAL(workflow.size(), 2); @@ -69,7 +73,7 @@ BOOST_AUTO_TEST_CASE(qc_factory_local_test) } { - auto workflow = InfrastructureGenerator::generateLocalInfrastructure(configFilePath, "o2flp3"); + auto workflow = InfrastructureGenerator::generateLocalInfrastructure(configTree, "o2flp3"); BOOST_REQUIRE_EQUAL(workflow.size(), 0); } @@ -78,7 +82,9 @@ BOOST_AUTO_TEST_CASE(qc_factory_local_test) BOOST_AUTO_TEST_CASE(qc_factory_remote_test) { std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testSharedConfig.json"; - auto workflow = InfrastructureGenerator::generateRemoteInfrastructure(configFilePath); + auto configInterface = ConfigurationFactory::getConfiguration(configFilePath); + auto configTree = configInterface->getRecursive(); + auto workflow = InfrastructureGenerator::generateRemoteInfrastructure(configTree); // the infrastructure should consist of a proxy, merger and checker for the 'skeletonTask' (its taskRunner is declared to be // local) and also taskRunner and checker for the 'abcTask' and 'xyzTask'. @@ -168,7 +174,9 @@ BOOST_AUTO_TEST_CASE(qc_factory_remote_test) BOOST_AUTO_TEST_CASE(qc_factory_standalone_test) { std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testSharedConfig.json"; - auto workflow = InfrastructureGenerator::generateStandaloneInfrastructure(configFilePath); + auto configInterface = ConfigurationFactory::getConfiguration(configFilePath); + auto configTree = configInterface->getRecursive(); + auto workflow = InfrastructureGenerator::generateStandaloneInfrastructure(configTree); // the infrastructure should consist of 3 TaskRunners, 1 PostProcessingRunner, 4 CheckRunners (including one for PP), 1 AggregatorRunner BOOST_REQUIRE_EQUAL(workflow.size(), 9); @@ -230,29 +238,31 @@ BOOST_AUTO_TEST_CASE(qc_factory_standalone_test) BOOST_AUTO_TEST_CASE(qc_factory_empty_config) { std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testEmptyConfig.json"; + auto configInterface = ConfigurationFactory::getConfiguration(configFilePath); + auto configTree = configInterface->getRecursive(); { WorkflowSpec workflow; - BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateStandaloneInfrastructure(workflow, configFilePath)); + BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateStandaloneInfrastructure(workflow, configTree)); BOOST_CHECK_EQUAL(workflow.size(), 0); } { WorkflowSpec workflow; - BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateLocalInfrastructure(workflow, configFilePath, "asdf")); + BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateLocalInfrastructure(workflow, configTree, "asdf")); BOOST_CHECK_EQUAL(workflow.size(), 0); } { WorkflowSpec workflow; - BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateRemoteInfrastructure(workflow, configFilePath)); + BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateRemoteInfrastructure(workflow, configTree)); BOOST_CHECK_EQUAL(workflow.size(), 0); } { WorkflowSpec workflow; - BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateLocalBatchInfrastructure(workflow, configFilePath, "file.root")); + BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateLocalBatchInfrastructure(workflow, configTree, "file.root")); BOOST_CHECK_EQUAL(workflow.size(), 0); } { WorkflowSpec workflow; - BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateRemoteBatchInfrastructure(workflow, configFilePath, "file.root")); + BOOST_REQUIRE_NO_THROW(InfrastructureGenerator::generateRemoteBatchInfrastructure(workflow, configTree, "file.root")); BOOST_CHECK_EQUAL(workflow.size(), 0); } } @@ -261,8 +271,10 @@ BOOST_AUTO_TEST_CASE(qc_infrastructure_local_batch_test) { std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testSharedConfig.json"; std::cout << configFilePath << std::endl; + auto configInterface = ConfigurationFactory::getConfiguration(configFilePath); + auto configTree = configInterface->getRecursive(); { - auto workflow = InfrastructureGenerator::generateLocalBatchInfrastructure(configFilePath, "file.root"); + auto workflow = InfrastructureGenerator::generateLocalBatchInfrastructure(configTree, "file.root"); BOOST_REQUIRE_EQUAL(workflow.size(), 4); @@ -302,7 +314,9 @@ BOOST_AUTO_TEST_CASE(qc_infrastructure_local_batch_test) BOOST_AUTO_TEST_CASE(qc_infrastructure_remote_batch_test) { std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testSharedConfig.json"; - auto workflow = InfrastructureGenerator::generateRemoteBatchInfrastructure(configFilePath, "file.root"); + auto configInterface = ConfigurationFactory::getConfiguration(configFilePath); + auto configTree = configInterface->getRecursive(); + auto workflow = InfrastructureGenerator::generateRemoteBatchInfrastructure(configTree, "file.root"); BOOST_REQUIRE_EQUAL(workflow.size(), 7); diff --git a/Framework/test/testWorkflow.cxx b/Framework/test/testWorkflow.cxx index 21fc424bca..4f1af055cd 100644 --- a/Framework/test/testWorkflow.cxx +++ b/Framework/test/testWorkflow.cxx @@ -68,7 +68,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const&) DataSampling::GenerateInfrastructure(specs, dataSamplingTree); // Generation of the QC topology (one task, one checker in this case) - quality_control::generateStandaloneInfrastructure(specs, qcConfigurationSource); + quality_control::generateStandaloneInfrastructure(specs, configInterface->getRecursive()); // Finally the receiver DataProcessorSpec receiver{ diff --git a/Modules/ITS/src/runITS.cxx b/Modules/ITS/src/runITS.cxx index e8f7333182..cdf9734b07 100644 --- a/Modules/ITS/src/runITS.cxx +++ b/Modules/ITS/src/runITS.cxx @@ -88,7 +88,7 @@ WorkflowSpec defineDataProcessing(const ConfigContext& config) DataSampling::GenerateInfrastructure(specs, dataSamplingTree); // Generation of the QC topology (one task, one checker in this case) - quality_control::generateStandaloneInfrastructure(specs, qcConfigurationSource); + quality_control::generateStandaloneInfrastructure(specs, configInterface->getRecursive()); return specs; } diff --git a/Modules/PHOS/src/runQCPHOSRaw.cxx b/Modules/PHOS/src/runQCPHOSRaw.cxx index 1c47273d25..a1aea97eb9 100644 --- a/Modules/PHOS/src/runQCPHOSRaw.cxx +++ b/Modules/PHOS/src/runQCPHOSRaw.cxx @@ -70,21 +70,21 @@ o2::framework::WorkflowSpec defineDataProcessing(o2::framework::ConfigContext co ILOG(Info, Support) << "To create both local and remote QC topologies, one does not have to add any of '--local' or '--remote' flags." << ENDM; } + auto configInterface = ConfigurationFactory::getConfiguration(qcConfigurationSource); if (config.options().get("local") || !config.options().get("remote")) { LOG(info) << "Local GenerateInfrastructure"; // Generation of Data Sampling infrastructure - auto configInterface = ConfigurationFactory::getConfiguration(qcConfigurationSource); auto dataSamplingTree = configInterface->getRecursive("dataSamplingPolicies"); DataSampling::GenerateInfrastructure(specs, dataSamplingTree); LOG(info) << "Local: generateLocalInfrastructure"; // Generation of the local QC topology (local QC tasks) - o2::quality_control::generateLocalInfrastructure(specs, qcConfigurationSource, config.options().get("host")); + o2::quality_control::generateLocalInfrastructure(specs, configInterface->getRecursive(), config.options().get("host")); } if (config.options().get("remote") || !config.options().get("local")) { // Generation of the remote QC topology (task for QC servers, mergers and all checkers) - o2::quality_control::generateRemoteInfrastructure(specs, qcConfigurationSource); + o2::quality_control::generateRemoteInfrastructure(specs, configInterface->getRecursive()); } LOG(info) << "Done ";