From a5773aca741bc1ffc608ef036d9f32656c598a4f Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 7 Jul 2026 18:15:05 +0200 Subject: [PATCH 1/3] Method to extract ConfigurableParam as JSON string and update from such string --- .../include/CommonUtils/ConfigurableParam.h | 5 + Common/Utils/src/ConfigurableParam.cxx | 98 ++++++++++++++----- Common/Utils/test/testConfigurableParam.cxx | 36 +++++++ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/Common/Utils/include/CommonUtils/ConfigurableParam.h b/Common/Utils/include/CommonUtils/ConfigurableParam.h index 0a6da30aa39f5..fa4acd2744703 100644 --- a/Common/Utils/include/CommonUtils/ConfigurableParam.h +++ b/Common/Utils/include/CommonUtils/ConfigurableParam.h @@ -407,6 +407,8 @@ class ConfigurableParam // writes a human readable INI or JSON file depending on the extension static void write(std::string const& filename, std::string const& keyOnly = ""); + static std::string asJSON(std::string const& keyOnly = ""); + // can be used instead of using API on concrete child classes template static T getValueAs(std::string key) @@ -494,6 +496,9 @@ class ConfigurableParam // be updated, absence of data for any of requested params will lead to fatal static void updateFromFile(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false); + // update from a JSON string with the same filtering semantics as updateFromFile + static void updateFromJSONString(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false); + // interface for use from the CCDB API; allows to sync objects read from CCDB with the information // stored in the registry; modifies given object as well as registry virtual void syncCCDBandRegistry(void* obj) = 0; diff --git a/Common/Utils/src/ConfigurableParam.cxx b/Common/Utils/src/ConfigurableParam.cxx index 202433b04e4c4..4de478a6d0e9f 100644 --- a/Common/Utils/src/ConfigurableParam.cxx +++ b/Common/Utils/src/ConfigurableParam.cxx @@ -37,6 +37,7 @@ #endif #include #include +#include #include #include #include @@ -754,6 +755,29 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const // ------------------------------------------------------------------ +std::string ConfigurableParam::asJSON(std::string const& keyOnly) +{ + initPropertyTree(); // update the boost tree before writing + std::ostringstream os; + if (!keyOnly.empty()) { // write ini for selected key only + try { + boost::property_tree::ptree kTree; + auto keys = o2::utils::Str::tokenize(keyOnly, " ,;", true, true); + for (const auto& k : keys) { + kTree.add_child(k, sPtree->get_child(k)); + } + boost::property_tree::write_json(os, kTree); + } catch (const boost::property_tree::ptree_bad_path& err) { + LOG(fatal) << "non-existing key " << keyOnly << " provided to writeJSON"; + } + } else { + boost::property_tree::write_json(os, *sPtree); + } + return os.str(); +} + +// ------------------------------------------------------------------ + void ConfigurableParam::initPropertyTree() { sPtree->clear(); @@ -870,26 +894,10 @@ void ConfigurableParam::printAllRegisteredParamNames() // ------------------------------------------------------------------ -// Update the storage map of params from the given configuration file. -// It can be in JSON or INI format. -// If nonempty comma-separated paramsList is provided, only those params will -// be updated, absence of data for any of requested params will lead to fatal -// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated -// (to allow preference of run-time settings) -void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly) +namespace +{ +void updateFromPropertyTree(boost::property_tree::ptree const& pt, std::string const& source, std::string const& paramsList, bool unchangedOnly) { - if (!sIsFullyInitialized) { - initialize(); - } - - auto cfgfile = o2::utils::Str::trim_copy(configFile); - - if (cfgfile.length() == 0) { - return; - } - - boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile); - std::vector> keyValPairs; auto request = o2::utils::Str::tokenize(paramsList, ',', true); std::unordered_map requestMap; @@ -913,7 +921,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin auto name = subKey.first; auto value = subKey.second.get_value(); std::string key = mainKey + "." + name; - if (!unchangedOnly || getProvenance(key) == kCODE) { + if (!unchangedOnly || ConfigurableParam::getProvenance(key) == ConfigurableParam::kCODE) { std::pair pair = std::make_pair(key, o2::utils::Str::trim_copy(value)); keyValPairs.push_back(pair); } @@ -928,16 +936,62 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin // make sure all requested params were retrieved for (const auto& req : requestMap) { if (req.second == 0) { - throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, configFile)); + throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, source)); } } try { - setValues(keyValPairs); + ConfigurableParam::setValues(keyValPairs); } catch (std::exception const& error) { LOG(error) << "Error while setting values " << error.what(); } } +} // namespace + +// Update the storage map of params from the given configuration file. +// It can be in JSON or INI format. +// If nonempty comma-separated paramsList is provided, only those params will +// be updated, absence of data for any of requested params will lead to fatal +// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated +// (to allow preference of run-time settings) +void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly) +{ + if (!sIsFullyInitialized) { + initialize(); + } + + auto cfgfile = o2::utils::Str::trim_copy(configFile); + + if (cfgfile.length() == 0) { + return; + } + + updateFromPropertyTree(ConfigurableParamReaders::readConfigFile(cfgfile), configFile, paramsList, unchangedOnly); +} + +// ------------------------------------------------------------------ + +void ConfigurableParam::updateFromJSONString(std::string const& configJSON, std::string const& paramsList, bool unchangedOnly) +{ + if (!sIsFullyInitialized) { + initialize(); + } + + auto json = o2::utils::Str::trim_copy(configJSON); + if (json.length() == 0) { + return; + } + + boost::property_tree::ptree pt; + std::istringstream input(json); + try { + boost::property_tree::read_json(input, pt); + } catch (const boost::property_tree::ptree_error& e) { + LOG(fatal) << "Failed to read JSON config string (" << e.what() << ")"; + } + + updateFromPropertyTree(pt, "provided JSON string", paramsList, unchangedOnly); +} // ------------------------------------------------------------------ // ------------------------------------------------------------------ diff --git a/Common/Utils/test/testConfigurableParam.cxx b/Common/Utils/test/testConfigurableParam.cxx index f0c3c59c79c35..6fd0344cdd1be 100644 --- a/Common/Utils/test/testConfigurableParam.cxx +++ b/Common/Utils/test/testConfigurableParam.cxx @@ -151,6 +151,42 @@ BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_Json) std::remove(testFileName.c_str()); } +BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_UnchangedOnly) +{ + ConfigurableParam::setValue("TestParam.ulValue", "2"); + ConfigurableParam::setProvenance("TestParam", "lValue", ConfigurableParam::kCODE); + ConfigurableParam::setProvenance("TestParam", "ulValue", ConfigurableParam::kRT); + ConfigurableParam::updateFromJSONString(R"json({"TestParam":{"lValue":"77","ulValue":"88"}})json", "TestParam", true); + + BOOST_CHECK_EQUAL(TestParam::Instance().lValue, 77); + BOOST_CHECK_EQUAL(TestParam::Instance().ulValue, 2); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_FromAsJSON) +{ + ConfigurableParam::setValue("TestParam.iValue", "321"); + ConfigurableParam::setValue("TestParam.sValue", "json-source"); + ConfigurableParam::setValues({{"TestParam.map", "{7:8,9:10}"}}); + const auto mapBefore = TestParam::Instance().map; + const auto json = ConfigurableParam::asJSON("TestParam"); + + ConfigurableParam::setValue("TestParam.iValue", "999"); + ConfigurableParam::setValue("TestParam.sValue", "json-modified"); + ConfigurableParam::setValues({{"TestParam.map", "{1:2}"}}); + ConfigurableParam::updateFromJSONString(json, "TestParam"); + + BOOST_CHECK_EQUAL(TestParam::Instance().iValue, 321); + BOOST_CHECK_EQUAL(TestParam::Instance().sValue, "json-source"); + BOOST_CHECK_EQUAL(TestParam::Instance().map.size(), mapBefore.size()); + BOOST_CHECK_EQUAL(TestParam::Instance().map.at(7), mapBefore.at(7)); + BOOST_CHECK_EQUAL(TestParam::Instance().map.at(9), mapBefore.at(9)); +} + +BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_ParamsListMissing) +{ + BOOST_CHECK_THROW(ConfigurableParam::updateFromJSONString(ConfigurableParam::asJSON("TestParam"), "MissingParam"), std::runtime_error); +} + BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_ROOT) { // test for root file serialization From ad38631435dbad978b790f8f8e4a6a5d8e6eab14 Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 28 Jul 2026 18:56:59 +0200 Subject: [PATCH 2/3] Add all barrel tracking configurables to AOD metadata Add all barrel tracking configurables to AOD metadata: write them as ConfigParam__.json files from every device (1st lane only in pipelined) Also, send them as META//0 sporadic DPL output only once for the direct collection by the metadata writer (when it will be supported by the DPL). --- .../StrangenessTrackingSpec.h | 1 + .../src/CosmicsMatchingSpec.cxx | 25 +++++++++++++- .../src/PrimaryVertexingSpec.cxx | 23 +++++++++++-- .../src/SecondaryVertexingSpec.cxx | 20 +++++++++-- .../src/StrangenessTrackingSpec.cxx | 22 +++++++++++- .../src/TOFMatcherSpec.cxx | 19 +++++++++-- .../src/TPCITSMatchingSpec.cxx | 26 +++++++++++--- .../include/ITSWorkflow/TrackerSpec.h | 1 + .../ITSMFT/ITS/workflow/src/TrackerSpec.cxx | 20 +++++++++-- .../include/MFTWorkflow/TrackerSpec.h | 1 + .../ITSMFT/MFT/workflow/src/TrackerSpec.cxx | 20 ++++++++--- .../TRDWorkflow/TRDGlobalTrackingSpec.h | 1 + .../workflow/src/TRDGlobalTrackingSpec.cxx | 17 ++++++++-- .../include/GPUWorkflow/GPUWorkflowSpec.h | 2 ++ GPU/Workflow/src/GPUWorkflowITS.cxx | 7 ++-- GPU/Workflow/src/GPUWorkflowSpec.cxx | 34 +++++++++++++++++-- 16 files changed, 208 insertions(+), 31 deletions(-) diff --git a/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/StrangenessTrackingSpec.h b/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/StrangenessTrackingSpec.h index 8367cef7f51b0..bbc2cdc80995e 100644 --- a/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/StrangenessTrackingSpec.h +++ b/Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/StrangenessTrackingSpec.h @@ -50,6 +50,7 @@ class StrangenessTrackerSpec : public framework::Task private: void updateTimeDependentParams(framework::ProcessingContext& pc); + void storeConfigs(framework::ProcessingContext& pc); bool mUseMC = false; TStopwatch mTimer; diff --git a/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx index d10330db09ea2..338bb86bd8033 100644 --- a/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/CosmicsMatchingSpec.cxx @@ -11,14 +11,18 @@ /// @file CosmicsMatchingSpec.cxx +#include +#include #include #include #include "TStopwatch.h" #include "GlobalTracking/MatchCosmics.h" +#include "GlobalTracking/MatchCosmicsParams.h" #include "DataFormatsITSMFT/TopologyDictionary.h" #include "DataFormatsTPC/Constants.h" #include "ReconstructionDataFormats/GlobalTrackID.h" #include "Framework/ConfigParamRegistry.h" +#include "Framework/DeviceSpec.h" #include "GlobalTrackingWorkflow/CosmicsMatchingSpec.h" #include "ReconstructionDataFormats/GlobalTrackAccessor.h" #include "ReconstructionDataFormats/GlobalTrackID.h" @@ -71,6 +75,7 @@ class CosmicsMatchingSpec : public Task private: void updateTimeDependentParams(ProcessingContext& pc); + void storeConfigs(ProcessingContext& pc); std::shared_ptr mDataRequest; std::shared_ptr mGGCCDBRequest; o2::tpc::VDriftHelper mTPCVDriftHelper{}; @@ -98,7 +103,7 @@ void CosmicsMatchingSpec::run(ProcessingContext& pc) RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); updateTimeDependentParams(pc); // Make sure this is called after recoData.collectData, which may load some conditions - + storeConfigs(pc); mMatching.process(recoData); pc.outputs().snapshot(Output{"GLO", "COSMICTRC", 0}, mMatching.getCosmicTracks()); if (mUseMC) { @@ -107,6 +112,22 @@ void CosmicsMatchingSpec::run(ProcessingContext& pc) mTimer.Stop(); } +void CosmicsMatchingSpec::storeConfigs(ProcessingContext& pc) +{ + static bool first = true; + if (first) { + first = false; + if (pc.services().get().inputTimesliceId == 0) { + const auto& conf = MatchCosmicsParams::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, conf.getName()), conf.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "COSMICMATCHER", 0}, md); + } + } +} + void CosmicsMatchingSpec::updateTimeDependentParams(ProcessingContext& pc) { o2::base::GRPGeomHelper::instance().checkUpdates(pc); @@ -193,6 +214,8 @@ DataProcessorSpec getCosmicsMatchingSpec(GTrackID::mask_t src, bool usePV, bool o2::tpc::VDriftHelper::requestCCDBInputs(dataRequest->inputs); dataRequest->inputs.emplace_back("corrMap", o2::header::gDataOriginTPC, "TPCCORRMAP", 0, Lifetime::Timeframe); + outputs.emplace_back("META", "COSMICMATCHER", 0, Lifetime::Sporadic); + return DataProcessorSpec{ "cosmics-matcher", dataRequest->inputs, diff --git a/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx index c1d7b62bbf731..e92d5c35152e1 100644 --- a/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/PrimaryVertexingSpec.cxx @@ -11,12 +11,15 @@ /// @file PrimaryVertexingSpec.cxx +#include +#include #include #include #include "DataFormatsGlobalTracking/RecoContainer.h" #include "DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h" #include "DataFormatsITSMFT/TrkClusRef.h" #include "DataFormatsCalibration/MeanVertexObject.h" +#include "DataFormatsCalibration/MeanVertexBiasParam.h" #include "ReconstructionDataFormats/TrackTPCITS.h" #include "ReconstructionDataFormats/GlobalTrackID.h" #include "DetectorsBase/Propagator.h" @@ -59,6 +62,7 @@ class PrimaryVertexingSpec : public Task private: void updateTimeDependentParams(ProcessingContext& pc); + void storeConfigs(ProcessingContext& pc); std::shared_ptr mDataRequest; std::shared_ptr mGGCCDBRequest; o2::vertexing::PVertexer mVertexer; @@ -98,7 +102,7 @@ void PrimaryVertexingSpec::run(ProcessingContext& pc) o2::globaltracking::RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); // select tracks of needed type, with minimal cuts, the real selected will be done in the vertexer updateTimeDependentParams(pc); // Make sure this is called after recoData.collectData, which may load some conditions - + storeConfigs(pc); std::vector tracks; std::vector tracksMCInfo; std::vector gids; @@ -180,6 +184,8 @@ void PrimaryVertexingSpec::run(ProcessingContext& pc) vertices[iv].setFlags(PVertex::UPCMode); } } + } else { + storeConfigs(pc); } pc.outputs().snapshot(Output{"GLO", "PVTX", 0}, vertices); @@ -197,12 +203,23 @@ void PrimaryVertexingSpec::run(ProcessingContext& pc) mVertexer.getTimeReAttach().CpuTime(), mVertexer.getTotTrials(), mVertexer.getNTZClusters(), mVertexer.getMaxTrialsPerCluster(), mVertexer.getLongestClusterTimeMS(), mVertexer.getLongestClusterMult(), mVertexer.getNIniFound(), mVertexer.getNKilledBCValid(), mVertexer.getNKilledIntCand(), mVertexer.getNKilledDebris(), mVertexer.getNKilledQuality(), mVertexer.getNKilledITSOnly()); +} +void PrimaryVertexingSpec::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; + const auto& confPV = PVertexerParams::Instance(); + const auto& confMV = o2::dataformats::MeanVertexBiasParam::Instance(); if (pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, PVertexerParams::Instance().getName()), PVertexerParams::Instance().getName()); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, confPV.getName()), confPV.getName()); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, confMV.getName()), confMV.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(confPV.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confPV.getName()).c_str())); + md.Add(new TObjString(confMV.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confMV.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "PVERTEXER", 0}, md); } } } @@ -289,6 +306,8 @@ DataProcessorSpec getPrimaryVertexingSpec(GTrackID::mask_t src, bool skip, bool true); dataRequest->inputs.emplace_back("meanvtx", "GLO", "MEANVERTEX", 0, Lifetime::Condition, ccdbParamSpec("GLO/Calib/MeanVertex", {}, 1)); + outputs.emplace_back("META", "PVERTEXER", 0, Lifetime::Sporadic); + return DataProcessorSpec{ "primary-vertexing", dataRequest->inputs, diff --git a/Detectors/GlobalTrackingWorkflow/src/SecondaryVertexingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/SecondaryVertexingSpec.cxx index afce2861be2fb..3a9de8662f4b5 100644 --- a/Detectors/GlobalTrackingWorkflow/src/SecondaryVertexingSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/SecondaryVertexingSpec.cxx @@ -11,6 +11,8 @@ /// @file SecondaryVertexingSpec.cxx +#include +#include #include #include "DataFormatsCalibration/MeanVertexObject.h" #include "Framework/CCDBParamSpec.h" @@ -65,6 +67,7 @@ class SecondaryVertexingSpec : public Task void finaliseCCDB(ConcreteDataMatcher& matcher, void* obj) final; private: + void storeConfigs(ProcessingContext& pc); void updateTimeDependentParams(ProcessingContext& pc); std::shared_ptr mDataRequest; std::shared_ptr mGGCCDBRequest; @@ -110,7 +113,7 @@ void SecondaryVertexingSpec::run(ProcessingContext& pc) o2::globaltracking::RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); updateTimeDependentParams(pc); - + storeConfigs(pc); mVertexer.process(recoData, pc); mTimer.Stop(); @@ -119,15 +122,25 @@ void SecondaryVertexingSpec::run(ProcessingContext& pc) mVertexer.getNV0s(), calls[0] - fitCalls[0], mVertexer.getNCascades(), calls[1] - fitCalls[1], mVertexer.getN3Bodies(), calls[2] - fitCalls[2], mVertexer.getNStrangeTracks(), mTimer.CpuTime() - timeCPU0, mTimer.RealTime() - timeReal0); fitCalls = calls; +} +void SecondaryVertexingSpec::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; if (pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, SVertexerParams::Instance().getName()), SVertexerParams::Instance().getName()); + const auto& confSV = SVertexerParams::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, confSV.getName()), confSV.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(confSV.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confSV.getName()).c_str())); if (mEnableStrangenessTracking) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, o2::strangeness_tracking::StrangenessTrackingParamConfig::Instance().getName()), o2::strangeness_tracking::StrangenessTrackingParamConfig::Instance().getName()); + const auto& confST = o2::strangeness_tracking::StrangenessTrackingParamConfig::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, confST.getName()), confST.getName()); + md.Add(new TObjString(confST.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confST.getName()).c_str())); } + pc.outputs().snapshot(Output{"META", "SVERTEXER", 0}, md); } } } @@ -298,6 +311,7 @@ DataProcessorSpec getSecondaryVertexingSpec(GTrackID::mask_t src, bool enableCas LOG(info) << "Strangeness tracker will use MC"; } } + outputs.emplace_back("META", "SVERTEXER", 0, Lifetime::Sporadic); return DataProcessorSpec{ "secondary-vertexing", diff --git a/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx index e313940b0a91e..c438f869773a5 100644 --- a/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/StrangenessTrackingSpec.cxx @@ -11,6 +11,8 @@ /// \file StrangenessTrackingSpec.cxx /// \brief +#include +#include #include "TGeoGlobalMagField.h" #include "Framework/ConfigParamRegistry.h" #include "Field/MagneticField.h" @@ -20,6 +22,7 @@ #include "ITSWorkflow/TrackerSpec.h" #include "ITSWorkflow/TrackReaderSpec.h" #include "Framework/CCDBParamSpec.h" +#include "Framework/DeviceSpec.h" #include "DataFormatsParameters/GRPObject.h" #include "DataFormatsITSMFT/ROFRecord.h" @@ -68,7 +71,7 @@ void StrangenessTrackerSpec::run(framework::ProcessingContext& pc) o2::globaltracking::RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); updateTimeDependentParams(pc); - + storeConfigs(pc); auto geom = o2::its::GeometryTGeo::Instance(); mTracker.loadData(recoData); mTracker.prepareITStracks(); @@ -83,6 +86,22 @@ void StrangenessTrackerSpec::run(framework::ProcessingContext& pc) mTimer.Stop(); } +void StrangenessTrackerSpec::storeConfigs(framework::ProcessingContext& pc) +{ + static bool first = true; + if (first) { + first = false; + if (pc.services().get().inputTimesliceId == 0) { + const auto& conf = StrangenessTrackingParamConfig::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, conf.getName()), conf.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "STRTRACKER", 0}, md); + } + } +} + ///_______________________________________ void StrangenessTrackerSpec::updateTimeDependentParams(ProcessingContext& pc) { @@ -162,6 +181,7 @@ DataProcessorSpec getStrangenessTrackerSpec(o2::dataformats::GlobalTrackID::mask outputs.emplace_back("GLO", "STRANGETRACKS_MC", 0, Lifetime::Timeframe); LOG(info) << "Strangeness tracker will use MC"; } + outputs.emplace_back("META", "STRTRACKER", 0, Lifetime::Sporadic); return DataProcessorSpec{ "strangeness-tracker", diff --git a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx index 746e572c506b8..17ee87b8d52cd 100644 --- a/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx @@ -13,6 +13,8 @@ #include #include +#include +#include #include "TStopwatch.h" #include "Framework/ConfigParamRegistry.h" #include "DetectorsBase/GeometryManager.h" @@ -68,6 +70,7 @@ class TOFMatcherSpec : public Task private: void updateTimeDependentParams(ProcessingContext& pc); + void storeConfigs(ProcessingContext& pc); std::shared_ptr mDataRequest; std::shared_ptr mGGCCDBRequest; o2::tpc::VDriftHelper mTPCVDriftHelper{}; @@ -141,6 +144,7 @@ void TOFMatcherSpec::run(ProcessingContext& pc) RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); updateTimeDependentParams(pc); + storeConfigs(pc); auto creationTime = pc.services().get().creation; LOG(debug) << "isTrackSourceLoaded: TPC -> " << recoData.isTrackSourceLoaded(o2::dataformats::GlobalTrackID::Source::TPC); @@ -214,15 +218,23 @@ void TOFMatcherSpec::run(ProcessingContext& pc) pc.outputs().snapshot(Output{o2::header::gDataOriginTOF, "MATCHABLES_17", 0}, mMatcher.getMatchedTracksPair(17)); } + mTimer.Stop(); +} + +void TOFMatcherSpec::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; if (pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, MatchTOFParams::Instance().getName()), MatchTOFParams::Instance().getName()); + const auto& conf = MatchTOFParams::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, conf.getName()), conf.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "TOFMATCHER", 0}, md); } } - - mTimer.Stop(); } void TOFMatcherSpec::endOfStream(EndOfStreamContext& ec) @@ -309,6 +321,7 @@ DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bo outputs.emplace_back(o2::header::gDataOriginTOF, "MATCHABLES_16", 0, Lifetime::Timeframe); outputs.emplace_back(o2::header::gDataOriginTOF, "MATCHABLES_17", 0, Lifetime::Timeframe); } + outputs.emplace_back("META", "TOFMATCHER", 0, Lifetime::Sporadic); return DataProcessorSpec{ "tof-matcher", diff --git a/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx b/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx index 079fe5455fd4a..7f63b61e02be0 100644 --- a/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx +++ b/Detectors/GlobalTrackingWorkflow/src/TPCITSMatchingSpec.cxx @@ -12,9 +12,11 @@ /// @file TPCITSMatchingSpec.cxx #include - +#include +#include #include "GlobalTracking/MatchTPCITS.h" #include "GlobalTracking/MatchTPCITSParams.h" +#include "FT0Reconstruction/InteractionTag.h" #include "DataFormatsITSMFT/TopologyDictionary.h" #include "DataFormatsTPC/Constants.h" #include "Framework/DataProcessorSpec.h" @@ -80,6 +82,7 @@ class TPCITSMatchingDPL : public Task private: void updateTimeDependentParams(ProcessingContext& pc); + void storeConfigs(ProcessingContext& pc); std::shared_ptr mDataRequest; std::shared_ptr mGGCCDBRequest; o2::tpc::VDriftHelper mTPCVDriftHelper{}; @@ -112,6 +115,7 @@ void TPCITSMatchingDPL::run(ProcessingContext& pc) RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); updateTimeDependentParams(pc); // Make sure this is called after recoData.collectData, which may load some conditions + storeConfigs(pc); static pmr::vector dummyMCLab, dummyMCLabAB; static pmr::vector> dummyCalib; @@ -125,15 +129,26 @@ void TPCITSMatchingDPL::run(ProcessingContext& pc) mMatching.run(recoData, matchedTracks, ABTrackletRefs, ABTrackletClusterIDs, matchLabels, ABTrackletLabels, calib); + mTimer.Stop(); +} + +void TPCITSMatchingDPL::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; + const auto& confMatch = MatchTPCITSParams::Instance(); + const auto& confInt = ft0::InteractionTag::Instance(); if (pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, MatchTPCITSParams::Instance().getName()), MatchTPCITSParams::Instance().getName()); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, confMatch.getName()), confMatch.getName()); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, confInt.getName()), confInt.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(confMatch.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confMatch.getName()).c_str())); + md.Add(new TObjString(confInt.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(confInt.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "TPCITSMATCHER", 0}, md); } } - - mTimer.Stop(); } void TPCITSMatchingDPL::endOfStream(EndOfStreamContext& ec) @@ -295,6 +310,9 @@ DataProcessorSpec getTPCITSMatchingSpec(GTrackID::mask_t src, bool useFT0, bool if (requestCTPLumi) { dataRequest->inputs.emplace_back("lumiCTP", o2::header::gDataOriginCTP, "LUMICTP", 0, Lifetime::Timeframe); } + + outputs.emplace_back("META", "TPCITSMATCHER", 0, Lifetime::Sporadic); + return DataProcessorSpec{ "itstpc-track-matcher", dataRequest->inputs, diff --git a/Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/TrackerSpec.h b/Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/TrackerSpec.h index 8ce63efcb7a3b..22a8092ec69c5 100644 --- a/Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/TrackerSpec.h +++ b/Detectors/ITSMFT/ITS/workflow/include/ITSWorkflow/TrackerSpec.h @@ -57,6 +57,7 @@ class TrackerDPL : public framework::Task private: void end(); void updateTimeDependentParams(framework::ProcessingContext& pc); + void storeConfigs(framework::ProcessingContext& pc); std::unique_ptr mRecChain = nullptr; std::unique_ptr mChainITS = nullptr; std::shared_ptr mGGCCDBRequest; diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx index bbafc48e931ed..a198e638c5bac 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx @@ -10,7 +10,8 @@ // or submit itself to any jurisdiction. #include - +#include +#include #include "Framework/ControlService.h" #include "Framework/ConfigParamRegistry.h" #include "Framework/CCDBParamSpec.h" @@ -60,15 +61,27 @@ void TrackerDPL::run(ProcessingContext& pc) auto realt = mTimer.RealTime(); mTimer.Start(false); mITSTrackingInterface.updateTimeDependentParams(pc); + storeConfigs(pc); mITSTrackingInterface.run(pc); mTimer.Stop(); LOGP(info, "CPU Reconstruction time for this TF {:.2f} s (cpu), {:.2f} s (wall)", mTimer.CpuTime() - cput, mTimer.RealTime() - realt); +} + +void TrackerDPL::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; if (pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, o2::its::VertexerParamConfig::Instance().getName()), o2::its::VertexerParamConfig::Instance().getName()); - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, o2::its::TrackerParamConfig::Instance().getName()), o2::its::TrackerParamConfig::Instance().getName()); + const auto& vtconf = o2::its::VertexerParamConfig::Instance(); + const auto& trconf = o2::its::TrackerParamConfig::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, vtconf.getName()), vtconf.getName()); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, trconf.getName()), trconf.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(vtconf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(vtconf.getName()).c_str())); + md.Add(new TObjString(trconf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(trconf.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "ITSTRACKER", 0}, md); } } } @@ -138,6 +151,7 @@ DataProcessorSpec getTrackerSpec(bool useMC, bool doStag, bool useGeom, int trgT outputs.emplace_back("ITS", "VERTICESMCPUR", 0, Lifetime::Timeframe); outputs.emplace_back("ITS", "TRACKSMCTR", 0, Lifetime::Timeframe); } + outputs.emplace_back("META", "ITSTRACKER", 0, Lifetime::Sporadic); return DataProcessorSpec{ .name = "its-tracker", diff --git a/Detectors/ITSMFT/MFT/workflow/include/MFTWorkflow/TrackerSpec.h b/Detectors/ITSMFT/MFT/workflow/include/MFTWorkflow/TrackerSpec.h index 8bd290caf5a41..3112e3efef5e6 100644 --- a/Detectors/ITSMFT/MFT/workflow/include/MFTWorkflow/TrackerSpec.h +++ b/Detectors/ITSMFT/MFT/workflow/include/MFTWorkflow/TrackerSpec.h @@ -44,6 +44,7 @@ class TrackerDPL : public o2::framework::Task private: void updateTimeDependentParams(framework::ProcessingContext& pc); + void storeConfigs(framework::ProcessingContext& pc); ///< MFT readout mode bool mMFTTriggered = false; ///< MFT readout is triggered diff --git a/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx index 6ceb04b3c4df6..e3bd557435ec0 100644 --- a/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx +++ b/Detectors/ITSMFT/MFT/workflow/src/TrackerSpec.cxx @@ -18,7 +18,8 @@ #include "MFTTracking/Tracker.h" #include "MFTTracking/TrackCA.h" #include "MFTBase/GeometryTGeo.h" - +#include +#include #include #include @@ -64,6 +65,7 @@ void TrackerDPL::run(ProcessingContext& pc) mTimer[SWTot].Start(false); updateTimeDependentParams(pc); + storeConfigs(pc); gsl::span patterns = pc.inputs().get>("patterns"); auto compClusters = pc.inputs().get>("compClusters"); auto ntracks = 0; @@ -325,15 +327,23 @@ void TrackerDPL::run(ProcessingContext& pc) pc.outputs().snapshot(Output{"MFT", "TRACKSMCTR", 0}, allTrackLabels); } + mTimer[SWTot].Stop(); +} + +void TrackerDPL::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; if (pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, o2::mft::MFTTrackingParam::Instance().getName()), o2::mft::MFTTrackingParam::Instance().getName()); + const auto& conf = o2::mft::MFTTrackingParam::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, conf.getName()), conf.getName()); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(conf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(conf.getName()).c_str())); + pc.outputs().snapshot(Output{"META", "MFTTRACKER", 0}, md); } } - - mTimer[SWTot].Stop(); } void TrackerDPL::endOfStream(EndOfStreamContext& ec) @@ -462,6 +472,8 @@ DataProcessorSpec getTrackerSpec(bool useMC, bool useGeom, int nThreads) outputs.emplace_back("MFT", "TRACKSMCTR", 0, Lifetime::Timeframe); } + outputs.emplace_back("META", "MFTTRACKER", 0, Lifetime::Sporadic); + return DataProcessorSpec{ "mft-tracker", inputs, diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingSpec.h index acb061a4bcc86..c3a5be6a45649 100644 --- a/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingSpec.h +++ b/Detectors/TRD/workflow/include/TRDWorkflow/TRDGlobalTrackingSpec.h @@ -66,6 +66,7 @@ class TRDGlobalTracking : public o2::framework::Task private: void updateTimeDependentParams(o2::framework::ProcessingContext& pc); + void storeConfigs(o2::framework::ProcessingContext& pc); o2::gpu::GPUTRDTracker* mTracker{nullptr}; ///< TRD tracking engine o2::gpu::GPUReconstruction* mRec{nullptr}; ///< GPU reconstruction pointer, handles memory for the tracker diff --git a/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx b/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx index 4aecd0e3df054..e541482dbde2a 100644 --- a/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx +++ b/Detectors/TRD/workflow/src/TRDGlobalTrackingSpec.cxx @@ -11,6 +11,8 @@ /// @file TRDGlobalTrackingSpec.cxx +#include +#include #include "TRDWorkflow/TRDGlobalTrackingSpec.h" #include "TRDBase/Geometry.h" #include "DetectorsCommonDataFormats/DetectorNameConf.h" @@ -48,6 +50,7 @@ #include "GPUO2InterfaceConfiguration.h" #include "GPUO2InterfaceUtils.h" #include "GPUSettings.h" +#include "GPUO2ConfigurableParam.h" #include "GPUDataTypesIO.h" #include "GPUTRDDef.h" #include "GPUTRDTrack.h" @@ -281,6 +284,7 @@ void TRDGlobalTracking::run(ProcessingContext& pc) o2::globaltracking::RecoContainer inputTracks; inputTracks.collectData(pc, *mDataRequest); updateTimeDependentParams(pc); + storeConfigs(pc); mChainTracking->ClearIOPointers(); mTPCClusterIdxStruct = &inputTracks.inputsTPCclusters->clusterIndex; @@ -564,15 +568,22 @@ void TRDGlobalTracking::run(ProcessingContext& pc) } } + mTimer.Stop(); +} + +void TRDGlobalTracking::storeConfigs(ProcessingContext& pc) +{ static bool first = true; if (first) { first = false; if (pc.services().get().inputTimesliceId == 0) { o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, "GPU_rec_trd"), "GPU_rec_trd"); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(o2::gpu::internal::GPUConfigurableParamGPUSettingsRecTRD::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::gpu::internal::GPUConfigurableParamGPUSettingsRecTRD::Instance().getName()).c_str())); + pc.outputs().snapshot(Output{"META", "TRDTRACKER", 0}, md); } } - - mTimer.Stop(); } bool TRDGlobalTracking::refitITSTPCTRDTrack(TrackTRD& trk, float timeTRD, o2::globaltracking::RecoContainer* recoCont) @@ -1019,6 +1030,8 @@ DataProcessorSpec getTRDGlobalTrackingSpec(bool useMC, GTrackID::mask_t src, boo } } + outputs.emplace_back("META", "TRDTRACKER", 0, Lifetime::Sporadic); + std::string processorName = o2::utils::Str::concat_string("trd-globaltracking", GTrackID::getSourcesNames(src)); std::regex reg("[,\\[\\]]+"); processorName = regex_replace(processorName, reg, "_"); diff --git a/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h b/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h index 7b6bffcac9e1e..978c5f312cfe7 100644 --- a/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h +++ b/GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h @@ -162,6 +162,8 @@ class GPURecoWorkflowSpec : public o2::framework::Task aligned_unique_buffer_ptr mFastTransformBuffer; }; + void storeConfigs(o2::framework::ProcessingContext& pc); + /// initialize TPC options from command line void initFunctionTPCCalib(o2::framework::InitContext& ic); void initFunctionITS(o2::framework::InitContext& ic); diff --git a/GPU/Workflow/src/GPUWorkflowITS.cxx b/GPU/Workflow/src/GPUWorkflowITS.cxx index ac9834d3eacd1..2a0e36d65bb7a 100644 --- a/GPU/Workflow/src/GPUWorkflowITS.cxx +++ b/GPU/Workflow/src/GPUWorkflowITS.cxx @@ -23,6 +23,8 @@ #include "CommonUtils/NameConf.h" #include "ITStracking/TrackingInterface.h" #include "ITStracking/TrackingConfigParam.h" +#include +#include #ifdef ENABLE_UPGRADES #include "ITS3Reconstruction/TrackingInterface.h" @@ -36,11 +38,6 @@ int32_t GPURecoWorkflowSpec::runITSTracking(o2::framework::ProcessingContext& pc mITSTimeFrame->setDevicePropagator(mGPUReco->GetDeviceO2Propagator()); LOGP(debug, "GPUChainITS is giving me device propagator: {}", (void*)mGPUReco->GetDeviceO2Propagator()); mITSTrackingInterface->run(pc); - static bool first = true; - if (mNTFs == 1 && pc.services().get().inputTimesliceId == 0) { - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, o2::its::VertexerParamConfig::Instance().getName()), o2::its::VertexerParamConfig::Instance().getName()); - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, o2::its::TrackerParamConfig::Instance().getName()), o2::its::TrackerParamConfig::Instance().getName()); - } return 0; } diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx index 01a82ebdba361..e51a45044da2b 100644 --- a/GPU/Workflow/src/GPUWorkflowSpec.cxx +++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx @@ -14,6 +14,9 @@ /// @since 2018-04-18 /// @brief Processor spec for running TPC CA tracking +#include +#include +#include "GPUO2ConfigurableParam.h" #include "GPUWorkflow/GPUWorkflowSpec.h" #include "Headers/DataHeader.h" #include "Framework/WorkflowSpec.h" // o2::framework::mergeInputs @@ -75,6 +78,7 @@ #include "GPUReconstructionConvert.h" #include "DetectorsRaw/RDHUtils.h" #include "ITStracking/TrackingInterface.h" +#include "ITStracking/TrackingConfigParam.h" #include "GPUWorkflowInternal.h" #include "GPUDataTypesQA.h" // #include "Framework/ThreadPool.h" @@ -767,6 +771,9 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc) } // ------------------------------ Actual processing ------------------------------ + if (mNTFs == 1 && pc.services().get().inputTimesliceId == 0) { + storeConfigs(pc); + } if ((int32_t)(ptrs.tpcZS != nullptr) + (int32_t)(ptrs.tpcPackedDigits != nullptr && (ptrs.tpcZS == nullptr || ptrs.tpcPackedDigits->tpcDigitsMC == nullptr)) + (int32_t)(ptrs.clustersNative != nullptr) + (int32_t)(ptrs.tpcCompressedClusters != nullptr) != 1) { throw std::runtime_error("Invalid input for gpu tracking"); @@ -805,9 +812,6 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc) mNTFDumps++; } } - if (mNTFs == 1 && pc.services().get().inputTimesliceId == 0) { // TPC ConfigurableCarams are somewhat special, need to construct by hand - o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, "rec_tpc"), "GPU_rec_tpc,GPU_rec,GPU_proc_param,GPU_proc,GPU_global,trackTuneParams"); - } std::unique_ptr ptrsDump; if (mConfParam->dumpBadTFMode == 2) { @@ -1020,6 +1024,29 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc) LOG(info) << "GPU Reconstruction time for this TF " << mTimer->CpuTime() - cput << " s (cpu), " << mTimer->RealTime() - realt << " s (wall)"; } +void GPURecoWorkflowSpec::storeConfigs(ProcessingContext& pc) +{ + // TPC ConfigurableCarams are somewhat special, need to construct by hand + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, "rec_tpc"), "GPU_rec_tpc,GPU_rec,GPU_proc_param,GPU_proc,GPU_global,trackTuneParams"); + TMap md; + md.SetOwnerKeyValue(); + md.Add(new TObjString(o2::gpu::internal::GPUConfigurableParamGPUSettingsRecTPC::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::gpu::internal::GPUConfigurableParamGPUSettingsRecTPC::Instance().getName()).c_str())); + md.Add(new TObjString(o2::gpu::internal::GPUConfigurableParamGPUSettingsRec::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::gpu::internal::GPUConfigurableParamGPUSettingsRec::Instance().getName()).c_str())); + md.Add(new TObjString(o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingParam::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingParam::Instance().getName()).c_str())); + md.Add(new TObjString(o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessing::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessing::Instance().getName()).c_str())); + md.Add(new TObjString(o2::gpu::internal::GPUConfigurableParamGPUSettingsO2::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::gpu::internal::GPUConfigurableParamGPUSettingsO2::Instance().getName()).c_str())); + md.Add(new TObjString(o2::globaltracking::TrackTuneParams::Instance().getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(o2::globaltracking::TrackTuneParams::Instance().getName()).c_str())); + if (mSpecConfig.runITSTracking) { + const auto& vtconf = o2::its::VertexerParamConfig::Instance(); + const auto& trconf = o2::its::TrackerParamConfig::Instance(); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, vtconf.getName()), vtconf.getName()); + o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get().name, trconf.getName()), trconf.getName()); + md.Add(new TObjString(vtconf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(vtconf.getName()).c_str())); + md.Add(new TObjString(trconf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(trconf.getName()).c_str())); + } + pc.outputs().snapshot(Output{"META", "GPUTRACKER", 0}, md); +} + void GPURecoWorkflowSpec::doCalibUpdates(o2::framework::ProcessingContext& pc, calibObjectStruct& oldCalibObjects) { GPUCalibObjectsConst newCalibObjects; @@ -1368,6 +1395,7 @@ Outputs GPURecoWorkflowSpec::outputs() if (mSpecConfig.outputErrorQA) { outputSpecs.emplace_back(gDataOriginGPU, "ERRORQA", 0, Lifetime::Timeframe); } + outputSpecs.emplace_back("META", "GPUTRACKER", 0, Lifetime::Sporadic); if (mSpecConfig.runITSTracking) { outputSpecs.emplace_back(gDataOriginITS, "TRACKS", 0, Lifetime::Timeframe); From 970d411619582547733b20852f6c56767eee3f4a Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 28 Jul 2026 19:02:18 +0200 Subject: [PATCH 3/3] Option to create configParams metadata from json dumps With the option `--collect-config-files` provided, the AODProducer will collect the json dumps of ConfigParams made by the upstream processes and will add them as ` : ` TObjString pairs in the AOD metadata. Once the DPL supports collecting META//0 sporadic inputs, the metadata will be created directly from these inputs, so this option will not be needed for the data processing. But in the case of MC, collecting json files via this option is the only way to add configs to the AOD metadata (as it is done in the sim_challenge.sh). --- .../AODProducerWorkflowSpec.h | 3 + Detectors/AOD/src/AODProducerWorkflowSpec.cxx | 109 +++++++++++++++++- prodtests/sim_challenge.sh | 2 +- 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h b/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h index 02f1b2582d74b..21728e78a7a20 100644 --- a/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h +++ b/Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h @@ -246,6 +246,8 @@ class AODProducerWorkflowDPL : public Task return std::uint64_t(mStartIR.toLong()) + relativeTime_to_LocalBC(relativeTimeStampInNS); } + bool collectConfigFiles(std::vector& keys, std::vector& values, int indent = -1); + bool mThinTracks{false}; bool mPropTracks{false}; bool mPropMuons{false}; @@ -280,6 +282,7 @@ class AODProducerWorkflowDPL : public Task bool mEnableFITextra = false; bool mEnableTRDextra = false; bool mFieldON = false; + bool mCollectConfigFiles = false; const float cSpeed = 0.029979246f; // speed of light in TOF units GID::mask_t mInputSources; diff --git a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx index 73b852d550ffe..385a9930d8f66 100644 --- a/Detectors/AOD/src/AODProducerWorkflowSpec.cxx +++ b/Detectors/AOD/src/AODProducerWorkflowSpec.cxx @@ -54,6 +54,8 @@ #include "Framework/TableBuilder.h" #include "Framework/CCDBParamSpec.h" #include "CommonUtils/TreeStreamRedirector.h" +#include "CommonUtils/KeyValParam.h" +#include "CommonUtils/NameConf.h" #include "FT0Base/Geometry.h" #include "GlobalTracking/MatchTOF.h" #include "ReconstructionDataFormats/Cascade.h" @@ -88,6 +90,7 @@ #include "MathUtils/Utils.h" #include "Math/SMatrix.h" #include "TString.h" +#include #include #include #include @@ -1902,6 +1905,8 @@ void AODProducerWorkflowDPL::init(InitContext& ic) mUseSigFiltMC = ic.options().get("mc-signal-filt"); + mCollectConfigFiles = ic.options().get("collect-config-files"); + // set no truncation if selected by user if (mTruncate != 1) { LOG(info) << "Truncation is not used!"; @@ -2670,6 +2675,10 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc) mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag, mUser}; add_additional_meta_info(mMetaDataKeys, mMetaDataVals); + if (mCollectConfigFiles) { + collectConfigFiles(mMetaDataKeys, mMetaDataVals); + } + pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys); pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals); @@ -3405,6 +3414,102 @@ std::vector AODProducerWorkflowDPL::fillBCFlags(const o2::globaltrackin return flags; } +bool AODProducerWorkflowDPL::collectConfigFiles(std::vector& keys, std::vector& values, int indent) +{ + // collect JSON-files of ConfigParams dumped by different upstream processors and add to medata + static std::string pattern, directory; + static size_t cachedNumberOfFiles = 0, cachedTotalFileSize = 0; + static bool first = true, discard = false; + if (discard) { + return false; + } + std::error_code ec; + if (first) { + first = false; + pattern = o2::base::NameConf::Instance().getConfigOutputFileName("*"); + auto dir = o2::conf::KeyValParam::Instance().getOutputDir(); + if (dir == "/dev/null") { + LOGP(warn, "ConfigParams output is disabled, abandoning {} files collection for metadata", pattern); + discard = true; + return false; + } + directory = (dir.empty() || dir == "none") ? "." : dir; + if (!std::filesystem::is_directory(directory, ec)) { + LOGP(error, R"(No directory "{}" is found to look for {} configuration files)", directory, pattern); + discard = true; + return false; + } + } + static std::unordered_map cachedMap; + std::vector files; + size_t currentTotalFileSize = 0; + + for (const auto& entry : std::filesystem::directory_iterator(directory)) { + if (!entry.is_regular_file()) { + continue; + } + const std::string fileName = entry.path().filename().string(); + if (fnmatch(pattern.c_str(), fileName.c_str(), 0) != 0) { + continue; + } + const auto fileSize = entry.file_size(ec); + if (ec) { + LOGP(error, "Cannot determine size of file {}, reason: {}", entry.path().string(), ec.message()); + } + files.push_back(entry.path()); + currentTotalFileSize += static_cast(fileSize); + } + + if (files.size() != cachedNumberOfFiles || currentTotalFileSize != cachedTotalFileSize) { // need to create a new map + cachedNumberOfFiles = files.size(); + cachedTotalFileSize = currentTotalFileSize; + cachedMap.clear(); + } + + if (!files.empty() && cachedMap.empty()) { + for (const auto& fname : files) { + std::ifstream input(fname); + if (!input) { + LOGP(error, "Cannot open JSON file {}", fname.string()); + cachedTotalFileSize = 0; // will trigger a new trial next time + continue; + } + nlohmann::json document; + try { + input >> document; + } catch (const nlohmann::json::parse_error& e) { + LOGP(error, "Cannot parse JSON file {}, reason: {}", fname.string(), e.what()); + cachedTotalFileSize = 0; // will trigger a new trial next time + continue; + } + + if (!document.is_object()) { + LOGP(error, "Top-level JSON value is not an object in file: {}", fname.string()); + cachedTotalFileSize = 0; // will trigger a new trial next time + continue; + } + + for (auto it = document.begin(); it != document.end(); ++it) { + const std::string& key = it.key(); + if (cachedMap.find(key) != cachedMap.end()) { + LOGP(error, "Duplicate top-level key {} in file {}", key, fname.string()); + continue; + } + LOGP(info, "Adding json config {} from file {} to AOD metadata", key, fname.string()); + nlohmann::json valueDocument = nlohmann::json::object(); + valueDocument[key] = it.value(); + cachedMap[key] = valueDocument.dump(indent); + } + } + } + + for (const auto& kv : cachedMap) { + keys.push_back(kv.first.c_str()); + values.push_back(kv.second.c_str()); + } + return true; +} + void AODProducerWorkflowDPL::endOfStream(EndOfStreamContext& /*ec*/) { LOGF(info, "aod producer dpl total timing: Cpu: %.3e Real: %.3e s in %d slots", @@ -3565,7 +3670,9 @@ DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, boo ConfigParamSpec{"trackqc-tpc-pt", VariantType::Float, 0.2f, {"Keep TPC standalone track with this pt"}}, ConfigParamSpec{"with-streamers", VariantType::String, "", {"Bit-mask to steer writing of intermediate streamer files"}}, ConfigParamSpec{"seed", VariantType::Int, 0, {"Set seed for random generator used for sampling (0 (default) means using a random_device)"}}, - ConfigParamSpec{"mc-signal-filt", VariantType::Bool, false, {"Enable usage of signal filtering (only for MC with embedding)"}}}}; + ConfigParamSpec{"mc-signal-filt", VariantType::Bool, false, {"Enable usage of signal filtering (only for MC with embedding)"}}, + ConfigParamSpec{"collect-config-files", VariantType::Bool, false, {"Collect ConfigParams json files written by upsteam processors"}}, + }}; } } // namespace o2::aodproducer diff --git a/prodtests/sim_challenge.sh b/prodtests/sim_challenge.sh index a7c7e7f7993d7..a2cf44dc3a66d 100755 --- a/prodtests/sim_challenge.sh +++ b/prodtests/sim_challenge.sh @@ -276,7 +276,7 @@ if [ "$doreco" == "1" ]; then # echo "Return status of strangeness tracking: $?" echo "Producing AOD" - taskwrapper aod.log o2-aod-producer-workflow $gloOpt --aod-writer-keep dangling --aod-writer-resfile "AO2D" --aod-writer-resmode UPDATE --aod-timeframe-id 1 --run-number 300000 + taskwrapper aod.log o2-aod-producer-workflow $gloOpt --aod-writer-keep dangling --aod-writer-resfile "AO2D" --aod-writer-resmode UPDATE --aod-timeframe-id 1 --run-number 300000 --collect-config-files echo "Return status of AOD production: $?" # let's do some very basic analysis tests (mainly to enlarge coverage in full CI) and enabled when SIM_CHALLENGE_ANATESTING=ON