From dad7fc4a4a1e940ef3fc08aedc1fe77345787848 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Tue, 5 Oct 2021 11:04:16 +0200 Subject: [PATCH 1/2] [QC-592] A facility to store ROOT objects from files to QCDB This makes it possible to take QAResults.root generated by DPL Analysis Tasks and put it to QCDB. --- Framework/CMakeLists.txt | 9 +- Framework/src/runUploadRootObjects.cxx | 146 +++++++++++++++++++++++++ doc/Advanced.md | 44 +++++++- 3 files changed, 191 insertions(+), 8 deletions(-) create mode 100644 Framework/src/runUploadRootObjects.cxx diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt index f70ae76d4b..deade25baf 100644 --- a/Framework/CMakeLists.txt +++ b/Framework/CMakeLists.txt @@ -135,7 +135,8 @@ set(EXE_SRCS src/runPostProcessing.cxx src/runPostProcessingOCC.cxx src/runLocationCalculator.cxx - src/runMergerCalculator.cxx) + src/runMergerCalculator.cxx + src/runUploadRootObjects.cxx) set(EXE_NAMES o2-qc-run-producer @@ -149,7 +150,8 @@ set(EXE_NAMES o2-qc-run-postprocessing o2-qc-run-postprocessing-occ o2-qc-location-calculator - o2-qc-merger-calculator) + o2-qc-merger-calculator + o2-qc-upload-root-objects) # These were the original names before the convention changed. We will get rid # of them but for the time being we want to create symlinks to avoid confusion. @@ -166,7 +168,8 @@ set(EXE_OLD_NAMES qcRunPostProcessing qcRunPostProcessingOCC o2-qc-location-calculator - o2-qc-merger-calculator) + o2-qc-merger-calculator + o2-qc-upload-root-objects) # As per https://stackoverflow.com/questions/35765106/symbolic-links-cmake macro(install_symlink filepath sympath) diff --git a/Framework/src/runUploadRootObjects.cxx b/Framework/src/runUploadRootObjects.cxx new file mode 100644 index 0000000000..8b9218911e --- /dev/null +++ b/Framework/src/runUploadRootObjects.cxx @@ -0,0 +1,146 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file runUploadRootObjects.cxx +/// \author Piotr Konopka +/// +/// \brief This is an executable which reads ROOT files with objects and puts them to QCDB. +/// +/// This is an executable which reads QAResults.root generated by DPL analysis tasks and puts them to QCDB. +/// It will ignore the directory structure and put all objects in under the task name specified as the argument. +/// By default the current date and time will be used as the start of validity, and the object will be valid for 10 years. + +#include "QualityControl/QcInfoLogger.h" +#include "QualityControl/CcdbDatabase.h" +#include "QualityControl/MonitorObject.h" + +#include +#include +#include +#include +#include +#include + +namespace bpo = boost::program_options; +using namespace o2::quality_control::core; +using namespace o2::quality_control::repository; + +int main(int argc, const char* argv[]) +{ + size_t objectsUploaded = 0; + + try { + bpo::options_description desc{ "Options" }; + desc.add_options() // + ("help,h", "Help screen") // + ("input-file", bpo::value()->default_value("./QAResults.root"), "Path to the ROOT file with objects to insert.") // + ("qcdb-url", bpo::value()->default_value("ccdb-test.cern.ch:8080"), "URL to the QCDB.") // + ("task-name", bpo::value(), "Name of the task which the objects belong to. Use / to make directories") // + ("detector-code", bpo::value()->default_value("TST"), "3-letter detector code. Put AOD for analysis tasks") // + ("validity-start", bpo::value()->default_value(0), "Start of objects validity in ms since epoch") // + ("validity-end", bpo::value()->default_value(0), "End of objects validity in ms since epoch") // + ("run-number", bpo::value()->default_value(0), "Run number of objects (put 0 for trends across runs)") // + ("period-name", bpo::value()->default_value("unknown"), "Period name of the objects") // todo one could ask logbook + ("pass-name", bpo::value()->default_value("unknown"), "Calib/reco/sim pass name") // + ("provenance", bpo::value()->default_value("qc"), "Object path prefix used to mark if data comes from detector (use qc) or simulation (use qc_mc)"); + + bpo::variables_map vm; + store(parse_command_line(argc, argv, desc), vm); + notify(vm); + + if (vm.count("help")) { + // no infologger here, because the message is too long. + std::cout << desc << std::endl; + return 0; + } + + /// Read and validate arguments + auto inputFilePath = vm["input-file"].as(); + auto qcdbUrl = vm["qcdb-url"].as(); + auto taskName = vm["task-name"].as(); + auto detectorCode = vm["detector-code"].as(); + auto validityStart = vm["validity-start"].as(); + auto validityEnd = vm["validity-end"].as(); + auto runNumber = vm["run-number"].as(); + auto periodName = vm["period-name"].as(); + auto passName = vm["pass-name"].as(); + auto provenance = vm["provenance"].as(); + + if (validityStart == 0) { + validityStart = CcdbDatabase::getCurrentTimestamp(); + } + if (validityEnd == 0) { + validityEnd = validityStart + 1000ull * 60 * 60 * 24 * 365 * 10; + } + if (validityStart > validityEnd) { + throw std::runtime_error("Validity start (" + std::to_string(validityStart) + ") is further in the future than validity end (" + std::to_string(validityEnd) + ")"); + } + if (provenance != "qc" && provenance != "qc_mc") { + throw std::runtime_error("Provenance should be either 'qc' (for data from detector) or 'qc_mc' (for simulated data), while '" + provenance + "' was given."); + } + + /// Open ROOT file + auto* file = new TFile(inputFilePath.c_str(), "READ"); + if (file->IsZombie()) { + throw std::runtime_error("File '" + inputFilePath + "' is zombie."); + } + if (!file->IsOpen()) { + throw std::runtime_error("Failed to open the file: " + inputFilePath); + } + ILOG(Info) << "Input file '" << inputFilePath << "' successfully open." << ENDM; + + /// Open CCDB interface + CcdbDatabase database; + database.connect(qcdbUrl, "", "", ""); + + /// Upload the objects + std::function browseFileAndUpload = [&](TDirectoryFile* directory) { + TIter next(directory->GetListOfKeys()); + TKey* key; + while ((key = (TKey*)next())) { + auto storedTObj = directory->Get(key->GetName()); + if (storedTObj != nullptr) { + if (storedTObj->InheritsFrom("TDirectoryFile")) { + browseFileAndUpload(dynamic_cast(storedTObj)); + } else { + std::shared_ptr mo = std::make_shared(storedTObj, taskName, "unknown", detectorCode, runNumber, periodName, passName, provenance); + mo->setIsOwner(false); + database.storeMO(mo, validityStart, validityEnd); + objectsUploaded++; + } + } + delete storedTObj; + } + }; + + browseFileAndUpload(file); + + file->Close(); + delete file; + + database.disconnect(); + + } catch (const bpo::error& ex) { + ILOG(Error, Ops) << "Exception caught: " << ex.what() << ENDM; + return 1; + } catch (const boost::exception& ex) { + ILOG(Error, Ops) << "Exception caught: " << boost::current_exception_diagnostic_information(true) << ENDM; + return 1; + } + + if (objectsUploaded > 0) { + ILOG(Info, Support) << "Successfully uploaded " << objectsUploaded << " objects to the QCDB." << ENDM; + } else { + ILOG(Info, Support) << "No objects were uploaded to the QCDB. Maybe the file is empty?" << ENDM; + } + return 0; +} \ No newline at end of file diff --git a/doc/Advanced.md b/doc/Advanced.md index 63e700e394..57f0a316e5 100644 --- a/doc/Advanced.md +++ b/doc/Advanced.md @@ -17,7 +17,8 @@ Advanced topics * [Moving window](#moving-window) * [Writing a DPL data producer](#writing-a-dpl-data-producer) * [QC with DPL Analysis](#qc-with-dpl-analysis) - * [Getting AODs directly](#getting-aods-directly) + * [Uploading objects to QCDB](#uploading-objects-to-qcdb) + * [Getting AODs in QC Tasks](#getting-aods-in-qc-tasks) * [Merging with other analysis workflows](#merging-with-other-analysis-workflows) * [Enabling a workflow to run on Hyperloop](#enabling-a-workflow-to-run-on-hyperloop) * [CCDB / QCDB](#ccdb--qcdb) @@ -368,15 +369,48 @@ You will probably write it in your detector's O2 directory rather than in the QC ## QC with DPL Analysis -It is possible to attach QC to the Run 3 Analysis Tasks, as they use Data Processing Layer, just as -QC. AOD tables can be requested as direct data sources and then read by a QC task with -TableConsumer. One can also request AOD tables directly from an AOD file. +QC offers several ways to interact with the DPL Analysis framework. +One allows to [upload root objects generated](#uploading-objects-to-qcdb) by an Analysis Task into QCDB. +It is also possible to [run QC workflows alongside Analysis Tasks](#merging-with-other-analysis-workflows), + as they are also based on the Data Processing Layer. +AOD tables can be [requested as direct data sources](#getting-aods-in-qc-tasks) and then read by a QC task with TableConsumer. In this piece of documentation it is assumed that the users already have some idea about QC and [DPL Analysis](https://aliceo2group.github.io/analysis-framework), and they have an access to AOD files following the Run 3 data model. -### Getting AODs directly +### Uploading objects to QCDB + +To upload objects written to a file by an Analysis Task to QCDB, one may use the following command: +```shell script +o2-qc-upload-root-objects \ + --input-file ./QAResults.root \ + --qcdb-url ccdb-test.cern.ch:8080 \ + --task-name AnalysisFromFileTest \ + --detector-code TST \ + --provenance qc_mc \ + --pass-name passMC \ + --period-name SimChallenge \ + --run-number 49999 +``` + +See the `--help` message for explanation of the arguments. +If everything went well, the objects should be accessible in [the test QCG instance](https://qcg-test.cern.ch) under +the directories listed in the logs: +``` +2021-10-05 10:59:41.408998 QC infologger initialized +2021-10-05 10:59:41.409053 Input file './QAResults.root' successfully open. +... +2021-10-05 10:59:41.585893 Storing MonitorObject qc_mc/TST/MO/AnalysisFromFileTest/hMcEventCounter +2021-10-05 10:59:41.588649 Storing MonitorObject qc_mc/TST/MO/AnalysisFromFileTest/hGlobalBcFT0 +2021-10-05 10:59:41.591542 Storing MonitorObject qc_mc/TST/MO/AnalysisFromFileTest/hTimeT0Aall +2021-10-05 10:59:41.594386 Storing MonitorObject qc_mc/TST/MO/AnalysisFromFileTest/hTimeT0Call +2021-10-05 10:59:41.597743 Successfully uploaded 10 objects to the QCDB. +``` +Notice that the executable will ignore the directory structure in the input file and upload all objects to one directory. +If you need a different behaviour, please contact the developers. + +### Getting AODs in QC Tasks First, let's see how to get data directly from an AOD file. To read the table, we will use TableConsumer from DPL, as in [the example of a QC analysis task](../Modules/Example/src/AnalysisTask.cxx): From f3a4ba77a8e4215d944727af8a505a9e54246b12 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Tue, 5 Oct 2021 11:38:27 +0200 Subject: [PATCH 2/2] fix english, no default for run number --- Framework/src/runUploadRootObjects.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Framework/src/runUploadRootObjects.cxx b/Framework/src/runUploadRootObjects.cxx index 8b9218911e..43271e3ceb 100644 --- a/Framework/src/runUploadRootObjects.cxx +++ b/Framework/src/runUploadRootObjects.cxx @@ -44,11 +44,11 @@ int main(int argc, const char* argv[]) ("help,h", "Help screen") // ("input-file", bpo::value()->default_value("./QAResults.root"), "Path to the ROOT file with objects to insert.") // ("qcdb-url", bpo::value()->default_value("ccdb-test.cern.ch:8080"), "URL to the QCDB.") // - ("task-name", bpo::value(), "Name of the task which the objects belong to. Use / to make directories") // + ("task-name", bpo::value(), "Name of the task to which the objects belong. Use / to make directories") // ("detector-code", bpo::value()->default_value("TST"), "3-letter detector code. Put AOD for analysis tasks") // ("validity-start", bpo::value()->default_value(0), "Start of objects validity in ms since epoch") // ("validity-end", bpo::value()->default_value(0), "End of objects validity in ms since epoch") // - ("run-number", bpo::value()->default_value(0), "Run number of objects (put 0 for trends across runs)") // + ("run-number", bpo::value(), "Run number of objects (put 0 for many runs)") // ("period-name", bpo::value()->default_value("unknown"), "Period name of the objects") // todo one could ask logbook ("pass-name", bpo::value()->default_value("unknown"), "Calib/reco/sim pass name") // ("provenance", bpo::value()->default_value("qc"), "Object path prefix used to mark if data comes from detector (use qc) or simulation (use qc_mc)");