Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Handle invalid entries in <data-access> of site-local-config.xml #40961

Merged
merged 1 commit into from Mar 16, 2023
Merged

Handle invalid entries in <data-access> of site-local-config.xml #40961

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 5 additions & 4 deletions FWCore/Catalog/src/FileLocator.cc
View file
Open in desktop
Expand Up @@ -268,8 +268,8 @@ namespace edm {
//let enforce that site-local-config.xml and storage.json contains valid catalogs in <data-access>, in which site defined in site-local-config.xml <data-access> should be found in storage.json
if (found_site == json.end()) {
cms::Exception ex("FileCatalog");
ex << "Can not find site and volume " << aCatalog.site << ", " << aCatalog.volume << " in " << filename_storage
<< ". Check site-local-config.xml <data-access> and storage.json";
ex << "Can not find storage site \"" << aCatalog.storageSite << "\" and volume \"" << aCatalog.volume
<< "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
ex.addContext("edm::FileLocator:init()");
throw ex;
}
Expand All @@ -283,8 +283,9 @@ namespace edm {
//let enforce that site-local-config.xml and storage.json contains valid catalogs, in which protocol defined in site-local-config.xml <data-access> should be found in storage.json
if (found_protocol == protocols.end()) {
cms::Exception ex("FileCatalog");
ex << "Can not find protocol " << aCatalog.protocol
<< " in storage.json. Check site-local-config.xml <data-access> and storage.json";
ex << "Can not find protocol \"" << aCatalog.protocol << "\" for the storage site \"" << aCatalog.storageSite
<< "\" and volume \"" << aCatalog.volume
<< "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
ex.addContext("edm::FileLocator:init()");
throw ex;
}
Expand Down
44 changes: 30 additions & 14 deletions FWCore/Catalog/src/InputFileCatalog.cc
View file
Open in desktop
Expand Up @@ -9,6 +9,7 @@

#include "FWCore/Utilities/interface/Exception.h"
#include "FWCore/Utilities/interface/EDMException.h"
#include "FWCore/MessageLogger/interface/MessageLogger.h"

#include <boost/algorithm/string.hpp>

Expand Down Expand Up @@ -56,8 +57,11 @@ namespace edm {
throw ex;
}

edm::CatalogAttributes inputOverride_struct(
tmps[0], tmps[1], tmps[2], tmps[3], tmps[4]); //site, subSite,storageSite,volume,protocol
edm::CatalogAttributes inputOverride_struct(tmps[0], //current-site
tmps[1], //current-subSite
tmps[2], //desired-data-access-site
tmps[3], //desired-data-access-volume
tmps[4]); //desired-data-access-protocol

overrideFileLocator_ =
std::make_unique<FileLocator>(inputOverride_struct); // propagate_const<T> has no reset() function
Expand All @@ -76,30 +80,42 @@ namespace edm {
std::vector<std::string> const& tmp_dataCatalogs = localconfservice->trivialDataCatalogs();
if (!fileLocators_trivalCatalog_.empty())
fileLocators_trivalCatalog_.clear();

//require the first file locator to success so obvious mistakes in data catalogs, typos for example, can be catched early. Note that tmp_dataCatalogs is not empty at this point. The protection is done inside the trivialDataCatalogs() of SiteLocalConfigService
fileLocators_trivalCatalog_.push_back(std::make_unique<FileLocator>(tmp_dataCatalogs.front()));

for (auto it = tmp_dataCatalogs.begin() + 1; it != tmp_dataCatalogs.end(); ++it) {
//Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init).
for (const auto& catalog : tmp_dataCatalogs) {
try {
fileLocators_trivalCatalog_.push_back(std::make_unique<FileLocator>(*it));
fileLocators_trivalCatalog_.push_back(std::make_unique<FileLocator>(catalog));
} catch (cms::Exception const& e) {
continue;
edm::LogWarning("InputFileCatalog")
<< "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
<< "Skip this catalog";
}
}
if (fileLocators_trivalCatalog_.empty()) {
cms::Exception ex("FileCatalog");
ex << "Unable to construct any file locator in InputFileCatalog::init";
ex.addContext("Calling edm::InputFileCatalog::init()");
throw ex;
}
} else if (catType == edm::CatalogType::RucioCatalog) {
std::vector<edm::CatalogAttributes> const& tmp_dataCatalogs = localconfservice->dataCatalogs();
if (!fileLocators_.empty())
fileLocators_.clear();
//require the first file locator to success so obvious mistakes in data catalogs, typos for example, can be catched early. Note that tmp_dataCatalogs is not empty at this point. The protection is done inside the dataCatalogs() of SiteLocalConfigService
fileLocators_.push_back(std::make_unique<FileLocator>(tmp_dataCatalogs.front()));
for (auto it = tmp_dataCatalogs.begin() + 1; it != tmp_dataCatalogs.end(); ++it) {
//Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init).
for (const auto& catalog : tmp_dataCatalogs) {
try {
fileLocators_.push_back(std::make_unique<FileLocator>(*it));
fileLocators_.push_back(std::make_unique<FileLocator>(catalog));
} catch (cms::Exception const& e) {
continue;
edm::LogWarning("InputFileCatalog")
<< "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
<< "Skip this catalog";
}
}
if (fileLocators_.empty()) {
cms::Exception ex("FileCatalog");
ex << "Unable to construct any file locator in InputFileCatalog::init";
ex.addContext("Calling edm::InputFileCatalog::init()");
throw ex;
}
} else {
cms::Exception ex("FileCatalog");
ex << "Undefined catalog type";
Expand Down