Add test for cross-site catalog access

cms-sw · Jan 19, 2023 · 0715cf0 · 0715cf0
1 parent c0c2b19
commit 0715cf0
Show file tree

Hide file tree

Showing 8 changed files with 128 additions and 4 deletions.
diff --git a/FWCore/Catalog/src/FileLocator.cc b/FWCore/Catalog/src/FileLocator.cc
@@ -268,8 +268,8 @@ namespace edm {
     //let enforce that site-local-config.xml and storage.json contains valid catalogs in <data-access>, in which site defined in site-local-config.xml <data-access> should be found in storage.json
     if (found_site == json.end()) {
       cms::Exception ex("FileCatalog");
-      ex << "Can not find site and volume " << aCatalog.site << ", " << aCatalog.volume
-         << " in storage.json. Check site-local-config.xml <data-access> and storage.json";
+      ex << "Can not find site and volume " << aCatalog.site << ", " << aCatalog.volume << " in " << filename_storage
+         << ". Check site-local-config.xml <data-access> and storage.json";
       ex.addContext("edm::FileLocator:init()");
       throw ex;
     }

diff --git a/FWCore/Services/test/BuildFile.xml b/FWCore/Services/test/BuildFile.xml
@@ -3,7 +3,7 @@
   <use name="FWCore/Framework"/>
 </library>
 
-<library file="SiteLocalConfigServiceTester.cc" name="SiteLocalConfigUnitTestClient">
+<library file="SiteLocalConfigServiceTester.cc SiteLocalConfigServiceCatalogTester.cc" name="SiteLocalConfigUnitTestClient">
   <flags EDM_PLUGIN="1"/>
   <use name="FWCore/Services"/>
   <use name="FWCore/Framework"/>

diff --git a/FWCore/Services/test/SiteLocalConfigServiceCatalogTester.cc b/FWCore/Services/test/SiteLocalConfigServiceCatalogTester.cc
@@ -0,0 +1,45 @@
+#include "FWCore/Framework/interface/global/EDAnalyzer.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
+#include "FWCore/Catalog/interface/SiteLocalConfig.h"
+#include "FWCore/Catalog/interface/InputFileCatalog.h"
+#include "FWCore/Framework/interface/MakerMacros.h"
+#include "FWCore/Utilities/interface/Exception.h"
+
+#include <string>
+
+namespace edmtest {
+  class SiteLocalConfigServiceCatalogTester : public edm::global::EDAnalyzer<> {
+  public:
+    SiteLocalConfigServiceCatalogTester(const edm::ParameterSet& iPSet);
+
+    void analyze(edm::StreamID, const edm::Event&, const edm::EventSetup&) const override {}
+  };
+
+  SiteLocalConfigServiceCatalogTester::SiteLocalConfigServiceCatalogTester(const edm::ParameterSet& iPSet) {
+    std::string const overrideCatalog;
+
+    auto const& files = iPSet.getUntrackedParameter<std::vector<edm::ParameterSet>>("files");
+    for (auto const& filePSet : files) {
+      auto const& fileName = filePSet.getUntrackedParameter<std::string>("file");
+      unsigned int catalogIndex = filePSet.getUntrackedParameter<unsigned int>("catalogIndex");
+      auto const& expectResult = filePSet.getUntrackedParameter<std::string>("expectResult");
+
+      edm::InputFileCatalog catalog{std::vector{fileName}, overrideCatalog};
+      edm::FileCatalogItem const& item = catalog.fileCatalogItems()[0];
+      if (catalogIndex >= item.fileNames().size()) {
+        throw cms::Exception("Assert") << "Asked catalog " << catalogIndex << " from InputFileCatalog that had only "
+                                       << item.fileNames().size() << " entries";
+      }
+      auto const& result = item.fileName(catalogIndex);
+
+      if (result != expectResult) {
+        throw cms::Exception("Assert") << "InputFileCatalog gave '" << result << "' for catalog " << catalogIndex
+                                       << ", expected '" << expectResult << "'";
+      }
+    }
+  }
+}  // namespace edmtest
+
+using SiteLocalConfigServiceCatalogTester = edmtest::SiteLocalConfigServiceCatalogTester;
+DEFINE_FWK_MODULE(SiteLocalConfigServiceCatalogTester);
diff --git a/FWCore/Services/test/cross-site-local-config.testfile b/FWCore/Services/test/cross-site-local-config.testfile
@@ -0,0 +1,22 @@
+<site-local-config>
+<site name="DUMMY">
+  <event-data>
+    <catalog url="trivialcatalog_file:/dummy/storage.xml?protocol=dcap"/>
+  </event-data>
+  <data-access>
+    <catalog volume="LocalVolume" protocol="XRootD"/>
+    <catalog site="DUMMY_CROSS_SITE" volume="CMSXrootdFederation" protocol="XRootD"/>
+  </data-access>
+  <local-stage-out>
+    <se-name value="cmssrm.dummy.foo"/>
+    <command value="srm"/>
+    <catalog url="trivialcatalog_file:/dummy/storage.xml?protocol=srm"/>
+  </local-stage-out>
+  <calib-data>
+    <frontier-connect>
+      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
+      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
+    </frontier-connect>
+  </calib-data>
+</site>
+</site-local-config>
diff --git a/FWCore/Services/test/dummy_storage.json b/FWCore/Services/test/dummy_storage.json
@@ -0,0 +1,19 @@
+[
+   {  "site": "DUMMY",
+      "volume": "LocalVolume",
+      "protocols": [
+         {  "protocol": "XRootD",
+            "access": "global-rw",
+            "comment": "xrootd write to dCache/EOS endpoint directly",
+            "rules": [
+               {  "lfn": "/+store/temp/user/(.*)",
+                  "pfn": "root://cmseos.fnal.gov//eos/uscms/store/temp/user/$1"
+               },
+               {  "lfn": "/+store/(.*)",
+                  "pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1"
+               }
+            ]
+         }
+      ]
+   }
+]
diff --git a/FWCore/Services/test/dummycross_storage.json b/FWCore/Services/test/dummycross_storage.json
@@ -0,0 +1,11 @@
+[
+   {  "site": "DUMMY_CROSS_SITE",
+      "volume": "CMSXrootdFederation",
+      "protocols": [
+         {  "protocol": "XRootD",
+            "access": "global-ro",
+            "prefix": "root://xrootd-cms.infn.it/"
+         }
+      ]
+   }
+]
diff --git a/FWCore/Services/test/test_sitelocalconfig.sh b/FWCore/Services/test/test_sitelocalconfig.sh
@@ -6,6 +6,7 @@ function die { echo $1: status $2 ;  exit $2; }
 mkdir -p ${CMSSW_BASE}/test/SITECONF
 mkdir -p ${CMSSW_BASE}/test/SITECONF/local
 mkdir -p ${CMSSW_BASE}/test/SITECONF/local/JobConfig
+mkdir -p ${CMSSW_BASE}/test/SITECONF/DUMMY_CROSS_SITE
 
 export SITECONFIG_PATH=${CMSSW_BASE}/test/SITECONF/local
 
@@ -24,5 +25,9 @@ cp ${LOCAL_TEST_DIR}/no-source-site-local-config.testfile ${CMSSW_BASE}/test/SIT
 F3=${LOCAL_TEST_DIR}/test_sitelocalconfig_override_cfg.py
 (cmsRun $F3 ) || die "Failure using $F3 with no-source site-local-config" $?
 
-
+cp ${LOCAL_TEST_DIR}/cross-site-local-config.testfile  ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml
+cp ${LOCAL_TEST_DIR}/dummy_storage.json ${CMSSW_BASE}/test/SITECONF/local/storage.json
+cp ${LOCAL_TEST_DIR}/dummycross_storage.json ${CMSSW_BASE}/test/SITECONF/DUMMY_CROSS_SITE/storage.json
+F4=${LOCAL_TEST_DIR}/test_sitelocalconfig_catalog_cfg.py
+(cmsRun $F4 ) || die "Failure using $F4" $?
 
diff --git a/FWCore/Services/test/test_sitelocalconfig_catalog_cfg.py b/FWCore/Services/test/test_sitelocalconfig_catalog_cfg.py
@@ -0,0 +1,22 @@
+import FWCore.ParameterSet.Config as cms
+process = cms.Process("TEST")
+
+process.source = cms.Source("EmptySource")
+process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(1))
+
+process.tester = cms.EDAnalyzer("SiteLocalConfigServiceCatalogTester",
+    files = cms.untracked.VPSet(
+        cms.untracked.PSet(
+            file = cms.untracked.string("/store/a/b.root"),
+            catalogIndex = cms.untracked.uint32(0),
+            expectResult = cms.untracked.string("root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/a/b.root")
+        ),
+        cms.untracked.PSet(
+            file = cms.untracked.string("/store/a/b.root"),
+            catalogIndex = cms.untracked.uint32(1),
+            expectResult = cms.untracked.string("root://xrootd-cms.infn.it//store/a/b.root")
+        ),
+    )
+)
+
+process.o = cms.EndPath(process.tester)