Add test for cross-site catalog access

FWCore/Catalog/src/FileLocator.cc

@@ -268,8 +268,8 @@ namespace edm {
     //let enforce that site-local-config.xml and storage.json contains valid catalogs in <data-access>, in which site defined in site-local-config.xml <data-access> should be found in storage.json
     if (found_site == json.end()) {
       cms::Exception ex("FileCatalog");
-      ex << "Can not find site and volume " << aCatalog.site << ", " << aCatalog.volume
-         << " in storage.json. Check site-local-config.xml <data-access> and storage.json";
+      ex << "Can not find site and volume " << aCatalog.site << ", " << aCatalog.volume << " in " << filename_storage
+         << ". Check site-local-config.xml <data-access> and storage.json";
       ex.addContext("edm::FileLocator:init()");
       throw ex;
     }

FWCore/Services/test/BuildFile.xml

@@ -3,7 +3,7 @@
   <use name="FWCore/Framework"/>
 </library>
 
-<library file="SiteLocalConfigServiceTester.cc" name="SiteLocalConfigUnitTestClient">
+<library file="SiteLocalConfigServiceTester.cc SiteLocalConfigServiceCatalogTester.cc" name="SiteLocalConfigUnitTestClient">
   <flags EDM_PLUGIN="1"/>
   <use name="FWCore/Services"/>
   <use name="FWCore/Framework"/>

FWCore/Services/test/SiteLocalConfigServiceCatalogTester.cc

@@ -0,0 +1,45 @@
+#include "FWCore/Framework/interface/global/EDAnalyzer.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
+#include "FWCore/Catalog/interface/SiteLocalConfig.h"
+#include "FWCore/Catalog/interface/InputFileCatalog.h"
+#include "FWCore/Framework/interface/MakerMacros.h"
+#include "FWCore/Utilities/interface/Exception.h"
+
+#include <string>
+
+namespace edmtest {
+  class SiteLocalConfigServiceCatalogTester : public edm::global::EDAnalyzer<> {
+  public:
+    SiteLocalConfigServiceCatalogTester(const edm::ParameterSet& iPSet);
+
+    void analyze(edm::StreamID, const edm::Event&, const edm::EventSetup&) const override {}
+  };
+
+  SiteLocalConfigServiceCatalogTester::SiteLocalConfigServiceCatalogTester(const edm::ParameterSet& iPSet) {
+    std::string const overrideCatalog;
+
+    auto const& files = iPSet.getUntrackedParameter<std::vector<edm::ParameterSet>>("files");
+    for (auto const& filePSet : files) {
+      auto const& fileName = filePSet.getUntrackedParameter<std::string>("file");
+      unsigned int catalogIndex = filePSet.getUntrackedParameter<unsigned int>("catalogIndex");
+      auto const& expectResult = filePSet.getUntrackedParameter<std::string>("expectResult");
+
+      edm::InputFileCatalog catalog{std::vector{fileName}, overrideCatalog};
+      edm::FileCatalogItem const& item = catalog.fileCatalogItems()[0];
+      if (catalogIndex >= item.fileNames().size()) {
+        throw cms::Exception("Assert") << "Asked catalog " << catalogIndex << " from InputFileCatalog that had only "
+                                       << item.fileNames().size() << " entries";
+      }
+      auto const& result = item.fileName(catalogIndex);
+
+      if (result != expectResult) {
+        throw cms::Exception("Assert") << "InputFileCatalog gave '" << result << "' for catalog " << catalogIndex
+                                       << ", expected '" << expectResult << "'";
+      }
+    }
+  }
+}  // namespace edmtest
+
+using SiteLocalConfigServiceCatalogTester = edmtest::SiteLocalConfigServiceCatalogTester;
+DEFINE_FWK_MODULE(SiteLocalConfigServiceCatalogTester);

FWCore/Services/test/sitelocalconfig/catalog/dummycross/storage.json

@@ -0,0 +1,11 @@
+[
+   {  "site": "DUMMY_CROSS_SITE",
+      "volume": "CMSXrootdFederation",
+      "protocols": [
+         {  "protocol": "XRootD",
+            "access": "global-ro",
+            "prefix": "root://xrootd-cms.infn.it/"
+         }
+      ]
+   }
+]

FWCore/Services/test/sitelocalconfig/catalog/local/storage.json

@@ -0,0 +1,19 @@
+[
+   {  "site": "DUMMY",
+      "volume": "LocalVolume",
+      "protocols": [
+         {  "protocol": "XRootD",
+            "access": "global-rw",
+            "comment": "xrootd write to dCache/EOS endpoint directly",
+            "rules": [
+               {  "lfn": "/+store/temp/user/(.*)",
+                  "pfn": "root://cmseos.fnal.gov//eos/uscms/store/temp/user/$1"
+               },
+               {  "lfn": "/+store/(.*)",
+                  "pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1"
+               }
+            ]
+         }
+      ]
+   }
+]

FWCore/Services/test/sitelocalconfig/catalog/site-local-config.xml

@@ -0,0 +1,22 @@
+<site-local-config>
+<site name="DUMMY">
+  <event-data>
+    <catalog url="trivialcatalog_file:/dummy/storage.xml?protocol=dcap"/>
+  </event-data>
+  <data-access>
+    <catalog volume="LocalVolume" protocol="XRootD"/>
+    <catalog site="DUMMY_CROSS_SITE" volume="CMSXrootdFederation" protocol="XRootD"/>
+  </data-access>
+  <local-stage-out>
+    <se-name value="cmssrm.dummy.foo"/>
+    <command value="srm"/>
+    <catalog url="trivialcatalog_file:/dummy/storage.xml?protocol=srm"/>
+  </local-stage-out>
+  <calib-data>
+    <frontier-connect>
+      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
+      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
+    </frontier-connect>
+  </calib-data>
+</site>
+</site-local-config>

FWCore/Services/test/test_catch2_SiteLocalConfigService.cc

@@ -19,9 +19,10 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") {
     }
   }
   REQUIRE(not dirString.empty());
-  SECTION("full-site-local-config.testfile") {
+  SECTION("sitelocalconfig/full/site-local-config.xml") {
     edm::ParameterSet pset;
-    pset.addUntrackedParameter<std::string>("siteLocalConfigFileUrl", dirString + "/full-site-local-config.testfile");
+    pset.addUntrackedParameter<std::string>("siteLocalConfigFileUrl",
+                                            dirString + "/sitelocalconfig/full/site-local-config.xml");
 
     edm::service::SiteLocalConfigService slc(pset);
 
@@ -62,7 +63,8 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") {
 
   SECTION("overrides") {
     edm::ParameterSet pset;
-    pset.addUntrackedParameter<std::string>("siteLocalConfigFileUrl", dirString + "/full-site-local-config.testfile");
+    pset.addUntrackedParameter<std::string>("siteLocalConfigFileUrl",
+                                            dirString + "/sitelocalconfig/full/site-local-config.xml");
 
     pset.addUntrackedParameter<std::string>("overrideSourceCacheTempDir", "/a/d");
     pset.addUntrackedParameter<double>("overrideSourceCacheMinFree", 10.);
@@ -117,10 +119,10 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") {
     REQUIRE(slc.localConnectSuffix() == "OverrideSuffix");
   }
 
-  SECTION("throwtest-site-local-config.testfile") {
+  SECTION("sitelocalconfig/throw/site-local-config.xml") {
     edm::ParameterSet pset;
     pset.addUntrackedParameter<std::string>("siteLocalConfigFileUrl",
-                                            dirString + "/throwtest-site-local-config.testfile");
+                                            dirString + "/sitelocalconfig/throw/site-local-config.xml");
 
     REQUIRE_THROWS_AS(edm::service::SiteLocalConfigService(pset), cms::Exception);
   }

FWCore/Services/test/test_sitelocalconfig.sh

@@ -6,23 +6,28 @@ function die { echo $1: status $2 ;  exit $2; }
 mkdir -p ${CMSSW_BASE}/test/SITECONF
 mkdir -p ${CMSSW_BASE}/test/SITECONF/local
 mkdir -p ${CMSSW_BASE}/test/SITECONF/local/JobConfig
+mkdir -p ${CMSSW_BASE}/test/SITECONF/DUMMY_CROSS_SITE
 
 export SITECONFIG_PATH=${CMSSW_BASE}/test/SITECONF/local
 
-cp ${LOCAL_TEST_DIR}/no-source-site-local-config.testfile ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml
+cp ${LOCAL_TEST_DIR}/sitelocalconfig/no_source/site-local-config.xml ${CMSSW_BASE}/test/SITECONF/local/JobConfig/
 F1=${LOCAL_TEST_DIR}/test_sitelocalconfig_no_source_cfg.py
 (cmsRun $F1 ) || die "Failure using $F1" $?
 
-cp ${LOCAL_TEST_DIR}/source-site-local-config.testfile ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml
+cp ${LOCAL_TEST_DIR}/sitelocalconfig/source/site-local-config.xml ${CMSSW_BASE}/test/SITECONF/local/JobConfig/
 F2=${LOCAL_TEST_DIR}/test_sitelocalconfig_source_cfg.py
 (cmsRun $F2 ) || die "Failure using $F2" $?
 
 F3=${LOCAL_TEST_DIR}/test_sitelocalconfig_override_cfg.py
 (cmsRun $F3 ) || die "Failure using $F3" $?
 
-cp ${LOCAL_TEST_DIR}/no-source-site-local-config.testfile ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml
+cp ${LOCAL_TEST_DIR}/sitelocalconfig/no_source/site-local-config.xml ${CMSSW_BASE}/test/SITECONF/local/JobConfig/
 F3=${LOCAL_TEST_DIR}/test_sitelocalconfig_override_cfg.py
 (cmsRun $F3 ) || die "Failure using $F3 with no-source site-local-config" $?
 
-
+cp ${LOCAL_TEST_DIR}/sitelocalconfig/catalog/site-local-config.xml  ${CMSSW_BASE}/test/SITECONF/local/JobConfig/
+cp ${LOCAL_TEST_DIR}/sitelocalconfig/catalog/local/storage.json ${CMSSW_BASE}/test/SITECONF/local/
+cp ${LOCAL_TEST_DIR}/sitelocalconfig/catalog/dummycross/storage.json ${CMSSW_BASE}/test/SITECONF/DUMMY_CROSS_SITE/
+F4=${LOCAL_TEST_DIR}/test_sitelocalconfig_catalog_cfg.py
+(cmsRun $F4 ) || die "Failure using $F4" $?
 

FWCore/Services/test/test_sitelocalconfig_catalog_cfg.py

@@ -0,0 +1,22 @@
+import FWCore.ParameterSet.Config as cms
+process = cms.Process("TEST")
+
+process.source = cms.Source("EmptySource")
+process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(1))
+
+process.tester = cms.EDAnalyzer("SiteLocalConfigServiceCatalogTester",
+    files = cms.untracked.VPSet(
+        cms.untracked.PSet(
+            file = cms.untracked.string("/store/a/b.root"),
+            catalogIndex = cms.untracked.uint32(0),
+            expectResult = cms.untracked.string("root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/a/b.root")
+        ),
+        cms.untracked.PSet(
+            file = cms.untracked.string("/store/a/b.root"),
+            catalogIndex = cms.untracked.uint32(1),
+            expectResult = cms.untracked.string("root://xrootd-cms.infn.it//store/a/b.root")
+        ),
+    )
+)
+
+process.o = cms.EndPath(process.tester)