Initialize BranchDescription from Dictionary only if the branch is pr…

…esent

DataFormats/Common/src/setIsMergeable.cc

@@ -17,7 +17,8 @@ namespace edm {
     // data member can only be true for run or lumi products.
     // It defaults to false. Also if it is true that means it
     // was already set.
-    if (desc.branchType() == InRun || desc.branchType() == InLumi) {
+    // Set it only for branches that are present
+    if (desc.present() and (desc.branchType() == InRun or desc.branchType() == InLumi)) {
       if (!desc.isMergeable()) {
         TClass* wrapperBaseTClass = TypeWithDict::byName("edm::WrapperBase").getClass();
         TClass* tClass = desc.wrappedType().getClass();

DataFormats/Provenance/interface/BranchDescription.h

@@ -158,9 +158,11 @@ namespace edm {
       BranchID switchAliasForBranchID_;
 
       // A TypeWithDict object for the wrapped object
+      // This is set if and only if the dropped_ is false
       TypeWithDict wrappedType_;
 
       // A TypeWithDict object for the unwrapped object
+      // This is set if and only if the dropped_ is false
       TypeWithDict unwrappedType_;
 
       // The split level of the branch, as marked

DataFormats/TestObjects/interface/ToyProducts.h

@@ -63,6 +63,15 @@ namespace edmtest {
     ProductWithNoDictionary dummy;
   };
 
+  template <int TAG>
+  struct TransientIntParentT {
+    explicit TransientIntParentT(int i = 0) : value(i) {}
+
+    cms_int32_t value;
+  };
+  constexpr int TransientIntParentTag = 1;
+  using TransientIntParent = TransientIntParentT<TransientIntParentTag>;
+
   struct Int16_tProduct {
     explicit Int16_tProduct(int16_t i = 0, uint16_t j = 1) : value(i), uvalue(j) {}
     ~Int16_tProduct() {}

DataFormats/TestObjects/test/BuildFile_extra.xml

@@ -0,0 +1,9 @@
+<library file="" name="DataFormatsTestObjectsParent1">
+  <flags LCG_DICT_HEADER="classes.h"/>
+  <flags LCG_DICT_XML="classes_def_parent1.xml"/>
+</library>
+
+<library file="" name="DataFormatsTestObjectsParent2">
+  <flags LCG_DICT_HEADER="classes.h"/>
+  <flags LCG_DICT_XML="classes_def_parent2.xml"/>
+</library>

DataFormats/TestObjects/test/classes.h

@@ -0,0 +1,2 @@
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/TestObjects/interface/ToyProducts.h"

DataFormats/TestObjects/test/classes_def_parent1.xml

@@ -0,0 +1,4 @@
+<lcgdict>
+ <class name="edmtest::TransientIntParentT<1>" persistent="false"/>
+ <class name="edm::Wrapper<edmtest::TransientIntParentT<1>>" persistent="false"/>
+</lcgdict>

DataFormats/TestObjects/test/classes_def_parent2.xml

@@ -0,0 +1,4 @@
+<lcgdict>
+ <class name="edmtest::TransientIntParentT<2>" persistent="false"/>
+ <class name="edm::Wrapper<edmtest::TransientIntParentT<2>>" persistent="false"/>
+</lcgdict>

FWCore/Framework/interface/stream/ThinningProducer.h

@@ -160,6 +160,12 @@ namespace edm {
     ProductRegistry::ProductList const& productList = productRegistry.productList();
     for (auto const& product : productList) {
       BranchDescription const& desc = product.second;
+      if (desc.dropped()) {
+        // Dropped branch does not have type information, but they can
+        // be ignored here because all of the parent/thinned/association
+        // branches are expected to be present
+        continue;
+      }
       if (desc.unwrappedType().typeInfo() == typeid(Collection)) {
         if (desc.produced() && desc.moduleLabel() == moduleDescription().moduleLabel() &&
             desc.productInstanceName().empty()) {

FWCore/Framework/test/stubs/ToyIntProducers.cc

@@ -350,6 +350,46 @@ namespace edmtest {
     e.emplace(putToken_, result.product()->value);
   }
 
+  //--------------------------------------------------------------------
+  //
+  // Produces a TransientIntParent instance.
+  //
+  class TransientIntParentProducer : public edm::global::EDProducer<> {
+  public:
+    explicit TransientIntParentProducer(edm::ParameterSet const& p)
+        : token_{produces<TransientIntParent>()}, value_(p.getParameter<int>("ivalue")) {}
+    void produce(edm::StreamID, edm::Event& e, edm::EventSetup const& c) const override;
+
+  private:
+    const edm::EDPutTokenT<TransientIntParent> token_;
+    const int value_;
+  };
+
+  void TransientIntParentProducer::produce(edm::StreamID, edm::Event& e, edm::EventSetup const&) const {
+    // EventSetup is not used.
+    e.emplace(token_, value_);
+  }
+
+  //--------------------------------------------------------------------
+  //
+  // Produces a IntProduct instance from a TransientIntParent
+  //
+  class IntProducerFromTransientParent : public edm::global::EDProducer<> {
+  public:
+    explicit IntProducerFromTransientParent(edm::ParameterSet const& p)
+        : putToken_{produces<IntProduct>()}, getToken_{consumes(p.getParameter<edm::InputTag>("src"))} {}
+    void produce(edm::StreamID, edm::Event& e, edm::EventSetup const& c) const override;
+
+  private:
+    const edm::EDPutTokenT<IntProduct> putToken_;
+    const edm::EDGetTokenT<TransientIntParent> getToken_;
+  };
+
+  void IntProducerFromTransientParent::produce(edm::StreamID, edm::Event& e, edm::EventSetup const&) const {
+    // EventSetup is not used.
+    e.emplace(putToken_, e.get(getToken_).value);
+  }
+
   //--------------------------------------------------------------------
   //
   // Produces an Int16_tProduct instance.
@@ -907,6 +947,8 @@ DEFINE_FWK_MODULE(ConsumingIntProducer);
 DEFINE_FWK_MODULE(EventNumberIntProducer);
 DEFINE_FWK_MODULE(TransientIntProducer);
 DEFINE_FWK_MODULE(IntProducerFromTransient);
+DEFINE_FWK_MODULE(edmtest::TransientIntParentProducer);
+DEFINE_FWK_MODULE(edmtest::IntProducerFromTransientParent);
 DEFINE_FWK_MODULE(Int16_tProducer);
 DEFINE_FWK_MODULE(AddIntsProducer);
 DEFINE_FWK_MODULE(AddAllIntsProducer);

IOPool/Input/src/PoolSource.cc

@@ -121,6 +121,10 @@ namespace edm {
           //now make sure this is marked as not dropped else the product will not be 'get'table from the Event
           auto itFound = fullList.find(item.first);
           if (itFound != fullList.end()) {
+            // If the branch in primary file was dropped, need to initilize the dictionary information
+            if (itFound->second.dropped()) {
+              itFound->second.initFromDictionary();
+            }
             itFound->second.setDropped(false);
           }
         }

IOPool/Input/src/RootFile.cc

@@ -1898,7 +1898,11 @@ namespace edm {
     ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
     for (auto& product : pList) {
       BranchDescription& prod = product.second;
-      prod.init();
+      // Initialize BranchDescription from dictionary only if the
+      // branch is present. This allows a subsequent job to process
+      // data where a dictionary of a transient parent branch has been
+      // removed from the release after the file has been written.
+      prod.initBranchName();
       if (prod.branchType() == InProcess) {
         std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
         auto it = std::find(processes.begin(), processes.end(), prod.processName());
@@ -1913,6 +1917,9 @@ namespace edm {
       } else {
         treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName()));
       }
+      if (prod.present()) {
+        prod.initFromDictionary();
+      }
     }
   }
 
@@ -2053,7 +2060,7 @@ namespace edm {
       TString tString;
       for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
         BranchDescription const& prod = it->second;
-        if (prod.branchType() != InEvent && prod.branchType() != InProcess) {
+        if (prod.present() and prod.branchType() != InEvent and prod.branchType() != InProcess) {
           TClass* cp = prod.wrappedType().getClass();
           void* p = cp->New();
           int offset = cp->GetBaseClassOffset(edProductClass_);

IOPool/Input/test/BuildFile.xml

@@ -11,4 +11,5 @@
   </library>
 
  <test name="TestIOPoolInputRepeating" command="testRepeatingCachedRootSource.sh"/>
+ <test name="TestIOPoolInputNoParentDictionary" command="testNoParentDictionary.sh"/>
 </environment>

IOPool/Input/test/PoolNoParentDictionaryTestStep1_cfg.py

@@ -0,0 +1,24 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("TESTPROD")
+process.maxEvents.input = 11
+
+process.source = cms.Source("EmptySource",
+    firstLuminosityBlock = cms.untracked.uint32(6),
+    numberEventsInLuminosityBlock = cms.untracked.uint32(3),
+    firstRun = cms.untracked.uint32(561),
+    numberEventsInRun = cms.untracked.uint32(7)
+)
+
+process.parentIntProduct = cms.EDProducer("edmtest::TransientIntParentProducer", ivalue = cms.int32(1))
+
+process.intProduct = cms.EDProducer("edmtest::IntProducerFromTransientParent",
+    src = cms.InputTag("parentIntProduct")
+)
+
+process.output = cms.OutputModule("PoolOutputModule",
+    fileName = cms.untracked.string('noparentdict_step1.root')
+)
+
+process.p = cms.Path(process.parentIntProduct+process.intProduct)
+process.ep = cms.EndPath(process.output)

IOPool/Input/test/PoolNoParentDictionaryTestStep2_cfg.py

@@ -0,0 +1,17 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("TESTANALYSIS")
+
+process.maxEvents.input = -1
+
+process.source = cms.Source("PoolSource",
+    setRunNumber = cms.untracked.uint32(621),
+    fileNames = cms.untracked.vstring('file:noparentdict_step1.root')
+)
+
+process.analysis = cms.EDAnalyzer("IntTestAnalyzer",
+    valueMustMatch = cms.untracked.int32(1),
+    moduleLabel = cms.untracked.InputTag("intProduct"),
+)
+
+process.p = cms.Path(process.analysis)

IOPool/Input/test/testNoParentDictionary.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -e
+
+function die { echo $1: status $2 ;  exit $2; }
+
+SCRAM_TEST_NAME=TestIOPoolInputNoParentDictionary
+rm -rf $SCRAM_TEST_NAME
+mkdir $SCRAM_TEST_NAME
+cd $SCRAM_TEST_NAME
+
+# Create a new CMSSW dev area
+OLD_CMSSW_BASE=${CMSSW_BASE}
+LD_LIBRARY_PATH_TO_OLD=$(echo $LD_LIBRARY_PATH | tr ':' '\n' | grep "^${CMSSW_BASE}/" | tr '\n' ':')
+scram -a $SCRAM_ARCH project $CMSSW_VERSION
+pushd $CMSSW_VERSION/src
+eval `scram run -sh`
+
+# Copy DataFormats/TestObjects code to be able to edit it to make ROOT header parsing to fail
+for DIR in ${OLD_CMSSW_BASE} ${CMSSW_RELEASE_BASE} ${CMSSW_FULL_RELEASE_BASE} ; do
+    if [ -d ${DIR}/src/DataFormats/TestObjects ]; then
+        mkdir DataFormats
+        cp -r ${DIR}/src/DataFormats/TestObjects DataFormats/
+        break
+    fi
+done
+if [ ! -e DataFormats/TestObjects ]; then
+    echo "Failed to copy DataFormats/TestObjects from local or release area"
+    exit 1;
+fi
+
+# Enable TransientIntParentT dictionaries
+cat DataFormats/TestObjects/test/BuildFile_extra.xml >> DataFormats/TestObjects/test/BuildFile.xml
+scram build -j $(nproc)
+
+popd
+
+# Add OLD_CMSSW_BASE in between CMSSW_BASE and CMSSW_RELEASE_BASE for
+# LD_LIBRARY_PATH and ROOT_INCLUDE_PATH
+export LD_LIBRARY_PATH=$(echo -n ${LD_LIBRARY_PATH} | sed -e "s|${CMSSW_BASE}/external/${SCRAM_ARCH}/lib:|${CMSSW_BASE}/external/${SCRAM_ARCH}/lib:${LD_LIBRARY_PATH_TO_OLD}:|")
+export ROOT_INCLUDE_PATH=$(echo -n ${ROOT_INCLUDE_PATH} | sed -e "s|${CMSSW_BASE}/src:|${CMSSW_BASE}/src:${OLD_CMSSW_BASE}/src:|")
+
+echo "Produce a file with TransientIntParentT<1> product"
+cmsRun ${LOCALTOP}/src/IOPool/Input/test/PoolNoParentDictionaryTestStep1_cfg.py || die 'Failed cmsRun PoolNoParentDictionaryTestStep1_cfg.py' $?
+
+
+# Then make attempt to load TransientIntParentT<1> to fail
+echo "PREVENT_HEADER_PARSING" >> ${CMSSW_BASE}/src/DataFormats/TestObjects/interface/ToyProducts.h
+rm ${CMSSW_BASE}/lib/${SCRAM_ARCH}/DataFormatsTestObjectsParent1_xr_rdict.pcm ${CMSSW_BASE}/lib/${SCRAM_ARCH}/libDataFormatsTestObjectsParent1.so
+sed -i -e 's/libDataFormatsTestObjectsParent1.so/libDataFormatsTestObjectsParent2.so/' ${CMSSW_BASE}/lib/${SCRAM_ARCH}/DataFormatsTestObjectsParent1_xr.rootmap
+
+echo "Read the file without TransientIntParentT<1> dictionary"
+cmsRun ${LOCALTOP}/src/IOPool/Input/test/PoolNoParentDictionaryTestStep2_cfg.py || die 'Failed cmsRun PoolNoParentDictionaryTestStep2_cfg.py' $?