Merge pull request #37831 from wddgit/resourceInformationService

Implement ResourceInformationService

FWCore/Framework/src/ScheduleItems.cc

@@ -16,14 +16,18 @@
 #include "FWCore/Framework/interface/SignallingProductRegistry.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
 #include "FWCore/Utilities/interface/BranchType.h"
 #include "FWCore/Utilities/interface/GetPassID.h"
+#include "FWCore/Utilities/interface/ResourceInformation.h"
 #include "FWCore/Version/interface/GetReleaseVersion.h"
 
 #include <memory>
 
 #include <set>
+#include <string>
+#include <vector>
 
 namespace edm {
   ScheduleItems::ScheduleItems()
@@ -121,6 +125,13 @@ namespace edm {
   }
 
   std::shared_ptr<CommonParams> ScheduleItems::initMisc(ParameterSet& parameterSet) {
+    edm::Service<edm::ResourceInformation> resourceInformationService;
+    if (resourceInformationService.isAvailable()) {
+      auto const& selectedAccelerators =
+          parameterSet.getUntrackedParameter<std::vector<std::string>>("@selected_accelerators");
+      resourceInformationService->initializeAcceleratorTypes(selectedAccelerators);
+    }
+
     act_table_ = std::make_unique<ExceptionToActionTable>(parameterSet);
     std::string processName = parameterSet.getParameter<std::string>("@process_name");
     processConfiguration_ = std::make_shared<ProcessConfiguration>(

FWCore/Framework/src/defaultCmsRunServices.cc

@@ -23,6 +23,8 @@ namespace edm {
                                             "AdaptorConfig",
                                             "SiteLocalConfigService",
                                             "StatisticsSenderService",
+                                            "ResourceInformationService",
+                                            "CPU",
                                             "CondorStatusService",
                                             "XrdStatisticsService"};
 

FWCore/Integration/test/testSubProcess_cfg.py

@@ -34,7 +34,6 @@
                                       , test2 = cms.untracked.string('z')
 )
 
-process.CPU = cms.Service("CPU")
 process.ZombieKillerService = cms.Service("ZombieKillerService")
 process.JobReportService = cms.Service("JobReportService")
 
@@ -70,6 +69,12 @@
 copyProcess.AdaptorConfig = cms.Service("AdaptorConfig",
     intentionallyIllegalParameter = cms.bool(True)
 )
+copyProcess.ResourceInformationService = cms.Service("ResourceInformationService",
+    intentionallyIllegalParameter = cms.bool(True)
+)
+copyProcess.CondorStatusService = cms.Service("CondorStatusService",
+    intentionallyIllegalParameter = cms.bool(True)
+)
 
 copyProcess.DoodadESSource = cms.ESSource("DoodadESSource"
                                           , appendToDataLabel = cms.string('abc')
@@ -144,7 +149,7 @@
 prod2Process = cms.Process("PROD2")
 copy2Process.addSubProcess(cms.SubProcess(prod2Process,
     outputCommands = cms.untracked.vstring(
-        "keep *", 
+        "keep *",
         "drop *_putInt_*_*"),
 ))
 prod2Process.DoodadESSource = cms.ESSource("DoodadESSource"
@@ -269,7 +274,7 @@
 prod2ProcessAlt = cms.Process("PROD2ALT")
 copy2Process.addSubProcess(cms.SubProcess(prod2ProcessAlt,
     outputCommands = cms.untracked.vstring(
-        "keep *", 
+        "keep *",
         "drop *_putInt_*_*"),
 ))
 prod2ProcessAlt.DoodadESSource = cms.ESSource("DoodadESSource"

FWCore/MessageService/test/u10_cfg.py

@@ -22,6 +22,10 @@
     )
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(1)
 )

FWCore/MessageService/test/u14_cfg.py

@@ -40,6 +40,10 @@
     debugModules = cms.untracked.vstring('*')
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(2)
 )

FWCore/MessageService/test/u16_cfg.py

@@ -64,6 +64,10 @@
     debugModules = cms.untracked.vstring('*')
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(2)
 )

FWCore/MessageService/test/u1_cfg.py

@@ -59,6 +59,10 @@
     debugModules = cms.untracked.vstring('*'),
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(2)
 )

FWCore/MessageService/test/u1d_cfg.py

@@ -60,6 +60,10 @@
     debugModules = cms.untracked.vstring('*')
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(2)
 )

FWCore/MessageService/test/u20_cfg.py

@@ -18,6 +18,10 @@
 process.MessageLogger.cerr.noTimeStamps = True
 process.MessageLogger.cerr.enableStatistics = True
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(1)
 )

FWCore/MessageService/test/u27_cfg.py

@@ -35,6 +35,10 @@
     )
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(1)
 )

FWCore/MessageService/test/u7_cfg.py

@@ -33,6 +33,10 @@
     )
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(2)
 )

FWCore/MessageService/test/u9_cfg.py

@@ -39,6 +39,10 @@
     )
 )
 
+process.CPU = cms.Service("CPU",
+    disableJobReportOutput = cms.untracked.bool(True)
+)
+
 process.maxEvents = cms.untracked.PSet(
     input = cms.untracked.int32(1)
 )

FWCore/Services/plugins/CPU.cc

@@ -1,7 +1,7 @@
 // -*- C++ -*-
 //
 // Package:     Services
-// Class  :     CPU
+// Class  :     edm::service::CPU
 //
 // Implementation:
 //
@@ -15,6 +15,7 @@
 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
 #include "FWCore/Utilities/interface/CPUServiceBase.h"
+#include "FWCore/Utilities/interface/ResourceInformation.h"
 
 #include "cpu_features/cpu_features_macros.h"
 
@@ -28,14 +29,14 @@
 #include "cpu_features/cpuinfo_ppc.h"
 #endif
 
-#include <iostream>
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <cstdio>
+#include <cstdlib>
 #include <string>
 #include <fstream>
 #include <sstream>
+#include <map>
 #include <set>
+#include <utility>
+#include <vector>
 #include <fmt/format.h>
 
 #ifdef __linux__
@@ -53,12 +54,10 @@ namespace edm {
 
       static void fillDescriptions(ConfigurationDescriptions &descriptions);
 
-      bool cpuInfo(std::string &models, double &avgSpeed) override;
-
     private:
       const bool reportCPUProperties_;
+      const bool disableJobReportOutput_;
 
-      bool cpuInfoImpl(std::string &models, double &avgSpeed, Service<JobReport> *reportSvc);
       bool parseCPUInfo(std::vector<std::pair<std::string, std::string>> &info);
       std::string getModels(const std::vector<std::pair<std::string, std::string>> &info);
       std::string getModelFromCPUFeatures();
@@ -150,7 +149,27 @@ namespace edm {
     }  // namespace
 
     CPU::CPU(const ParameterSet &iPS, ActivityRegistry &iRegistry)
-        : reportCPUProperties_(iPS.getUntrackedParameter<bool>("reportCPUProperties")) {
+        : reportCPUProperties_(iPS.getUntrackedParameter<bool>("reportCPUProperties")),
+          disableJobReportOutput_(iPS.getUntrackedParameter<bool>("disableJobReportOutput")) {
+      edm::Service<edm::ResourceInformation> resourceInformationService;
+      if (resourceInformationService.isAvailable()) {
+        std::vector<std::pair<std::string, std::string>> info;
+        if (parseCPUInfo(info)) {
+          std::string models = getModels(info);
+          double averageSpeed = getAverageSpeed(info);
+          resourceInformationService->setCpuModelsFormatted(models);
+          resourceInformationService->setCpuAverageSpeed(averageSpeed);
+
+          std::set<std::string> modelsSet;
+          for (const auto &entry : info) {
+            if (entry.first == "model name") {
+              modelsSet.insert(entry.second);
+            }
+          }
+          std::vector<std::string> modelsVector(modelsSet.begin(), modelsSet.end());
+          resourceInformationService->setCPUModels(modelsVector);
+        }
+      }
       iRegistry.watchPostEndJob(this, &CPU::postEndJob);
     }
 
@@ -159,10 +178,15 @@ namespace edm {
     void CPU::fillDescriptions(edm::ConfigurationDescriptions &descriptions) {
       edm::ParameterSetDescription desc;
       desc.addUntracked<bool>("reportCPUProperties", false);
+      desc.addUntracked<bool>("disableJobReportOutput", false);
       descriptions.add("CPU", desc);
     }
 
     void CPU::postEndJob() {
+      if (disableJobReportOutput_) {
+        return;
+      }
+
       Service<JobReport> reportSvc;
 
       std::vector<std::pair<std::string, std::string>> info;
@@ -205,17 +229,6 @@ namespace edm {
       reportSvc->reportPerformanceSummary("SystemCPU", reportCPUProperties);
     }
 
-    bool CPU::cpuInfo(std::string &models, double &avgSpeed) {
-      std::vector<std::pair<std::string, std::string>> info;
-      if (!parseCPUInfo(info)) {
-        return false;
-      }
-
-      models = getModels(info);
-      avgSpeed = getAverageSpeed(info);
-      return true;
-    }
-
     bool CPU::parseCPUInfo(std::vector<std::pair<std::string, std::string>> &info) {
       info.clear();
       std::ifstream fcpuinfo("/proc/cpuinfo");

FWCore/Services/plugins/CondorStatusUpdater.cc

@@ -1,10 +1,10 @@
 
 #include "DataFormats/Provenance/interface/ModuleDescription.h"
 #include "DataFormats/Provenance/interface/ParameterSetID.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/Utilities/interface/ResourceInformation.h"
 #include "FWCore/Utilities/interface/TimingServiceBase.h"
-#include "FWCore/Utilities/interface/CPUServiceBase.h"
 #include "FWCore/ServiceRegistry/interface/Service.h"
-#include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
@@ -51,6 +51,7 @@ namespace edm {
       bool updateChirpImpl(std::string const &key, std::string const &value);
       inline void update();
       void firstUpdate();
+      void secondUpdate();
       void lastUpdate();
       void updateImpl(time_t secsSinceLastUpdate);
 
@@ -81,6 +82,7 @@ namespace edm {
 
       std::uint_least64_t m_lastEventCount = 0;
     };
+    inline bool isProcessWideService(CondorStatusService const *) { return true; }
 
   }  // namespace service
 
@@ -148,6 +150,7 @@ void CondorStatusService::filePost(std::string const & /*lfn*/) {
 }
 
 void CondorStatusService::beginPre(PathsAndConsumesOfModulesBase const &, ProcessContext const &processContext) {
+  secondUpdate();
   if (!m_processParameterSetID.isValid()) {
     m_processParameterSetID = processContext.parameterSetID();
   }
@@ -231,23 +234,27 @@ void CondorStatusService::firstUpdate() {
   updateChirp("MaxLumis", "-1");
   updateChirp("Done", "false");
   updateChirpQuoted("Guid", edm::processGUID().toString());
+}
 
-  edm::Service<edm::CPUServiceBase> cpusvc;
-  std::string models;
-  double avgSpeed;
-  if (cpusvc.isAvailable() && cpusvc->cpuInfo(models, avgSpeed)) {
-    updateChirpQuoted("CPUModels", models);
-    updateChirp("CPUSpeed", avgSpeed);
+void CondorStatusService::secondUpdate() {
+  edm::Service<edm::ResourceInformation> resourceInformationService;
+  if (resourceInformationService.isAvailable()) {
+    std::string models = resourceInformationService->cpuModelsFormatted();
+    double avgSpeed = resourceInformationService->cpuAverageSpeed();
+    if (!models.empty()) {
+      updateChirpQuoted("CPUModels", models);
+      updateChirp("CPUSpeed", avgSpeed);
+    }
   }
 }
 
 void CondorStatusService::lastUpdate() {
   time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
   updateImpl(now - m_lastUpdate);
   updateChirp("Done", "true");
-  edm::Service<edm::CPUServiceBase> cpusvc;
-  if (!cpusvc.isAvailable()) {
-    std::cout << "At post, CPU service is NOT available.\n";
+  edm::Service<edm::ResourceInformation> resourceInformationService;
+  if (!resourceInformationService.isAvailable()) {
+    edm::LogWarning("CondorStatusService") << "At post, ResourceInformationService is NOT available.\n";
   }
 }
 
@@ -428,5 +435,7 @@ void CondorStatusService::fillDescriptions(ConfigurationDescriptions &descriptio
   descriptions.add("CondorStatusService", desc);
 }
 
+#include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
+
 typedef edm::serviceregistry::AllArgsMaker<edm::service::CondorStatusService> CondorStatusServiceMaker;
 DEFINE_FWK_SERVICE_MAKER(CondorStatusService, CondorStatusServiceMaker);