Add option to disable Celeritas offloading in celer-g4

app/celer-g4/ActionInitialization.cc

@@ -28,8 +28,6 @@ ActionInitialization::ActionInitialization() : init_celeritas_{true}
 {
     // Create params to be shared across worker threads
     params_ = std::make_shared<SharedParams>();
-    // Make global setup commands available to UI
-    GlobalSetup::Instance();
 }
 
 //---------------------------------------------------------------------------//

app/celer-g4/EventAction.cc

@@ -11,6 +11,7 @@
 #include <G4Event.hh>
 
 #include "corecel/Macros.hh"
+#include "corecel/sys/Environment.hh"
 #include "accel/ExceptionConverter.hh"
 
 #include "GlobalSetup.hh"
@@ -25,7 +26,9 @@ namespace app
  * Construct with thread-local Celeritas data.
  */
 EventAction::EventAction(SPConstParams params, SPTransporter transport)
-    : params_(params), transport_(transport)
+    : params_(params)
+    , transport_(transport)
+    , disable_offloading_(!celeritas::getenv("CELER_DISABLE").empty())
 {
     CELER_EXPECT(params_);
     CELER_EXPECT(transport_);
@@ -39,6 +42,9 @@ void EventAction::BeginOfEventAction(G4Event const* event)
 {
     CELER_LOG_LOCAL(debug) << "Starting event " << event->GetEventID();
 
+    if (disable_offloading_)
+        return;
+
     // Set event ID in local transporter
     ExceptionConverter call_g4exception{"celer0002"};
     CELER_TRY_HANDLE(transport_->SetEventId(event->GetEventID()),
@@ -53,9 +59,12 @@ void EventAction::EndOfEventAction(G4Event const* event)
 {
     CELER_EXPECT(event);
 
-    // Transport any tracks left in the buffer
-    ExceptionConverter call_g4exception{"celer0004", params_.get()};
-    CELER_TRY_HANDLE(transport_->Flush(), call_g4exception);
+    if (!disable_offloading_)
+    {
+        // Transport any tracks left in the buffer
+        ExceptionConverter call_g4exception{"celer0004", params_.get()};
+        CELER_TRY_HANDLE(transport_->Flush(), call_g4exception);
+    }
 
     if (GlobalSetup::Instance()->GetWriteSDHits())
     {

app/celer-g4/EventAction.hh

@@ -42,6 +42,7 @@ class EventAction final : public G4UserEventAction
   private:
     SPConstParams params_;
     SPTransporter transport_;
+    bool disable_offloading_;
 };
 
 //---------------------------------------------------------------------------//

app/celer-g4/GlobalSetup.cc

@@ -70,6 +70,11 @@ GlobalSetup::GlobalSetup()
             "Remove pointer suffix from input logical volume names");
         cmd.SetDefaultValue("true");
     }
+    {
+        auto& cmd = messenger_->DeclareProperty("physicsList", physics_list_);
+        cmd.SetGuidance("Select the physics list");
+        cmd.SetDefaultValue(physics_list_);
+    }
     {
         messenger_->DeclareMethod("magFieldZ",
                                   &GlobalSetup::SetMagFieldZTesla,

app/celer-g4/GlobalSetup.hh

@@ -38,6 +38,7 @@ class GlobalSetup
     int GetRootBufferSize() const { return root_buffer_size_; }
     bool GetWriteSDHits() const { return write_sd_hits_; }
     bool StripGDMLPointers() const { return strip_gdml_pointers_; }
+    std::string const& GetPhysicsList() const { return physics_list_; }
     //!@}
 
     //! Get a mutable reference to the setup options for DetectorConstruction
@@ -70,6 +71,7 @@ class GlobalSetup
     int root_buffer_size_{128000};
     bool write_sd_hits_{false};
     bool strip_gdml_pointers_{true};
+    std::string physics_list_{"FTFP_BERT"};
     G4ThreeVector field_;
 
     std::unique_ptr<G4GenericMessenger> messenger_;

app/celer-g4/RunAction.cc

@@ -16,6 +16,7 @@
 #include "corecel/Assert.hh"
 #include "corecel/Macros.hh"
 #include "corecel/io/Logger.hh"
+#include "corecel/sys/Environment.hh"
 #include "accel/ExceptionConverter.hh"
 
 #include "GlobalSetup.hh"
@@ -37,6 +38,7 @@ RunAction::RunAction(SPConstOptions options,
     , params_{std::move(params)}
     , transport_{std::move(transport)}
     , init_celeritas_{init_celeritas}
+    , disable_offloading_(!celeritas::getenv("CELER_DISABLE").empty())
 {
     CELER_EXPECT(options_);
     CELER_EXPECT(params_);
@@ -48,6 +50,9 @@ RunAction::RunAction(SPConstOptions options,
  */
 void RunAction::BeginOfRunAction(G4Run const* run)
 {
+    if (disable_offloading_)
+        return;
+
     CELER_EXPECT(run);
 
     ExceptionConverter call_g4exception{"celer0001"};
@@ -89,21 +94,25 @@ void RunAction::BeginOfRunAction(G4Run const* run)
  */
 void RunAction::EndOfRunAction(G4Run const*)
 {
-    CELER_LOG_LOCAL(status) << "Finalizing Celeritas";
     ExceptionConverter call_g4exception{"celer0005"};
 
-    if (transport_)
+    if (!disable_offloading_)
     {
-        // Deallocate Celeritas state data (ensures that objects are deleted on
-        // the thread in which they're created, necessary by some geant4
-        // thread-local allocators)
-        CELER_TRY_HANDLE(transport_->Finalize(), call_g4exception);
-    }
+        CELER_LOG_LOCAL(status) << "Finalizing Celeritas";
 
-    if (init_celeritas_)
-    {
-        // Clear shared data and write
-        CELER_TRY_HANDLE(params_->Finalize(), call_g4exception);
+        if (transport_)
+        {
+            // Deallocate Celeritas state data (ensures that objects are
+            // deleted on the thread in which they're created, necessary by
+            // some geant4 thread-local allocators)
+            CELER_TRY_HANDLE(transport_->Finalize(), call_g4exception);
+        }
+
+        if (init_celeritas_)
+        {
+            // Clear shared data and write
+            CELER_TRY_HANDLE(params_->Finalize(), call_g4exception);
+        }
     }
 
     if (GlobalSetup::Instance()->GetWriteSDHits())

app/celer-g4/RunAction.hh

@@ -53,6 +53,7 @@ class RunAction final : public G4UserRunAction
     SPParams params_;
     SPTransporter transport_;
     bool init_celeritas_;
+    bool disable_offloading_;
 };
 
 //---------------------------------------------------------------------------//

app/celer-g4/TrackingAction.cc

@@ -19,6 +19,7 @@
 
 #include "corecel/Assert.hh"
 #include "corecel/Macros.hh"
+#include "corecel/sys/Environment.hh"
 #include "accel/ExceptionConverter.hh"
 
 namespace celeritas
@@ -30,7 +31,9 @@ namespace app
  * Construct with Celeritas shared and thread-local data.
  */
 TrackingAction::TrackingAction(SPConstParams params, SPTransporter transport)
-    : params_(params), transport_(transport)
+    : params_(params)
+    , transport_(transport)
+    , disable_offloading_(!celeritas::getenv("CELER_DISABLE").empty())
 {
     CELER_EXPECT(params_);
     CELER_EXPECT(transport_);
@@ -46,6 +49,9 @@ TrackingAction::TrackingAction(SPConstParams params, SPTransporter transport)
  */
 void TrackingAction::PreUserTrackingAction(G4Track const* track)
 {
+    if (disable_offloading_)
+        return;
+
     CELER_EXPECT(track);
     CELER_EXPECT(*params_);
     CELER_EXPECT(*transport_);

app/celer-g4/TrackingAction.hh

@@ -42,6 +42,7 @@ class TrackingAction final : public G4UserTrackingAction
   private:
     SPConstParams params_;
     SPTransporter transport_;
+    bool disable_offloading_;
 };
 
 //---------------------------------------------------------------------------//

app/celer-g4/celer-g4.cc

@@ -6,6 +6,7 @@
 //! \file celer-g4/celer-g4.cc
 //---------------------------------------------------------------------------//
 
+#include <algorithm>
 #include <cstdlib>
 #include <iostream>
 #include <memory>
@@ -29,8 +30,11 @@
 #include "corecel/Assert.hh"
 #include "corecel/Macros.hh"
 #include "corecel/io/Logger.hh"
+#include "corecel/sys/Environment.hh"
 #include "corecel/sys/TypeDemangler.hh"
+#include "celeritas/ext/GeantPhysicsOptions.hh"
 #include "celeritas/ext/ScopedRootErrorHandler.hh"
+#include "celeritas/ext/detail/GeantPhysicsList.hh"
 #include "accel/ExceptionConverter.hh"
 #include "accel/Logger.hh"
 
@@ -74,20 +78,8 @@ void run(int argc, char** argv)
     CELER_LOG(info) << "Run manager type: "
                     << TypeDemangler<G4RunManager>{}(*run_manager);
 
-    // Construct geometry, SD factory, physics, actions
-    run_manager->SetUserInitialization(new DetectorConstruction{});
-    run_manager->SetUserInitialization(new FTFP_BERT{/* verbosity = */ 0});
-    run_manager->SetUserInitialization(new ActionInitialization());
-
-    std::vector<std::string> ignore_processes = {"CoulombScat"};
-    if (G4VERSION_NUMBER >= 1110)
-    {
-        CELER_LOG(warning) << "Default Rayleigh scattering 'MinKinEnergyPrim' "
-                              "is not compatible between Celeritas and "
-                              "Geant4@11.1: disabling Rayleigh scattering";
-        ignore_processes.push_back("Rayl");
-    }
-    GlobalSetup::Instance()->SetIgnoreProcesses(ignore_processes);
+    // Make global setup commands available to UI
+    GlobalSetup::Instance();
 
     G4UImanager* ui = G4UImanager::GetUIpointer();
     CELER_ASSERT(ui);
@@ -104,6 +96,39 @@ void run(int argc, char** argv)
     ui->ApplyCommand(std::string("/control/execute ")
                      + std::string(macro_filename));
 
+    std::vector<std::string> ignore_processes = {"CoulombScat"};
+    if (G4VERSION_NUMBER >= 1110)
+    {
+        CELER_LOG(warning) << "Default Rayleigh scattering 'MinKinEnergyPrim' "
+                              "is not compatible between Celeritas and "
+                              "Geant4@11.1: disabling Rayleigh scattering";
+        ignore_processes.push_back("Rayl");
+    }
+    GlobalSetup::Instance()->SetIgnoreProcesses(ignore_processes);
+
+    // Construct geometry, SD factory, physics, actions
+    run_manager->SetUserInitialization(new DetectorConstruction{});
+    if (GlobalSetup::Instance()->GetPhysicsList() == "FTFP_BERT")
+    {
+        run_manager->SetUserInitialization(new FTFP_BERT{/* verbosity = */ 0});
+    }
+    else if (GlobalSetup::Instance()->GetPhysicsList() == "GeantPhysicsList")
+    {
+        GeantPhysicsOptions opts;
+        if (std::find(ignore_processes.begin(), ignore_processes.end(), "Rayl")
+            != ignore_processes.end())
+        {
+            opts.rayleigh_scattering = false;
+        }
+        run_manager->SetUserInitialization(new detail::GeantPhysicsList{opts});
+    }
+    else
+    {
+        CELER_LOG(error) << "Unknown physics list '"
+                         << GlobalSetup::Instance()->GetPhysicsList() << "'";
+    }
+    run_manager->SetUserInitialization(new ActionInitialization());
+
     // Initialize run and process events
     CELER_LOG(status) << "Initializing run manager";
     run_manager->Initialize();
@@ -113,6 +138,13 @@ void run(int argc, char** argv)
     CELER_TRY_HANDLE(num_events = PrimaryGeneratorAction::NumEvents(),
                      ExceptionConverter{"demo-geant000"});
 
+    if (!celeritas::getenv("CELER_DISABLE").empty())
+    {
+        CELER_LOG(info)
+            << "Disabling Celeritas offloading since the 'CELER_DISABLE' "
+               "environment variable is present and non-empty";
+    }
+
     CELER_LOG(status) << "Transporting " << num_events << " events";
     run_manager->BeamOn(num_events);
 }
@@ -135,6 +167,7 @@ int main(int argc, char* argv[])
                   << "Environment variables:\n"
                   << "  G4FORCE_RUN_MANAGER_TYPE: MT or Serial\n"
                   << "  G4FORCENUMBEROFTHREADS: set CPU worker thread count\n"
+                  << "  CELER_DISABLE: nonempty disables offloading\n"
                   << "  CELER_DISABLE_DEVICE: nonempty disables CUDA\n"
                   << "  CELER_LOG: global logging level\n"
                   << "  CELER_LOG_LOCAL: thread-local logging level\n"