BRAYNS 582 - Return 50% load progress for circuits (#1204)

BlueBrain · Oct 4, 2023 · f551760 · f551760
1 parent 4657e8a
commit f551760
Show file tree

Hide file tree

Showing 9 changed files with 57 additions and 38 deletions.
diff --git a/plugins/CircuitExplorer/api/circuit/MorphologyCircuitBuilder.cpp b/plugins/CircuitExplorer/api/circuit/MorphologyCircuitBuilder.cpp
@@ -100,28 +100,28 @@ class ParallelMorphologyLoader
                 morphologies[idx] =
                     NeuronGeometryInstantiator<PrimitiveType>::instantiate(baseGeometry, positions[idx], rotations[idx]);
             }
+
+            // Return the number of cells loaded for the progress updater
+            return indices.size();
         };
 
-        auto updateMessage = std::string("Loading neurons");
-        auto loadTasks = std::deque<std::future<void>>();
+        auto loadTasks = std::deque<std::future<std::size_t>>();
 
         for (auto &[path, cellIndices] : morphologyMap)
         {
             if (loadTasks.size() == maxThreads)
             {
                 auto &topTask = loadTasks.front();
-                topTask.get();
+                progressUpdater.update(topTask.get());
                 loadTasks.pop_front();
-                progressUpdater.update(updateMessage);
             }
 
             loadTasks.push_back(std::async(loadFn, path, cellIndices));
         }
 
         for (auto &task : loadTasks)
         {
-            task.get();
-            progressUpdater.update(updateMessage);
+            progressUpdater.update(task.get());
         }
 
         return morphologies;

diff --git a/plugins/CircuitExplorer/io/BBPLoader.cpp b/plugins/CircuitExplorer/io/BBPLoader.cpp
@@ -51,13 +51,13 @@ struct SynapseImporter
 
         if (afferent)
         {
-            updater.update("Loading afferent synapses");
+            updater.update();
             auto model = bbploader::SynapseLoader::load(context, true);
             modelList.push_back(std::move(model));
         }
         if (efferent)
         {
-            updater.update("Loading efferent synapses");
+            updater.update();
             auto model = bbploader::SynapseLoader::load(context, false);
             modelList.push_back(std::move(model));
         }
@@ -113,7 +113,7 @@ std::vector<std::shared_ptr<brayns::Model>> BBPLoader::importFromFile(
     const brion::BlueConfig config(path);
     bbploader::ParameterCheck::checkInput(config, params);
 
-    ProgressUpdater updater(callback, 3);
+    ProgressUpdater updater(callback, 3, 0.5f);
 
     const brain::Circuit circuit(config);
     const auto gids = bbploader::GIDLoader::compute(config, circuit, params);
@@ -124,22 +124,24 @@ std::vector<std::shared_ptr<brayns::Model>> BBPLoader::importFromFile(
     auto model = std::make_shared<brayns::Model>(ModelType::neurons);
 
     // Load neurons
-    updater.beginStage(gids.size());
+    updater.beginStage("Neuron load", gids.size());
     auto compartments = bbploader::CellLoader::load(context, updater, *model);
     updater.endStage();
 
     // Load simulation
-    updater.beginStage();
+    updater.beginStage("Report load");
     bbploader::ReportLoader::load(context, compartments, updater, *model);
     updater.endStage();
 
     result.push_back(std::move(model));
 
     // Load synapses
-    updater.beginStage(2);
+    updater.beginStage("Synapse load", 2);
     SynapseImporter::import(context, result, updater);
     updater.endStage();
 
+    updater.end("Generating rendering structures. Might take a while");
+
     brayns::Log::info("[CE] {}: Loaded {} model(s) in {} second(s).", getName(), result.size(), timer.seconds());
 
     return result;

diff --git a/plugins/CircuitExplorer/io/CellPlacementLoader.cpp b/plugins/CircuitExplorer/io/CellPlacementLoader.cpp
@@ -115,11 +115,13 @@ class PopulationLoader
 
         auto model = std::make_shared<brayns::Model>(ModelType::neurons);
 
-        updater.beginStage(selection.flatSize());
+        updater.beginStage("Morphology load", selection.flatSize());
         MorphologyCircuitBuilder::build(*model, std::move(context), updater);
         sonataloader::NeuronMetadataFactory::create(*model, info.name);
         updater.endStage();
 
+        updater.end("Generating rendering structures. Might take a while");
+
         return model;
     }
 

diff --git a/plugins/CircuitExplorer/io/SonataLoader.cpp b/plugins/CircuitExplorer/io/SonataLoader.cpp
@@ -39,7 +39,7 @@ class ProgressUpdaterFactory
     static ProgressUpdater create(const brayns::LoaderProgress &callback, const SonataLoaderParameters &params)
     {
         auto numSteps = _computeNumSteps(params);
-        return ProgressUpdater(callback, numSteps);
+        return ProgressUpdater(callback, numSteps, 0.5f);
     }
 
 private:
@@ -105,7 +105,8 @@ std::vector<std::shared_ptr<brayns::Model>> SonataLoader::importFromFile(
         auto &nodeName = nodeParams.node_population;
         brayns::Log::info("[CE] - Loading {} node population.", nodeName);
 
-        progress.beginStage(2);
+        auto nodeMessage = nodeName + " nodes load";
+        progress.beginStage(nodeMessage, 2);
         auto nodes = config.getNodes(nodeName);
         auto nodeSelection = sl::NodeSelector::select(config, nodeParams);
         auto nodeModelType = sl::ModelTypeFinder::fromNodes(nodes, config);
@@ -120,7 +121,8 @@ std::vector<std::shared_ptr<brayns::Model>> SonataLoader::importFromFile(
             auto &edgeName = edgeParams.edge_population;
             brayns::Log::info("[CE] - Loading {} edge population.", edgeName);
 
-            progress.beginStage(2);
+            auto edgeMessage = edgeName + " edges load";
+            progress.beginStage(edgeMessage, 2);
             auto edges = config.getEdges(edgeName);
             auto edgeSelection = sl::EdgeSelector::select(config, edgeParams, nodeSelection);
             auto edgeModelType = sl::ModelTypeFinder::fromEdges(edges, edgeParams.load_afferent, config);
@@ -133,6 +135,8 @@ std::vector<std::shared_ptr<brayns::Model>> SonataLoader::importFromFile(
         }
     }
 
+    progress.end("Generating rendering structures. Might take a while");
+
     auto time = timer.seconds();
     brayns::Log::info("[CE] {}: done in {} second(s).", getName(), time);
 

diff --git a/plugins/CircuitExplorer/io/bbploader/ReportLoader.cpp b/plugins/CircuitExplorer/io/bbploader/ReportLoader.cpp
@@ -137,13 +137,12 @@ class ReportHandlerFactory
             return {};
         }
 
+        callback.update();
+
         if (reportType == bbploader::ReportType::Compartment)
         {
-            callback.update("Loading compartment report");
             return std::make_unique<CompartmentHandler>(context, compartments);
         }
-
-        callback.update("Loading spikes");
         return std::make_unique<SpikeHandler>(context, compartments);
     }
 };

diff --git a/plugins/CircuitExplorer/io/sonataloader/populations/nodes/VasculaturePopulationLoader.cpp b/plugins/CircuitExplorer/io/sonataloader/populations/nodes/VasculaturePopulationLoader.cpp
@@ -93,7 +93,7 @@ class VasculatureReportFactory
     static void create(sl::NodeLoadContext &context)
     {
         auto &cb = context.progress;
-        cb.update("Loading vasculature report");
+        cb.update();
 
         auto &params = context.params;
         auto reportType = params.report_type;

diff --git a/plugins/CircuitExplorer/io/sonataloader/populations/nodes/common/NeuronReportFactory.cpp b/plugins/CircuitExplorer/io/sonataloader/populations/nodes/common/NeuronReportFactory.cpp
@@ -236,16 +236,14 @@ class ReportHandler
         const std::vector<CellCompartments> &compartments)
     {
         auto &callback = context.progress;
+        callback.update();
 
         auto &params = context.params;
         auto reportType = params.report_type;
         if (reportType == sl::ReportType::Spikes)
         {
-            callback.update("Loading spikes");
             return SpikeReportData::create(context, compartments);
         }
-
-        callback.update("Loading compartment report");
         return CompartmentReportData::create(context, compartments);
     }
 };

diff --git a/plugins/CircuitExplorer/io/util/ProgressUpdater.cpp b/plugins/CircuitExplorer/io/util/ProgressUpdater.cpp
@@ -1,30 +1,44 @@
 #include "ProgressUpdater.h"
 
-ProgressUpdater::ProgressUpdater(const brayns::LoaderProgress &callback, const size_t numStages):
+#include <cassert>
+
+ProgressUpdater::ProgressUpdater(const brayns::LoaderProgress &callback, std::size_t numStages, float maxProgress):
     _callback(callback),
     _numStages(numStages),
-    _stageSize(1.f / static_cast<float>(_numStages)),
+    _stageSize(maxProgress / static_cast<float>(_numStages)),
     _currentStage(0),
     _currentStageChunkSize(1.f),
     _currentStageProgress(0.f)
 {
 }
 
-void ProgressUpdater::beginStage(const size_t numSubElements)
+void ProgressUpdater::beginStage(std::string_view message, std::size_t numSubElements)
 {
+    assert(_currentStageProgress == 0.f);
+
+    _currentMessage = std::string(message);
     _currentStageChunkSize = 1.f / static_cast<float>(numSubElements);
 }
 
-void ProgressUpdater::update(const std::string &message) noexcept
+void ProgressUpdater::update(std::size_t numSubElementsCompleted) noexcept
 {
     const auto globalProgress = (static_cast<float>(_currentStage) + _currentStageProgress) * _stageSize;
-    _currentStageProgress += _currentStageChunkSize;
-    _currentStageProgress = _currentStageProgress > 1.f ? 1.f : _currentStageProgress;
-    _callback.updateProgress(message, globalProgress);
+    _currentStageProgress += _currentStageChunkSize * static_cast<float>(numSubElementsCompleted);
+    _currentStageProgress = std::clamp(_currentStageProgress, 0.f, 1.f);
+    _callback.updateProgress(_currentMessage, globalProgress);
 }
 
 void ProgressUpdater::endStage()
 {
     ++_currentStage;
     _currentStageProgress = 0.f;
+    // Send a final update after each stage to show complete progress
+    update(0);
 }
+
+void ProgressUpdater::end(std::string_view message)
+{
+    assert(_currentStage == _numStages);
+
+    _callback.updateProgress(std::string(message), static_cast<float>(_numStages) * _stageSize);
+};
diff --git a/plugins/CircuitExplorer/io/util/ProgressUpdater.h b/plugins/CircuitExplorer/io/util/ProgressUpdater.h
@@ -5,20 +5,20 @@
 class ProgressUpdater
 {
 public:
-    ProgressUpdater(const brayns::LoaderProgress &callback, const size_t stages);
-
-    void beginStage(const size_t numSubElements = 1);
-
-    void update(const std::string &message) noexcept;
+    ProgressUpdater(const brayns::LoaderProgress &callback, std::size_t stages, float maxProgress = 1.f);
 
+    void beginStage(std::string_view message, std::size_t numSubElements = 1);
+    void update(std::size_t numSubElementsCompleted = 1) noexcept;
     void endStage();
+    void end(std::string_view message);
 
 private:
     const brayns::LoaderProgress &_callback;
-    const size_t _numStages{};
-    const float _stageSize{};
+    std::size_t _numStages;
+    float _stageSize;
 
-    size_t _currentStage{};
+    std::size_t _currentStage{};
     float _currentStageChunkSize{};
     float _currentStageProgress{};
+    std::string _currentMessage;
 };