Implemented Circuit::getAfferentProjections

This function gives access to synape data for afferent projections
external to the circuit.

brain/circuit.cpp

@@ -207,6 +207,13 @@ SynapsesStream Circuit::getAfferentSynapses(
     return SynapsesStream(*this, gids, true, prefetch);
 }
 
+SynapsesStream Circuit::getExternalAfferentSynapses(
+    const GIDSet& gids, const std::string& source,
+    const SynapsePrefetch prefetch) const
+{
+    return SynapsesStream(*this, gids, source, prefetch);
+}
+
 SynapsesStream Circuit::getEfferentSynapses(
     const GIDSet& gids, const SynapsePrefetch prefetch) const
 {

brain/circuit.h

@@ -156,6 +156,27 @@ class Circuit
         const GIDSet& gids,
         SynapsePrefetch prefetch = SynapsePrefetch::none) const;
 
+    /**
+     * Access all afferent synapses projected from another circuit into the
+     * given GIDs.
+     *
+     * @param gids the gids of the post-synaptic cells
+     * @param source the name of the projecting circuit. This corresponds
+     *        to the label of a Projection section in the CircuitConfig.
+     *        If the source doesn't exist an exception will be thrown by the
+     *        first operation that tries to access the data.
+     * \if pybind
+     * @param prefetch which synapse data to preload
+     * \else
+     * @param prefetch which synapse data to load on SynapsesStream.read()
+     * @return synapse data stream. The pre-synaptic GIDs in the synapses refer
+     *         to cells in the projecting circuit.
+     * \endif
+     */
+    BRAIN_API SynapsesStream getExternalAfferentSynapses(
+        const GIDSet& gids, const std::string& source,
+        SynapsePrefetch prefetch = SynapsePrefetch::none) const;
+
     /**
      * Access all efferent synapses of the given GIDs.
      *

brain/detail/circuit.h

@@ -53,6 +53,7 @@ namespace brain
 {
 const std::string summaryFilename("/nrn_summary.h5");
 const std::string afferentFilename("/nrn.h5");
+const std::string externalAfferentFilename("/proj_nrn.h5");
 const std::string efferentFilename("/nrn_efferent.h5");
 const std::string afferentPositionsFilename("/nrn_positions.h5");
 const std::string efferentPositionsFilename("/nrn_positions_efferent.h5");
@@ -156,6 +157,12 @@ class Circuit::Impl
         , _cache(keyv::Map::createCache())
         , _synapsePositionColumns(0)
     {
+        for (auto&& projection :
+             config.getSectionNames(brion::CONFIGSECTION_PROJECTION))
+        {
+            _afferentProjectionSources[projection] =
+                config.getProjectionSource(projection);
+        }
     }
 
     virtual ~Impl() {}
@@ -243,6 +250,32 @@ class Circuit::Impl
         return **_synapseAttributes[i];
     }
 
+    const brion::Synapse& getAfferentProjectionAttributes(
+        const std::string& name) const
+    {
+        auto& lockable = _externalAfferents[name];
+        lunchbox::ScopedWrite mutex(lockable);
+        auto& synapses = *lockable;
+        if (!synapses)
+        {
+            auto&& source = _afferentProjectionSources.find(name);
+            if (source == _afferentProjectionSources.end())
+            {
+                _externalAfferents.erase(name);
+                LBTHROW(std::runtime_error(
+                    "Afferent synaptic projection not found: " + name));
+            }
+            fs::path path(source->second.getPath() + externalAfferentFilename);
+            if (fs::is_regular_file(path) || fs::is_symlink(path))
+                synapses.reset(new brion::Synapse(path.string()));
+            else
+                // Trying with the afferent synapses filename as a fallback
+                synapses.reset(new brion::Synapse(source->second.getPath() +
+                                                  afferentFilename));
+        }
+        return *synapses;
+    }
+
     const brion::Synapse* getSynapseExtra() const
     {
         lunchbox::ScopedWrite mutex(_synapseExtra);
@@ -401,6 +434,7 @@ class Circuit::Impl
     const brion::URI _circuitSource;
     const brion::URI _morphologySource;
     const brion::URI _synapseSource;
+    std::unordered_map<std::string, brion::URI> _afferentProjectionSources;
     const brion::URIs _targetSources;
     mutable brion::Targets _targetParsers;
     mutable keyv::MapPtr _cache;
@@ -413,6 +447,9 @@ class Circuit::Impl
     mutable LockPtr<brion::Synapse> _synapseExtra;
     mutable LockPtr<brion::Synapse> _synapsePositions[2];
     mutable size_t _synapsePositionColumns;
+
+    mutable std::unordered_map<std::string, LockPtr<brion::Synapse>>
+        _externalAfferents;
 };
 
 class MVD2 : public Circuit::Impl

brain/detail/synapsesStream.h

@@ -51,10 +51,23 @@ struct SynapsesStream
     {
     }
 
+    SynapsesStream(const Circuit& circuit, const GIDSet& gids,
+                   const std::string& source, const SynapsePrefetch prefetch)
+        : _circuit(circuit)
+        , _afferent(true)
+        , _gids(gids)
+        , _externalSource(source)
+        , _prefetch(prefetch)
+        , _it(_gids.begin())
+    {
+    }
+
     const Circuit& _circuit;
     const bool _afferent;
     const GIDSet _gids;
     const GIDSet _filterGIDs;
+    // Source name for external afferent projections
+    const std::string _externalSource;
     const SynapsePrefetch _prefetch;
     GIDSet::const_iterator _it;
 
@@ -69,9 +82,17 @@ struct SynapsesStream
         GIDSet::const_iterator start = _it;
         std::advance(_it, count);
         GIDSet::const_iterator end = _it;
+
+        if (_externalSource.empty())
+        {
+            return std::async(std::launch::async, [&, start, end] {
+                return Synapses(_circuit, GIDSet(start, end), _filterGIDs,
+                                _afferent, _prefetch);
+            });
+        }
         return std::async(std::launch::async, [&, start, end] {
-            return Synapses(_circuit, GIDSet(start, end), _filterGIDs,
-                            _afferent, _prefetch);
+            return Synapses(_circuit, GIDSet(start, end), _externalSource,
+                            _prefetch);
         });
     }
 };

brain/python/circuit.cpp

@@ -171,6 +171,15 @@ SynapsesWrapper Circuit_getAfferentSynapses(
                            circuit);
 }
 
+SynapsesWrapper Circuit_getExternalAfferentSynapses(
+    const CircuitPtr& circuit, bp::object gids, const std::string& source,
+    const brain::SynapsePrefetch prefetch)
+{
+    return SynapsesWrapper(circuit->getExternalAfferentSynapses(
+                               gidsFromPython(gids), source, prefetch),
+                           circuit);
+}
+
 SynapsesWrapper Circuit_getEfferentSynapses(
     const CircuitPtr& circuit, bp::object gids,
     const brain::SynapsePrefetch prefetch)
@@ -249,6 +258,10 @@ circuitWrapper
          (selfarg, bp::arg("gids"),
           bp::arg("prefetch") = SynapsePrefetch::none),
          DOXY_FN(brain::Circuit::getAfferentSynapses))
+    .def("external_afferent_synapses", Circuit_getExternalAfferentSynapses,
+         (selfarg, bp::arg("gids"), bp::arg("source"),
+          bp::arg("prefetch") = SynapsePrefetch::none),
+         DOXY_FN(brain::Circuit::getAfferentProjections))
     .def("efferent_synapses", Circuit_getEfferentSynapses,
          (selfarg, bp::arg("gids"),
           bp::arg("prefetch") = SynapsePrefetch::none),

brain/synapses.cpp

@@ -80,6 +80,22 @@ struct Synapses::Impl : public Synapses::BaseImpl
             _loadPositions(gids, filterGIDs);
     }
 
+    Impl(const Circuit& circuit, const GIDSet& gids, const std::string& source,
+         const SynapsePrefetch prefetch)
+        : _circuit(circuit)
+        , _gids(prefetch != SynapsePrefetch::all ? gids : GIDSet())
+        , _afferent(true)
+        , _externalSource(source)
+        , _size(0)
+    {
+        if (prefetch == SynapsePrefetch::none)
+            // We don't have a summary file for projected afferent synapses.
+            return;
+
+        if (int(prefetch) & int(SynapsePrefetch::attributes))
+            _loadAttributes(gids, GIDSet());
+    }
+
 #define FILTER(gid)                                                      \
     if (!filterGIDs.empty() && filterGIDs.find(gid) == filterGIDs.end()) \
         continue;
@@ -123,14 +139,19 @@ struct Synapses::Impl : public Synapses::BaseImpl
             return;
 
         const brion::Synapse& synapseAttributes =
-            _circuit._impl->getSynapseAttributes(_afferent);
-        const brion::Synapse* synapseExtra = _circuit._impl->getSynapseExtra();
+            _externalSource.empty()
+                ? _circuit._impl->getSynapseAttributes(_afferent)
+                : _circuit._impl->getAfferentProjectionAttributes(
+                      _externalSource);
+        const brion::Synapse* synapseExtra =
+            _externalSource.empty() ? _circuit._impl->getSynapseExtra() : 0;
 
         const bool haveExtra = _afferent && synapseExtra;
         const bool haveSize = _size > 0;
         _allocateAttributes(haveSize ? _size
                                      : synapseAttributes.getNumSynapses(gids),
                             haveExtra);
+        const bool haveGIDs = _externalSource.empty() && haveSize;
 
         size_t i = 0;
         for (const auto gid : gids)
@@ -144,7 +165,7 @@ struct Synapses::Impl : public Synapses::BaseImpl
                 const uint32_t preGid = attr[j][0];
                 FILTER(preGid);
 
-                if (!haveSize)
+                if (!haveGIDs)
                 {
                     _preGID.get()[i] = preGid;
                     _postGID.get()[i] = gid;
@@ -180,6 +201,13 @@ struct Synapses::Impl : public Synapses::BaseImpl
 
     void _loadPositions(const GIDSet& gids, const GIDSet& filterGIDs) const
     {
+        if (!_externalSource.empty())
+        {
+            LBTHROW(
+                std::runtime_error("Synapse positions are not available "
+                                   "for external projection synapses"));
+        }
+
         if (_preCenterPositionX)
             return;
 
@@ -353,6 +381,17 @@ struct Synapses::Impl : public Synapses::BaseImpl
         _allocate(_postCenterPositionZ, size);
     }
 
+    void _ensureGIDs() const
+    {
+        if (_externalSource.empty() || _hasAttributes())
+            return;
+
+        lunchbox::ScopedWrite mutex(_lock);
+        // For external projections we don't have a summary file, so we load
+        // all the attributes instead.
+        _loadAttributes(_gids, _filterGIDs);
+    }
+
     void _ensureAttributes() const
     {
         if (_hasAttributes())
@@ -383,10 +422,25 @@ struct Synapses::Impl : public Synapses::BaseImpl
         return _preCenterPositionX.get() != nullptr;
     }
 
+    size_t _getSize() const
+    {
+        lunchbox::ScopedRead mutex(_lock);
+
+        if (!_externalSource.empty() && _size == 0)
+        {
+            const brion::Synapse& attributes =
+                _circuit._impl->getAfferentProjectionAttributes(
+                    _externalSource);
+            _size = attributes.getNumSynapses(_gids);
+        }
+        return _size;
+    }
+
     const Circuit& _circuit;
     const GIDSet _gids;
     const GIDSet _filterGIDs;
     const bool _afferent;
+    std::string _externalSource;
 
     template <typename T>
     struct FreeDeleter
@@ -436,10 +490,16 @@ struct Synapses::Impl : public Synapses::BaseImpl
     mutable std::mutex _lock;
 };
 
-Synapses::Synapses(const Circuit& circuit, const GIDSet& pre,
-                   const GIDSet& post, const bool afferent,
+Synapses::Synapses(const Circuit& circuit, const GIDSet& gids,
+                   const GIDSet& filterGIDs, const bool afferent,
                    const SynapsePrefetch prefetch)
-    : _impl(new Impl(circuit, pre, post, afferent, prefetch))
+    : _impl(new Impl(circuit, gids, filterGIDs, afferent, prefetch))
+{
+}
+
+Synapses::Synapses(const Circuit& circuit, const GIDSet& gids,
+                   const std::string& source, const SynapsePrefetch prefetch)
+    : _impl(new Impl(circuit, gids, source, prefetch))
 {
 }
 
@@ -448,9 +508,13 @@ Synapses::~Synapses()
 }
 
 Synapses::Synapses(const SynapsesStream& stream)
-    : _impl(new Impl(stream._impl->_circuit, stream._impl->_gids,
-                     stream._impl->_filterGIDs, stream._impl->_afferent,
-                     stream._impl->_prefetch))
+    : _impl(stream._impl->_externalSource.empty()
+                ? new Impl(stream._impl->_circuit, stream._impl->_gids,
+                           stream._impl->_filterGIDs, stream._impl->_afferent,
+                           stream._impl->_prefetch)
+                : new Impl(stream._impl->_circuit, stream._impl->_gids,
+                           stream._impl->_externalSource,
+                           stream._impl->_prefetch))
 {
 }
 
@@ -481,8 +545,7 @@ Synapses& Synapses::operator=(Synapses&& rhs)
 size_t Synapses::size() const
 {
     const Impl& impl = static_cast<const Impl&>(*_impl);
-    lunchbox::ScopedRead mutex(impl._lock);
-    return impl._size;
+    return impl._getSize();
 }
 
 bool Synapses::empty() const
@@ -517,6 +580,7 @@ const size_t* Synapses::indices() const
 const uint32_t* Synapses::preGIDs() const
 {
     const Impl& impl = static_cast<const Impl&>(*_impl);
+    impl._ensureGIDs();
     return impl._preGID.get();
 }
 
@@ -586,6 +650,7 @@ const float* Synapses::preCenterZPositions() const
 const uint32_t* Synapses::postGIDs() const
 {
     const Impl& impl = static_cast<const Impl&>(*_impl);
+    impl._ensureGIDs();
     return impl._postGID.get();
 }
 

brain/synapses.h

@@ -226,6 +226,8 @@ class Synapses
     friend struct detail::SynapsesStream;
     Synapses(const Circuit& circuit, const GIDSet& gids,
              const GIDSet& filterGIDs, bool afferent, SynapsePrefetch prefetch);
+    Synapses(const Circuit& circuit, const GIDSet& gids,
+             const std::string& source, SynapsePrefetch prefetch);
 };
 }
 

brain/synapsesStream.cpp

@@ -39,6 +39,13 @@ SynapsesStream::SynapsesStream(const Circuit& circuit, const GIDSet& preGIDs,
 {
 }
 
+SynapsesStream::SynapsesStream(const Circuit& circuit, const GIDSet& gids,
+                               const std::string& source,
+                               const SynapsePrefetch prefetch)
+    : _impl(new detail::SynapsesStream(circuit, gids, source, prefetch))
+{
+}
+
 SynapsesStream::~SynapsesStream()
 {
 }

brain/synapsesStream.h

@@ -74,6 +74,9 @@ class SynapsesStream
                    SynapsePrefetch prefetch);
     SynapsesStream(const Circuit& circuit, const GIDSet& preGIDs,
                    const GIDSet& postGIDs, SynapsePrefetch prefetch);
+    // Constructor for afferent external projections
+    SynapsesStream(const Circuit& circuit, const GIDSet& gids,
+                   const std::string& source, SynapsePrefetch prefetch);
 
     friend class Synapses;
     std::unique_ptr<detail::SynapsesStream> _impl;