From b4b789719af3c7f42e38d9be794efa6481059cb5 Mon Sep 17 00:00:00 2001 From: Josh Siegle Date: Thu, 15 Oct 2020 14:38:30 -0700 Subject: [PATCH 01/16] Restore events and spikes recording in Open Ephys format --- .../OpenEphysFormat/OriginalRecording.cpp | 29 ++++++++++++++++--- Source/Processors/RecordNode/RecordNode.cpp | 22 ++++++++++++-- Source/Processors/RecordNode/RecordNode.h | 2 ++ Source/Processors/RecordNode/RecordThread.cpp | 25 ++++++++++++++-- 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp b/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp index 6604698d0..e31b91747 100644 --- a/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp +++ b/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp @@ -69,7 +69,7 @@ String OriginalRecording::getEngineID() const void OriginalRecording::addSpikeElectrode(int index, const SpikeChannel* elec) { - spikeFileArray.add(nullptr); + //spikeFileArray.add(nullptr); // deprecated } void OriginalRecording::resetChannels() @@ -91,7 +91,7 @@ void OriginalRecording::openFiles(File rootFolder, int experimentNumber, int rec processorArray.clear(); lastProcId = 0; - //openFile(rootFolder, getEventChannel(0), 0); + openFile(rootFolder, getEventChannel(0), 0); openMessageFile(rootFolder); int nChannels = getNumRecordedChannels(); @@ -103,10 +103,15 @@ void OriginalRecording::openFiles(File rootFolder, int experimentNumber, int rec blockIndex.add(0); samplesSinceLastTimestamp.add(0); } - for (int i = 0; i < spikeFileArray.size(); i++) + + int nSpikes = getNumRecordedSpikes(); + + for (int i = 0; i < nSpikes; i++) { + spikeFileArray.add(nullptr); openSpikeFile(rootFolder, getSpikeChannel(i), i); } + } void OriginalRecording::openFile(File rootFolder, const InfoObjectCommon* ch, int channelIndex) @@ -214,11 +219,14 @@ void OriginalRecording::openSpikeFile(File rootFolder, const SpikeChannel* elec, if (!fileExists) { + String header = generateSpikeHeader(elec); fwrite(header.toUTF8(), 1, header.getNumBytesAsUTF8(), spFile); + std::cout << "Wrote header." << std::endl; } diskWriteLock.exit(); spikeFileArray.set(channelIndex, spFile); + std::cout << "Added file." << std::endl; } @@ -364,7 +372,7 @@ String OriginalRecording::generateSpikeHeader(const SpikeChannel* elec) header += "';\n"; header += "header.num_channels = "; - header += String(elec->getNumChannels()); + header += String(c); header += ";\n"; header += "header.sampleRate = "; @@ -592,6 +600,7 @@ void OriginalRecording::closeFiles() } } fileArray.clear(); + blockIndex.clear(); samplesSinceLastTimestamp.clear(); for (int i = 0; i < spikeFileArray.size(); i++) @@ -630,12 +639,18 @@ void OriginalRecording::closeFiles() void OriginalRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike) { + //std::cout << "Electrode index: " << electrodeIndex << std::endl; + if (spikeFileArray[electrodeIndex] == nullptr) return; + //std::cout << "Got spike" << std::endl; + HeapBlock spikeBuffer; const SpikeChannel* channel = getSpikeChannel(electrodeIndex); + //std::cout << "Got spike channel" << std::endl; + int totalSamples = channel->getTotalSamples() * channel->getNumChannels(); int numChannels = channel->getNumChannels(); int chanSamples = channel->getTotalSamples(); @@ -658,6 +673,8 @@ void OriginalRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike) zeromem(spikeBuffer.getData() + 32, 2 * sizeof(float)); *reinterpret_cast(spikeBuffer.getData() + 40) = channel->getSampleRate(); + //std::cout << "Allocated memory" << std::endl; + int ptrIdx = 0; uint16* dataIntPtr = reinterpret_cast(spikeBuffer.getData() + 42); const float* spikeDataPtr = spike->getDataPointer(); @@ -683,6 +700,8 @@ void OriginalRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike) ptrIdx += sizeof(int16); } + //std::cout << "Starting disk write" << std::endl; + diskWriteLock.enter(); fwrite(spikeBuffer, 1, totalBytes, spikeFileArray[electrodeIndex]); @@ -693,6 +712,8 @@ void OriginalRecording::writeSpike(int electrodeIndex, const SpikeEvent* spike) spikeFileArray[electrodeIndex]); // ptr to FILE object diskWriteLock.exit(); + + //std::cout << "Wrote to file" << std::endl; } void OriginalRecording::writeXml() diff --git a/Source/Processors/RecordNode/RecordNode.cpp b/Source/Processors/RecordNode/RecordNode.cpp index 54cf37b34..9a488f2c3 100755 --- a/Source/Processors/RecordNode/RecordNode.cpp +++ b/Source/Processors/RecordNode/RecordNode.cpp @@ -71,6 +71,7 @@ RecordNode::~RecordNode() void RecordNode::addInputChannel(const GenericProcessor* sourceNode, int chan) { + // not getting called if (chan != AccessClass::getProcessorGraph()->midiChannelIndex) { @@ -85,13 +86,12 @@ void RecordNode::addInputChannel(const GenericProcessor* sourceNode, int chan) } else { - for (int n = 0; n < sourceNode->getTotalEventChannels(); n++) { const EventChannel* orig = sourceNode->getEventChannel(n); //only add to the record node the events originating from this processor, to avoid duplicates if (orig->getSourceNodeID() == sourceNode->getNodeId()) - eventChannelArray.add(new EventChannel(*orig)); + nonOwnedEventChannelArray.add(new EventChannel(*orig)); } @@ -357,6 +357,24 @@ void RecordNode::updateSubprocessorMap() synchronizer->setSyncChannel(chan->getSourceNodeID(), chan->getSubProcessorIdx(), ch); } + } + + for (int ch = 0; ch < nonOwnedEventChannelArray.size(); ch++) + { + + EventChannel* chan = nonOwnedEventChannelArray[ch]; + int sourceID = chan->getSourceNodeID(); + int subProcID = chan->getSubProcessorIdx(); + + eventMap[sourceID][subProcID] = chan->getNumChannels(); + + if (dataChannelStates[sourceID][subProcID].size() && !syncChannelMap[sourceID][subProcID]) + { + syncOrderMap[sourceID][subProcID] = ch; + syncChannelMap[sourceID][subProcID] = 0; + synchronizer->setSyncChannel(chan->getSourceNodeID(), chan->getSubProcessorIdx(), ch); + } + } } diff --git a/Source/Processors/RecordNode/RecordNode.h b/Source/Processors/RecordNode/RecordNode.h index 36834c64d..9eddd7475 100755 --- a/Source/Processors/RecordNode/RecordNode.h +++ b/Source/Processors/RecordNode/RecordNode.h @@ -122,6 +122,8 @@ class RecordNode : public GenericProcessor, public FilenameComponentListener std::vector> subProcessorMap; std::vector startRecChannels; + Array nonOwnedEventChannelArray; + bool isSyncReady; //TODO: Need to validate these new methods diff --git a/Source/Processors/RecordNode/RecordThread.cpp b/Source/Processors/RecordNode/RecordThread.cpp index 392874580..bc13fcdc0 100644 --- a/Source/Processors/RecordNode/RecordThread.cpp +++ b/Source/Processors/RecordNode/RecordThread.cpp @@ -208,14 +208,24 @@ void RecordThread::writeSynchronizedData(const AudioSampleBuffer& dataBuffer, co std::vector spikes; int nSpikes = m_spikeQueue->getEvents(spikes, maxSpikes); + int total_null = 0; + int total_real = 0; + for (int sp = 0; sp < nSpikes; ++sp) { if (spikes[sp] == NULL) - std::cout << "Got NULL" << std::endl; - else + { + total_null++; + } + else{ + m_engine->writeSpike(spikes[sp]->getExtra(), &spikes[sp]->getData()); + total_real++; + } //m_engine->writeSpike(0, &spikes[sp]->getData()); } + if (total_null > 0) + std::cout << "Total null: " << total_null << ", total real: " << total_real << std::endl; } void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples, int maxEvents, int maxSpikes, bool lastBlock) @@ -277,10 +287,19 @@ void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples std::vector spikes; int nSpikes = m_spikeQueue->getEvents(spikes, maxSpikes); + //int total_null = 0; + //int total_real = 0; + for (int sp = 0; sp < nSpikes; ++sp) { - m_engine->writeSpike(spikes[sp]->getExtra(), &spikes[sp]->getData()); + if (spikes[sp] != NULL) + { + m_engine->writeSpike(spikes[sp]->getExtra(), &spikes[sp]->getData()); + } } + + //if (total_null > 0) + // std::cout << "Total null: " << total_null << ", total real: " << total_real << std::endl; } From afb90dc007cba13a33a7e5b920afbfadeebe92f0 Mon Sep 17 00:00:00 2001 From: Josh Siegle Date: Thu, 15 Oct 2020 14:47:14 -0700 Subject: [PATCH 02/16] Revert unnecessary changes --- .../OpenEphysFormat/OriginalRecording.cpp | 2 +- Source/Processors/RecordNode/RecordNode.cpp | 6 +++--- Source/Processors/RecordNode/RecordNode.h | 2 -- Source/Processors/RecordNode/RecordThread.cpp | 16 ---------------- 4 files changed, 4 insertions(+), 22 deletions(-) diff --git a/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp b/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp index e31b91747..643aa344a 100644 --- a/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp +++ b/Source/Processors/RecordNode/OpenEphysFormat/OriginalRecording.cpp @@ -372,7 +372,7 @@ String OriginalRecording::generateSpikeHeader(const SpikeChannel* elec) header += "';\n"; header += "header.num_channels = "; - header += String(c); + header += String(elec->getNumChannels()); header += ";\n"; header += "header.sampleRate = "; diff --git a/Source/Processors/RecordNode/RecordNode.cpp b/Source/Processors/RecordNode/RecordNode.cpp index 9a488f2c3..89f43d1d5 100755 --- a/Source/Processors/RecordNode/RecordNode.cpp +++ b/Source/Processors/RecordNode/RecordNode.cpp @@ -91,7 +91,7 @@ void RecordNode::addInputChannel(const GenericProcessor* sourceNode, int chan) const EventChannel* orig = sourceNode->getEventChannel(n); //only add to the record node the events originating from this processor, to avoid duplicates if (orig->getSourceNodeID() == sourceNode->getNodeId()) - nonOwnedEventChannelArray.add(new EventChannel(*orig)); + eventChannelArray.add(new EventChannel(*orig)); } @@ -359,10 +359,10 @@ void RecordNode::updateSubprocessorMap() } - for (int ch = 0; ch < nonOwnedEventChannelArray.size(); ch++) + for (int ch = 0; ch < eventChannelArray.size(); ch++) { - EventChannel* chan = nonOwnedEventChannelArray[ch]; + EventChannel* chan = eventChannelArray[ch]; int sourceID = chan->getSourceNodeID(); int subProcID = chan->getSubProcessorIdx(); diff --git a/Source/Processors/RecordNode/RecordNode.h b/Source/Processors/RecordNode/RecordNode.h index 9eddd7475..36834c64d 100755 --- a/Source/Processors/RecordNode/RecordNode.h +++ b/Source/Processors/RecordNode/RecordNode.h @@ -122,8 +122,6 @@ class RecordNode : public GenericProcessor, public FilenameComponentListener std::vector> subProcessorMap; std::vector startRecChannels; - Array nonOwnedEventChannelArray; - bool isSyncReady; //TODO: Need to validate these new methods diff --git a/Source/Processors/RecordNode/RecordThread.cpp b/Source/Processors/RecordNode/RecordThread.cpp index bc13fcdc0..9877f2689 100644 --- a/Source/Processors/RecordNode/RecordThread.cpp +++ b/Source/Processors/RecordNode/RecordThread.cpp @@ -208,24 +208,14 @@ void RecordThread::writeSynchronizedData(const AudioSampleBuffer& dataBuffer, co std::vector spikes; int nSpikes = m_spikeQueue->getEvents(spikes, maxSpikes); - int total_null = 0; - int total_real = 0; - for (int sp = 0; sp < nSpikes; ++sp) { if (spikes[sp] == NULL) { - total_null++; - } - else{ - m_engine->writeSpike(spikes[sp]->getExtra(), &spikes[sp]->getData()); - total_real++; } //m_engine->writeSpike(0, &spikes[sp]->getData()); } - if (total_null > 0) - std::cout << "Total null: " << total_null << ", total real: " << total_real << std::endl; } void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples, int maxEvents, int maxSpikes, bool lastBlock) @@ -287,9 +277,6 @@ void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples std::vector spikes; int nSpikes = m_spikeQueue->getEvents(spikes, maxSpikes); - //int total_null = 0; - //int total_real = 0; - for (int sp = 0; sp < nSpikes; ++sp) { if (spikes[sp] != NULL) @@ -298,9 +285,6 @@ void RecordThread::writeData(const AudioSampleBuffer& dataBuffer, int maxSamples } } - //if (total_null > 0) - // std::cout << "Total null: " << total_null << ", total real: " << total_real << std::endl; - } void RecordThread::forceCloseFiles() From 908a559b3c4e0780340eaeb9554614508fc63ec8 Mon Sep 17 00:00:00 2001 From: Josh Siegle Date: Thu, 15 Oct 2020 14:49:27 -0700 Subject: [PATCH 03/16] Remove redundant lines --- Source/Processors/RecordNode/RecordNode.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Source/Processors/RecordNode/RecordNode.cpp b/Source/Processors/RecordNode/RecordNode.cpp index 89f43d1d5..d78d80c85 100755 --- a/Source/Processors/RecordNode/RecordNode.cpp +++ b/Source/Processors/RecordNode/RecordNode.cpp @@ -341,24 +341,6 @@ void RecordNode::updateSubprocessorMap() eventMap.clear(); syncChannelMap.clear(); syncOrderMap.clear(); - for (int ch = 0; ch < eventChannelArray.size(); ch++) - { - - EventChannel* chan = eventChannelArray[ch]; - int sourceID = chan->getSourceNodeID(); - int subProcID = chan->getSubProcessorIdx(); - - eventMap[sourceID][subProcID] = chan->getNumChannels(); - - if (dataChannelStates[sourceID][subProcID].size() && !syncChannelMap[sourceID][subProcID]) - { - syncOrderMap[sourceID][subProcID] = ch; - syncChannelMap[sourceID][subProcID] = 0; - synchronizer->setSyncChannel(chan->getSourceNodeID(), chan->getSubProcessorIdx(), ch); - } - - } - for (int ch = 0; ch < eventChannelArray.size(); ch++) { @@ -376,7 +358,6 @@ void RecordNode::updateSubprocessorMap() } } - } int RecordNode::getNumSubProcessors() const From 036167e3d1dd31495f2b36bbb7729d16d4cececc Mon Sep 17 00:00:00 2001 From: Josh Siegle Date: Thu, 15 Oct 2020 14:50:53 -0700 Subject: [PATCH 04/16] == > != --- Source/Processors/RecordNode/RecordThread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Processors/RecordNode/RecordThread.cpp b/Source/Processors/RecordNode/RecordThread.cpp index 9877f2689..c0fbb7dbe 100644 --- a/Source/Processors/RecordNode/RecordThread.cpp +++ b/Source/Processors/RecordNode/RecordThread.cpp @@ -210,7 +210,7 @@ void RecordThread::writeSynchronizedData(const AudioSampleBuffer& dataBuffer, co for (int sp = 0; sp < nSpikes; ++sp) { - if (spikes[sp] == NULL) + if (spikes[sp] != NULL) { m_engine->writeSpike(spikes[sp]->getExtra(), &spikes[sp]->getData()); } From 007deff3f8e48c53756ecc654c7889d9ba0764e2 Mon Sep 17 00:00:00 2001 From: Pavel Kulik Date: Thu, 15 Oct 2020 20:14:29 -0700 Subject: [PATCH 05/16] Remove debug console --- Source/Main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Main.cpp b/Source/Main.cpp index b0735275b..c039b9477 100644 --- a/Source/Main.cpp +++ b/Source/Main.cpp @@ -65,7 +65,7 @@ class OpenEphysApplication : public JUCEApplication #ifdef WIN32 //glWinInit(); - if (AllocConsole()) + if (false)//(AllocConsole()) { freopen("CONOUT$","w",stdout); freopen("CONOUT$","w",stderr); From 17f732c2b78dd85cc44e8fc29d248f3572181f61 Mon Sep 17 00:00:00 2001 From: Pavel Kulik Date: Thu, 15 Oct 2020 20:15:03 -0700 Subject: [PATCH 06/16] Re-introduce NpyFile crash --- Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp b/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp index 26cb8a4cd..626f0136d 100644 --- a/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp +++ b/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp @@ -156,7 +156,7 @@ void NpyFile::writeHeader(const Array& typeList) void NpyFile::updateHeader() { - if (m_file != NULL) + if (true) { // overwrite the shape part of the header - even without explicitly calling // m_file->flush(), overwriting seems to trigger a flush to disk, From 24a3f4a4ddc608813752cd975f1c0a7caed6f641 Mon Sep 17 00:00:00 2001 From: Pavel Kulik Date: Thu, 15 Oct 2020 20:33:06 -0700 Subject: [PATCH 07/16] Add NpyFile debug output --- .../Processors/RecordNode/BinaryFormat/NpyFile.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp b/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp index 626f0136d..77b2fac4d 100644 --- a/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp +++ b/Source/Processors/RecordNode/BinaryFormat/NpyFile.cpp @@ -79,6 +79,19 @@ bool NpyFile::openFile(String path) // output stream buffer size defaults to 32768 bytes, but is irrelevant because // each updateHeader() call triggers a m_file->flush() to disk: m_file = file.createOutputStream(); + + if (m_file == nullptr) + { + LOGD("FAILED to open file @", path); + } + else + { + String pad = ""; + for (int i = 0; i < 162 - path.length(); i++) + pad += " "; + LOGD("Successfully opened file @", path, pad, m_file); + } + if (!m_file) return false; @@ -156,6 +169,7 @@ void NpyFile::writeHeader(const Array& typeList) void NpyFile::updateHeader() { + if (true) { // overwrite the shape part of the header - even without explicitly calling From fb6a6fef0ec1b0aa7e92337fb84492e38e167e39 Mon Sep 17 00:00:00 2001 From: Pavel Kulik Date: Thu, 15 Oct 2020 23:06:33 -0700 Subject: [PATCH 08/16] Connect to MessageCenter from RecordNode and ignore any duplicate messages --- .../MessageCenter/MessageCenter.cpp | 5 ++++ .../Processors/MessageCenter/MessageCenter.h | 2 ++ .../MessageCenter/MessageCenterEditor.h | 4 +-- .../ProcessorGraph/ProcessorGraph.cpp | 8 ++++-- Source/Processors/RecordNode/RecordNode.cpp | 28 ++++++++++++++++++- Source/Processors/RecordNode/RecordNode.h | 5 ++++ 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/Source/Processors/MessageCenter/MessageCenter.cpp b/Source/Processors/MessageCenter/MessageCenter.cpp index 4faf01632..4a2d731ac 100644 --- a/Source/Processors/MessageCenter/MessageCenter.cpp +++ b/Source/Processors/MessageCenter/MessageCenter.cpp @@ -63,6 +63,11 @@ AudioProcessorEditor* MessageCenter::createEditor() } +const EventChannel* MessageCenter::getMessageChannel() +{ + return getEventChannel(0); +} + void MessageCenter::setParameter(int parameterIndex, float newValue) { if (isRecording) diff --git a/Source/Processors/MessageCenter/MessageCenter.h b/Source/Processors/MessageCenter/MessageCenter.h index 32ce29d9c..6c216a297 100644 --- a/Source/Processors/MessageCenter/MessageCenter.h +++ b/Source/Processors/MessageCenter/MessageCenter.h @@ -61,6 +61,8 @@ class MessageCenter : public GenericProcessor /** A pointer to the Message Center editor. */ ScopedPointer messageCenterEditor; + const EventChannel* getMessageChannel(); + bool enable() override; bool disable() override; diff --git a/Source/Processors/MessageCenter/MessageCenterEditor.h b/Source/Processors/MessageCenter/MessageCenterEditor.h index c0093ae52..b66ecafa5 100644 --- a/Source/Processors/MessageCenter/MessageCenterEditor.h +++ b/Source/Processors/MessageCenter/MessageCenterEditor.h @@ -64,6 +64,8 @@ class MessageCenterEditor : public AudioProcessorEditor, String getLabelString(); + MessageCenter* messageCenter; + private: void buttonClicked(Button* button); @@ -85,8 +87,6 @@ class MessageCenterEditor : public AudioProcessorEditor, /** A JUCE button used to send messages. */ ScopedPointer