Record and load Midi in progress

src/features/SongRecording/index.ts

@@ -0,0 +1,169 @@
+import { getAudioContext } from '@/features/synth/utils'
+import { Midi } from "@tonejs/midi";
+import { MidiEvent } from "@/features/midi";
+
+let mediaRecorder: MediaRecorder
+let audioChunks: Blob[] = []
+
+// Node that listens all gainNotes for recording purposes
+let recordingDestinationNode: MediaStreamAudioDestinationNode
+function getRecordingDestinationNode() {
+    const audioContext = getAudioContext()
+    if (!recordingDestinationNode && audioContext != null) {
+        recordingDestinationNode = audioContext.createMediaStreamDestination()
+    }
+    return recordingDestinationNode
+}
+
+// Array that listens Midi messages
+let recordedMidiEvents: Array<MidiEvent> | null
+function getRecordedMidiEvents() {
+    return recordedMidiEvents
+}
+
+// Record audio
+function startRecordingAudio() {
+    const isAble = canRecordAudio()
+
+    if (isAble) {
+        mediaRecorder = new MediaRecorder(getRecordingDestinationNode().stream, {mimeType: 'audio/webm'})
+        audioChunks = []
+
+        mediaRecorder.onstop = () => { trySaveAudioFile() }
+        mediaRecorder.onerror = (error) => { console.error('MediaRecorder error:', error) }
+        mediaRecorder.ondataavailable = (event) => {
+            if (event.data.size > 0) {
+                audioChunks.push(event.data)
+            }
+        }
+
+        mediaRecorder.start()
+    }
+
+    return isAble
+}
+
+// TODO: Can't record audio state
+function canRecordAudio(): boolean {
+    return getRecordingDestinationNode() != null
+}
+
+function trySaveAudioFile() {
+    if (audioChunks.length > 0) {
+        const mimeType = audioChunks[0].type
+        const audioBlob = new Blob(audioChunks, {type: mimeType})
+        promptDownloadAudioFile(audioBlob, "Piano-Audio.webm", false)
+    }
+}
+
+function promptDownloadAudioFile(blob: Blob, defaultFileName: string, customName: boolean) {
+    const a = document.createElement("a")
+    a.style.display = "none"
+    document.body.appendChild(a)
+
+    const url = window.URL.createObjectURL(blob)
+    a.href = url
+
+    let fileName: string | null = defaultFileName
+    // This code blocks the page, TODO: Make a save as... window
+    if (customName) fileName = prompt("Enter a filename:", defaultFileName)
+    if (fileName) {
+        a.download = fileName
+        a.click()
+    }
+
+    window.URL.revokeObjectURL(url)
+    document.body.removeChild(a)
+}
+
+function stopRecordingAudio() {
+    const isAble = mediaRecorder && mediaRecorder.state !== 'inactive'
+    if (isAble) {
+        mediaRecorder.stop()
+    }
+}
+
+// Record Midi
+function startRecordingMidi() {
+    console.log("Starting recording midi")
+    recordedMidiEvents = []
+}
+
+function stopRecordingMidi() {
+    console.log("Stopping recording midi", recordedMidiEvents)
+    trySaveMidiFile()
+    recordedMidiEvents = null
+}
+
+function trySaveMidiFile() {
+    if (recordedMidiEvents && recordedMidiEvents.length > 1) {
+        console.log("Recorded Midi Events", recordedMidiEvents)
+        const midi = midiEventsToMidi(recordedMidiEvents)
+        const base64MidiData = Buffer.from(midi.toArray()).toString('base64');
+        console.log(encodeURIComponent(base64MidiData));
+        const midiBlob = new Blob([midi.toArray()], { type: "audio/midi" })
+        const midiBlobURL = URL.createObjectURL(midiBlob)
+        const downloadLink = document.createElement('a')
+        downloadLink.href = midiBlobURL
+        downloadLink.download = 'recorded_midi.mid'
+        document.body.appendChild(downloadLink)
+        downloadLink.click()
+        URL.revokeObjectURL(midiBlobURL)
+        document.body.removeChild(downloadLink)
+    }
+}
+
+type MidiNote = {
+    midi: number // Midi note
+    time: number // Time in seconds
+    velocity: number // normalized 0-1 velocity
+    duration: number // duration in seconds between noteOn and noteOff
+}
+
+function midiEventsToMidi(midiEvents: Array<MidiEvent>): Midi {
+    const midi = new Midi()
+    const track = midi.addTrack()
+    const notes: MidiNote[] = midiEventsToMidiNotes(midiEvents)
+    console.log("Notes", notes)
+    for (const note of notes) {
+        track.addNote(note)
+    }
+
+    return midi
+}
+
+function midiEventsToMidiNotes(midiEvents: Array<MidiEvent>): MidiNote[] {
+    const notes: MidiNote[] = []
+    const noteBuffer: MidiNote[] = []
+    let newNote: MidiNote = {
+        midi: 0,
+        time: 0,
+        velocity: 0,
+        duration: 0
+    }
+
+    const delay: number = (midiEvents.length > 0)? midiEvents[0].timeStamp : 0
+
+    for (const event of midiEvents) {
+        if (event.velocity > 0) {
+            // Note On
+            newNote.midi = event.note
+            newNote.time = (event.timeStamp - delay) / 1000
+            newNote.velocity = event.velocity / 100
+            noteBuffer.push({...newNote})
+        } else {
+            // Note Off
+            const foundNote = noteBuffer.find((note) => note.midi === event.note)
+            if (foundNote) {
+                foundNote.duration = (event.timeStamp - delay) / 1000 - foundNote.time
+                notes.push(foundNote)
+                noteBuffer.splice(noteBuffer.indexOf(foundNote), 1)
+            }
+        }
+    }
+
+    return notes
+}
+
+
+export { getRecordingDestinationNode, startRecordingAudio, stopRecordingAudio, getRecordedMidiEvents, startRecordingMidi, stopRecordingMidi }

src/features/data/index.tsx

@@ -5,6 +5,7 @@ import * as library from './library'
 import { useCallback, useMemo, useState } from 'react'
 import { useFetch } from '@/hooks'
 import { FetchState } from '@/hooks/useFetch'
+import { convertBase64MidiToSong } from '@/features/midi/utils'
 
 function handleSong(response: Response): Promise<Song> {
   return response.arrayBuffer().then(parseMidi)
@@ -40,6 +41,10 @@ export function useSong(id: string, source: SongSource): FetchState<Song> {
   return source === 'upload' ? uploadState : fetchState
 }
 
+export function useBase64Song(data: string): Song {
+  return convertBase64MidiToSong(data)
+}
+
 // TODO: replace with a signals-like library, so that setting from one component is reflected elsewhere.
 type SongManifestHookReturn = [SongMetadata[], (metadata: SongMetadata[]) => void]
 export function useSongManifest(): SongManifestHookReturn {

src/features/midi/index.tsx

@@ -1,6 +1,7 @@
 import { getNote } from '@/features/theory'
 import { MidiStateEvent } from '@/types'
 import { isBrowser } from '@/utils'
+import { getRecordedMidiEvents } from '@/features/SongRecording'
 
 export async function getMidiInputs(): Promise<WebMidi.MIDIInputMap> {
   if (!isBrowser() || !window.navigator.requestMIDIAccess) {
@@ -43,10 +44,11 @@ async function setupMidiDeviceListeners() {
   }
 }
 
-type MidiEvent = {
+export type MidiEvent = {
   type: 'on' | 'off'
   velocity: number
   note: number
+  timeStamp: number
 }
 
 function parseMidiMessage(event: WebMidi.MIDIMessageEvent): MidiEvent | null {
@@ -61,6 +63,7 @@ function parseMidiMessage(event: WebMidi.MIDIMessageEvent): MidiEvent | null {
     type: command === 0x9 ? 'on' : 'off',
     note: data[1],
     velocity: data[2],
+    timeStamp: event.timeStamp
   }
 }
 
@@ -157,6 +160,9 @@ function onMidiMessage(e: WebMidi.MIDIMessageEvent) {
   if (!msg) {
     return
   }
+
+  getRecordedMidiEvents()?.push(msg)
+
   const { note, velocity } = msg
   if (msg.type === 'on' && msg.velocity > 0) {
     midiState.press(note, velocity)

src/features/midi/utils.tsx

@@ -0,0 +1,62 @@
+import { Midi } from "@tonejs/midi"
+import fs from 'fs'
+import { Song, Track, Tracks, Bpm, SongNote } from '@/types'
+
+export function convertBase64MidiToSong(base64Midi: string) {
+    const binaryMidi = Buffer.from(decodeURIComponent(base64Midi), 'base64')
+    console.log(binaryMidi)
+    const midi = new Midi(binaryMidi)
+    console.log(midi)
+    let tracks: Tracks = {}
+    const bpms: Array<Bpm> = []
+    const notes: Array<SongNote> = []
+
+    let index: number = 0
+    for (const track of midi.tracks) {
+        const newTrack: Track = {
+            instrument: track.instrument.name.replace(/ /g,"_"),
+            name: "Track" + index
+        }
+        tracks[index] = {...newTrack}
+        for (const note of track.notes) {
+            const newNote: SongNote = {
+                type: "note",
+                midiNote: note.midi,
+                duration: note.duration,
+                track: index,
+                velocity: note.velocity,
+                time: note.time,
+                measure: note.bars // ??
+            }
+            notes.push(newNote)
+        }
+
+        index++
+    }
+
+    for (const tempos of midi.header.tempos) {
+        const newBpm: Bpm = {
+            time: tempos.time? tempos.time : 0,
+            bpm: tempos.bpm
+        }
+        bpms.push(newBpm)
+    }
+
+    // Not enough info to fill everything
+    const song: Song = {
+        tracks: tracks,
+        duration: midi.duration,
+        measures: [],
+        notes: notes,
+        bpms: bpms,
+        timeSignature: undefined,
+        keySignature: 'C',
+        items: notes,
+        backing: undefined
+    };
+
+    console.log(song)
+
+    return song
+}
+

src/features/pages/Freeplay/index.tsx

@@ -2,7 +2,7 @@ import React, { useCallback, useEffect, useRef, useState } from 'react'
 import { MidiStateEvent, SongConfig } from '@/types'
 import { SongVisualizer } from '@/features/SongVisualization'
 import { InstrumentName, useSynth } from '@/features/synth'
-import { startRecordingAudio, stopRecordingAudio } from '@/features/audioRecording'
+import { startRecordingAudio, startRecordingMidi, stopRecordingAudio, stopRecordingMidi } from '@/features/SongRecording'
 import midiState from '@/features/midi'
 import { useSingleton } from '@/hooks'
 import FreePlayer from './utils/freePlayer'
@@ -61,10 +61,13 @@ export default function FreePlay() {
           onClickRecord={(e) => {
             e.stopPropagation()
             if (!isRecordingAudio) {
-              setRecordingAudio(startRecordingAudio());
+              //startRecordingAudio()
+              startRecordingMidi()
+              setRecordingAudio(true)
             } else {
-              stopRecordingAudio();
-              setRecordingAudio(false);
+              //stopRecordingAudio()
+              stopRecordingMidi()
+              setRecordingAudio(false)
             }
           }}
           isRecordingAudio={isRecordingAudio}