feat: update audio merging service

.env

@@ -1,8 +1,7 @@
 # These are applied for all environments as default, update these appropriately
 # Create a .env.local file and update these values with your local connection string and secret keys
 CLIENT_HOST=http://localhost:3000
-MONGODB_URI=local connection string goes here
-PYTHON_HTTP_HOST=https://audio.0xen.dev
+MONGODB_URI=mongodb://localhost/arbor # local connection string
 BLOCKNATIVE_KEY=secret goes here
 NFT_STORAGE_KEY=secret goes here
 COVALENT_API_KEY=secret goes here

.env.development

@@ -1 +1 @@
-MONGODB_URI=
+MONGODB_URI=mongodb://localhost/arbor

components/ProjectDetails/ProjectDetails.tsx

@@ -23,23 +23,20 @@ import { saveAs } from 'file-saver'
 import JSZip from 'jszip'
 import dynamic from 'next/dynamic'
 import Link from 'next/link'
-import { useRouter } from 'next/router'
+// import { useRouter } from 'next/router'
 import { Fragment, useState } from 'react'
-import web3 from 'web3'
 
 import ImageOptimized from '../../components/ImageOptimized'
 import Notification from '../../components/Notification'
 import StemUploadDialog from '../../components/StemUploadDialog'
 import { useWeb3 } from '../../components/Web3Provider'
 import { NETWORK_CURRENCY } from '../../constants/networks'
-import logoBinary from '../../lib/logoBinary'
-import type { INft } from '../../models/nft.model'
 import type { IProjectDoc } from '../../models/project.model'
 import type { IStemDoc } from '../../models/stem.model'
 import OneIcon from '../../public/harmony_icon.svg'
 import { detailsStyles as styles } from '../../styles/Projects.styles'
+import { MergeAudioInput, useAudioUtils } from '../../utils/AudioUtilsProvider'
 import formatAddress from '../../utils/formatAddress'
-import { post } from '../../utils/http'
 
 // Because our stem player uses Web APIs for audio, we must ignore it for SSR to avoid errors
 const StemPlayer = dynamic(() => import('../../components/StemPlayer'), { ssr: false })
@@ -71,8 +68,9 @@ const ProjectDetails = (props: ProjectDetailsProps): JSX.Element | null => {
 	const [stems, setStems] = useState<Map<number, any>>(new Map())
 	const [isPlayingAll, setIsPlayingAll] = useState<boolean>(false)
 	// Hooks
-	const router = useRouter()
-	const { NFTStore, contracts, connected, currentUser, handleConnectWallet } = useWeb3()
+	// const router = useRouter()
+	const { /* NFTStore, contracts, */ connected, currentUser, handleConnectWallet } = useWeb3()
+	const { mergeAudio } = useAudioUtils()
 
 	if (!details) return null
 	const limitReached = details ? details.stems.length >= details.trackLimit : false
@@ -165,98 +163,109 @@ const ProjectDetails = (props: ProjectDetailsProps): JSX.Element | null => {
 
 				// Construct files and post to flattening service
 				const formData = new FormData()
-				files.forEach((data: Blob) => {
+				const blobData: Blob[] = []
+				let mergeAudioInputData: MergeAudioInput[] = []
+				files.forEach((data: Blob, name: string) => {
 					formData.append('files', data)
+					blobData.push(data)
+					mergeAudioInputData.push({
+						blob: data,
+						filename: name,
+					})
 				})
+				const stemHrefs: string[] = await details.stems.map(s => s.audioHref)
+				mergeAudioInputData = mergeAudioInputData.map((v, idx) => ({ ...v, href: stemHrefs[idx] }))
+				const song: Blob = await mergeAudio(mergeAudioInputData, 'mySong.wav')
+				console.log({ song })
 
 				// NOTE: We hit this directly with fetch because Next.js API routes have a 4MB limit
 				// See - https://nextjs.org/docs/messages/api-routes-response-size-limit
-				if (!process.env.PYTHON_HTTP_HOST) throw new Error('Flattening host not set.')
-				const response = await fetch(process.env.PYTHON_HTTP_HOST + '/merge', {
-					method: 'POST',
-					body: formData,
-				})
+				// 	if (!process.env.PYTHON_HTTP_HOST) throw new Error('Flattening host not set.')
+				// 	const response = await fetch(process.env.PYTHON_HTTP_HOST + '/merge', {
+				// 		method: 'POST',
+				// 		body: formData,
+				// 	})
 
 				// Catch flatten audio error
-				if (!response.ok) throw new Error('Failed to flatten the audio files')
-				if (response.body === null) throw new Error('Failed to flatten audio files, response body empty.')
+				// 	if (!response.ok) throw new Error('Failed to flatten the audio files')
+				// 	if (response.body === null) throw new Error('Failed to flatten audio files, response body empty.')
 
-				const flattenedAudioBlob = await response.blob()
-				if (!flattenedAudioBlob) throw new Error('Failed to flatten the audio files')
+				// 	const flattenedAudioBlob = await response.blob()
+				// 	if (!flattenedAudioBlob) throw new Error('Failed to flatten the audio files')
 
-				if (!mintingOpen) setMintingOpen(true)
-				setMintingMsg('Uploading to NFT.storage...')
+				// 	if (!mintingOpen) setMintingOpen(true)
+				// 	setMintingMsg('Uploading to NFT.storage...')
 
 				// Construct NFT.storage data and store
-				const nftsRes = await NFTStore.store({
-					name: details.name, // TODO: plus a version number?
-					description:
-						'An Arbor Audio NFT representing collaborative music from multiple contributors on the decentralized web.',
-					image: new Blob([Buffer.from(logoBinary, 'base64')], { type: 'image/*' }),
-					properties: {
-						createdOn: new Date().toISOString(),
-						createdBy: currentUser.address,
-						audio: flattenedAudioBlob,
-						collaborators: details.collaborators,
-						stems: details.stems.map((s: any) => s.metadataUrl),
-					},
-				})
+				// 	const nftsRes = await NFTStore.store({
+				// 		name: details.name, // TODO: plus a version number?
+				// 		description:
+				// 			'An Arbor Audio NFT representing collaborative music from multiple contributors on the decentralized web.',
+				// 		image: new Blob([Buffer.from(logoBinary, 'base64')], { type: 'image/*' }),
+				// 		properties: {
+				// 			createdOn: new Date().toISOString(),
+				// 			createdBy: currentUser.address,
+				// 			audio: flattenedAudioBlob,
+				// 			collaborators: details.collaborators,
+				// 			stems: details.stems.map((s: any) => s.metadataUrl),
+				// 		},
+				// 	})
 
 				// Check for data
-				if (!nftsRes) throw new Error('Failed to store on NFT.storage')
+				// 	if (!nftsRes) throw new Error('Failed to store on NFT.storage')
 
 				// Call smart contract and mint an nft out of the original CID
-				if (!mintingOpen) setMintingOpen(true)
-				setMintingMsg('Minting the NFT. This could take a moment...')
-				const amount = web3.utils.toWei('0.01', 'ether')
-				const mintRes: any = await contracts.nft.mintAndBuy(currentUser.address, nftsRes.url, details.collaborators, {
-					value: amount,
-					from: currentUser.address,
-					gasLimit: 2000000,
-				})
-				const receipt = await mintRes.wait()
+				// 	if (!mintingOpen) setMintingOpen(true)
+				// 	setMintingMsg('Minting the NFT. This could take a moment...')
+				// 	const amount = web3.utils.toWei('0.01', 'ether')
+				// 	const mintRes: any = await contracts.nft.mintAndBuy(currentUser.address, nftsRes.url, details.collaborators, {
+				// 		value: amount,
+				// 		from: currentUser.address,
+				// 		gasLimit: 2000000,
+				// 	})
+				// 	const receipt = await mintRes.wait()
 
 				// Add new NFT to database and user details
-				if (!mintingOpen) setMintingOpen(true)
-				setMintingMsg('Updating user details...')
-				const newNftPayload: INft = {
-					createdBy: currentUser.address,
-					owner: currentUser.address,
-					isListed: false,
-					listPrice: 0,
-					token: {
-						// TODO: Parse the correct arguments from the event receipt
-						id: parseInt(
-							receipt.events.TokenCreated?.returnValues.newTokenId ||
-								receipt.events.TokenCreated?.returnValues.tokenId ||
-								receipt.events.TokenCreated?.returnValues._tokenId ||
-								0, // default
-						),
-						tokenURI:
-							receipt.events.TokenCreated?.returnValues.newTokenURI ||
-							receipt.events.TokenCreated?.returnValues.tokenURI ||
-							receipt.events.TokenCreated?.returnValues._tokenURI ||
-							'', // default
-						data: mintRes,
-					},
-					name: details.name,
-					metadataUrl: nftsRes.url,
-					audioHref: nftsRes.data.properties.audio,
-					projectId: details._id.toString(),
-					collaborators: details.collaborators,
-					stems: details.stems, // Direct 1:1 deep clone
-				}
-				const nftCreated = await post('/nfts', newNftPayload)
-				if (!nftCreated.success) throw new Error(nftCreated.error)
+				// 	if (!mintingOpen) setMintingOpen(true)
+				// 	setMintingMsg('Updating user details...')
+				// 	const newNftPayload: INft = {
+				// 		createdBy: currentUser.address,
+				// 		owner: currentUser.address,
+				// 		isListed: false,
+				// 		listPrice: 0,
+				// 		token: {
+				// 			// TODO: Parse the correct arguments from the event receipt
+				// 			id: parseInt(
+				// 				receipt.events.TokenCreated?.returnValues.newTokenId ||
+				// 					receipt.events.TokenCreated?.returnValues.tokenId ||
+				// 					receipt.events.TokenCreated?.returnValues._tokenId ||
+				// 					0, // default
+				// 			),
+				// 			tokenURI:
+				// 				receipt.events.TokenCreated?.returnValues.newTokenURI ||
+				// 				receipt.events.TokenCreated?.returnValues.tokenURI ||
+				// 				receipt.events.TokenCreated?.returnValues._tokenURI ||
+				// 				'', // default
+				// 			data: mintRes,
+				// 		},
+				// 		name: details.name,
+				// 		metadataUrl: nftsRes.url,
+				// 		audioHref: nftsRes.data.properties.audio,
+				// 		projectId: details._id.toString(),
+				// 		collaborators: details.collaborators,
+				// 		stems: details.stems, // Direct 1:1 deep clone
+				// 	}
+				// 	const nftCreated = await post('/nfts', newNftPayload)
+				// 	if (!nftCreated.success) throw new Error(nftCreated.error)
 
 				// Notify success
-				if (!successOpen) setSuccessOpen(true)
-				setSuccessMsg('Success! You now own this music NFT, redirecting...')
-				setMinting(false)
-				setMintingMsg('')
+				// 	if (!successOpen) setSuccessOpen(true)
+				// 	setSuccessMsg('Success! You now own this music NFT, redirecting...')
+				// 	setMinting(false)
+				// 	setMintingMsg('')
 
 				// Route to user's profile page
-				router.push(`/users/${currentUser.address}`)
+				// 	router.push(`/users/${currentUser.address}`)
 			}
 		} catch (e: any) {
 			console.error(e)

components/StemQueue/StemQueue.tsx

@@ -8,6 +8,7 @@ import { useEffect, useState } from 'react'
 import type { IProjectDoc } from '../../models/project.model'
 import { IStemDoc } from '../../models/stem.model'
 import { IUserIdentity } from '../../models/user.model'
+import { useAudioUtils } from '../../utils/AudioUtilsProvider'
 import { update } from '../../utils/http'
 import signMessage from '../../utils/signMessage'
 import StemUploadDialog from '../StemUploadDialog'
@@ -54,6 +55,7 @@ const StemQueue = (props: StemQueueProps): JSX.Element => {
 	const [approveLoading, setApproveLoading] = useState<boolean>(false)
 	const [stems, setStems] = useState<Map<number, any>>(new Map())
 	const { contracts, currentUser, updateCurrentUser } = useWeb3()
+	const { getAudio } = useAudioUtils()
 
 	useEffect(() => {
 		const obj = {}
@@ -83,6 +85,7 @@ const StemQueue = (props: StemQueueProps): JSX.Element => {
 	// Play the wavesurfer file
 	const handlePlay = (id: number) => {
 		const stem = stems.get(id)
+		getAudio(stem.audioHref)
 		if (stem) stem.play()
 	}
 

next.config.js

@@ -18,7 +18,6 @@ const nextConfig = {
 		BLOCKNATIVE_KEY: process.env.BLOCKNATIVE_KEY,
 		NFT_STORAGE_KEY: process.env.NFT_STORAGE_KEY,
 		COVALENT_API_KEY: process.env.COVALENT_API_KEY,
-		PYTHON_HTTP_HOST: process.env.PYTHON_HTTP_HOST,
 		ALCHEMY_POLYGON_KEY: process.env.ALCHEMY_POLYGON_KEY,
 		ALCHEMY_POLYGON_TESTNET_KEY: process.env.ALCHEMY_POLYGON_TESTNET_KEY,
 	},
@@ -37,10 +36,19 @@ const nextConfig = {
 	// Ensure there's a direct match in vercel.json
 	async headers() {
 		return [
+			{
+				// ffmpeg.wasm support for all routes
+				source: '/:path*',
+				headers: [
+					{ key: 'Cross-Origin-Embedder-Policy', value: 'require-corp' },
+					{ key: 'Cross-Origin-Opener-Policy', value: 'same-origin' },
+				],
+			},
 			{
 				// matching all API routes
 				source: '/api/:path*',
 				headers: [
+					// Access control
 					{ key: 'Access-Control-Allow-Credentials', value: 'true' },
 					// { key: 'Access-Control-Allow-Origin', value: 'https://arbor-pr-*.herokuapp.com' },
 					// { key: 'Access-Control-Allow-Origin', value: 'https://ui-*-arbor-protocol.vercel.app' },
@@ -60,13 +68,73 @@ const nextConfig = {
 		if (!options.isServer) {
 			config.resolve.fallback.fs = false
 		}
+
+		config.module.rules.push({
+			test: /\.wasm$/,
+			type: 'webassembly/sync',
+		})
+
+		// // From https://github.com/vercel/next.js/issues/22581#issuecomment-864476385
+		// const ssrPlugin = config.plugins.find(plugin => plugin instanceof SSRPlugin)
+
+		// // Patch the NextJsSSRImport plugin to not throw with WASM generated chunks.
+		// function patchSsrPlugin(plugin) {
+		// 	plugin.apply = function apply(compiler) {
+		// 		compiler.hooks.compilation.tap('NextJsSSRImport', compilation => {
+		// 			compilation.mainTemplate.hooks.requireEnsure.tap('NextJsSSRImport', (code, chunk) => {
+		// 				// The patch that we need to ensure this plugin doesn't throw
+		// 				// with WASM chunks.
+		// 				if (!chunk.name) {
+		// 					return
+		// 				}
+
+		// 				// Update to load chunks from our custom chunks directory
+		// 				const outputPath = resolve('/')
+		// 				const pagePath = join('/', dirname(chunk.name))
+		// 				const relativePathToBaseDir = relative(pagePath, outputPath)
+		// 				// Make sure even in windows, the path looks like in unix
+		// 				// Node.js require system will convert it accordingly
+		// 				const relativePathToBaseDirNormalized = relativePathToBaseDir.replace(/\\/g, '/')
+		// 				return code
+		// 					.replace('require("./"', `require("${relativePathToBaseDirNormalized}/"`)
+		// 					.replace('readFile(join(__dirname', `readFile(join(__dirname, "${relativePathToBaseDirNormalized}"`)
+		// 			})
+		// 		})
+		// 	}
+		// }
+
+		// if (ssrPlugin) {
+		// 	patchSsrPlugin(ssrPlugin)
+		// }
+
 		return {
 			...config,
+			// Ensures that web workers can import scripts.
+			// output: {
+			// 	publicPath: '/_next/',
+			// },
 			// WASM support
+			// module: {
+			// 	...config.module,
+			// 	rules: [
+			// 		...config.module.rules,
+			// 		{
+			// 			test: /\.wasm$/,
+			// 			type: 'webassemblly/sync',
+			// 		},
+			// 	],
+			// },
 			experiments: {
 				asyncWebAssembly: true,
 				layers: true,
 			},
+			// From https://github.com/wasm-tool/wasm-pack-plugin
+			// plugins: [
+			// 	new WasmPackPlugin({
+			// 		crateDirectory: resolve('./rust'),
+			// 		args: '--log-level warn',
+			// 	}),
+			// ],
 		}
 	},
 }

package.json

@@ -28,7 +28,9 @@
     "report-gas": "REPORT_GAS=true hardhat test"
   },
   "engines": {
-    "node": ">=16"
+    "node": ">=16 <17",
+    "npm": ">=9 <10",
+    "yarn": ">=1.22.1"
   },
   "nextBundleAnalysis": {
     "budget": null,
@@ -38,10 +40,12 @@
   "dependencies": {
     "@emotion/react": "^11.9.3",
     "@emotion/styled": "^11.9.3",
+    "@ffmpeg/core": "^0.11.0",
+    "@ffmpeg/ffmpeg": "^0.11.6",
     "@metamask/detect-provider": "^1.2.0",
     "@mui/icons-material": "^5.8.4",
     "@mui/material": "^5.8.7",
-		"@next/bundle-analyzer": "^13.0.1",
+    "@next/bundle-analyzer": "^13.0.1",
     "@openzeppelin/contracts": "^4.7.3",
     "@semaphore-protocol/group": "^2.0.0",
     "@semaphore-protocol/identity": "^2.0.0",