Sound-Box.html


<html>

	<head>

		<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
		<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-converter"></script>
		<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/handpose"></script>
		<script src="https://unpkg.com/tone"></script>

	</head>

	<body>
		<video width=640 height=480 autoplay muted id="camera">
		</video>

		<canvas width=640 height=480 id="sound_box"> </canvas>
		<canvas width=640 height=480 id="sound_box_copy" style="visibility:hidden"> </canvas>

		<script>

			// generate sounds using hand gestures
			async function track_hand()
			{
				// load camera stream
				const stream = document.getElementById("camera");

				// load hand-pose model
				const model = await handpose.load();

				// load synthesizer
				var synth = new Tone.Synth().toMaster();

				// create sound box
				const canvas = document.getElementById("sound_box");
				const draw = canvas.getContext("2d");
				draw.globalAlpha = 0.6;

				// generate random color codes for each sound box
				const letters = "0123456789ABCDEF";

				// each box is 80 x 80
				for(var x=0; x<640; x=x+80)
				{
					for(var y=0; y<480; y=y+80)
					{
						var color_code = "#";
						// 6-digit HTML color code
						for(i=0; i < 6; i++)
						{
							var k = Math.floor(Math.random() * 16);
							color_code += letters[k];
						}
						// paint cell with generated color code
						draw.fillStyle = color_code;
						draw.fillRect(x, y, 80, 80);
					}
				}

				// save copy of sound box
				const canvas2 = document.getElementById("sound_box_copy");
				const draw2 = canvas2.getContext("2d");
				draw2.globalAlpha = 0.6;
				draw2.drawImage(canvas, 0, 0);

				while(1)
				{
					// refresh sound box
					draw.drawImage(canvas2, 0, 0);

					// track hand position
					const result = await model.estimateHands(stream);

					// check if hand is detected
					if(result.length > 0)
					{
						// get hand co-ordinates
						const hand = result[0];

						// get index finger tip position
						var index = hand.annotations.indexFinger;

						var index_x = Math.round(index[3][0]);
						var index_y = Math.round(index[3][1]);
						console.log(index_x, index_y);

						// map hand-position to sound-box position
						var x = Math.floor(index_x / 80);
						var y = Math.floor(index_y / 80);
						console.log(x, y);

						// highlight corresponding box
						draw.fillStyle = "#000000";
						draw.fillRect(x * 80, y * 80, 80, 80);

						// play synthesizer sound

						// oscillator frequency is function of box position (x+y)
						synth.triggerAttackRelease((x+y)*100, '4n');
					}
					// loop to process the next frame
					await tf.nextFrame();
				}
			}

			// capture live video stream from web camera
			video = document.getElementById("camera");
			if(navigator.mediaDevices.getUserMedia)
			{
				navigator.mediaDevices.getUserMedia({video: true})
					.then(function (stream) {video.srcObject = stream; });
			}

			// detect gestures once the video is ready
			main();

			function main()
			{
				// check if the video is loaded and ready for processing
				if(video.readyState == 4)
				{
					console.log("video is ready for processing..");
					track_hand();
				}
				else
				{
					console.log("nope, not ready yet..");
					setTimeout(main, 1000/30);
				}
			}

	  </script>

	</body>
</html>

	<html>

	<head>

	<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
	<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-converter"></script>
	<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/handpose"></script>
	<script src="https://unpkg.com/tone"></script>

	</head>

	<body>
	<video width=640 height=480 autoplay muted id="camera">
	</video>

	<canvas width=640 height=480 id="sound_box"> </canvas>
	<canvas width=640 height=480 id="sound_box_copy" style="visibility:hidden"> </canvas>

	<script>

	// generate sounds using hand gestures
	async function track_hand()
	{
	// load camera stream
	const stream = document.getElementById("camera");

	// load hand-pose model
	const model = await handpose.load();

	// load synthesizer
	var synth = new Tone.Synth().toMaster();

	// create sound box
	const canvas = document.getElementById("sound_box");
	const draw = canvas.getContext("2d");
	draw.globalAlpha = 0.6;

	// generate random color codes for each sound box
	const letters = "0123456789ABCDEF";

	// each box is 80 x 80
	for(var x=0; x<640; x=x+80)
	{
	for(var y=0; y<480; y=y+80)
	{
	var color_code = "#";
	// 6-digit HTML color code
	for(i=0; i < 6; i++)
	{
	var k = Math.floor(Math.random() * 16);
	color_code += letters[k];
	}
	// paint cell with generated color code
	draw.fillStyle = color_code;
	draw.fillRect(x, y, 80, 80);
	}
	}

	// save copy of sound box
	const canvas2 = document.getElementById("sound_box_copy");
	const draw2 = canvas2.getContext("2d");
	draw2.globalAlpha = 0.6;
	draw2.drawImage(canvas, 0, 0);

	while(1)
	{
	// refresh sound box
	draw.drawImage(canvas2, 0, 0);

	// track hand position
	const result = await model.estimateHands(stream);

	// check if hand is detected
	if(result.length > 0)
	{
	// get hand co-ordinates
	const hand = result[0];

	// get index finger tip position
	var index = hand.annotations.indexFinger;

	var index_x = Math.round(index[3][0]);
	var index_y = Math.round(index[3][1]);
	console.log(index_x, index_y);

	// map hand-position to sound-box position
	var x = Math.floor(index_x / 80);
	var y = Math.floor(index_y / 80);
	console.log(x, y);

	// highlight corresponding box
	draw.fillStyle = "#000000";
	draw.fillRect(x * 80, y * 80, 80, 80);

	// play synthesizer sound

	// oscillator frequency is function of box position (x+y)
	synth.triggerAttackRelease((x+y)*100, '4n');
	}
	// loop to process the next frame
	await tf.nextFrame();
	}
	}

	// capture live video stream from web camera
	video = document.getElementById("camera");
	if(navigator.mediaDevices.getUserMedia)
	{
	navigator.mediaDevices.getUserMedia({video: true})
	.then(function (stream) {video.srcObject = stream; });
	}

	// detect gestures once the video is ready
	main();

	function main()
	{
	// check if the video is loaded and ready for processing
	if(video.readyState == 4)
	{
	console.log("video is ready for processing..");
	track_hand();
	}
	else
	{
	console.log("nope, not ready yet..");
	setTimeout(main, 1000/30);
	}
	}

	</script>

	</body>
	</html>