Skip to content

Commit

Permalink
support conversion of binary packs to fs folder packs + enforce mp3 r…
Browse files Browse the repository at this point in the history 
…eencoding + cleanup + logs
  • Loading branch information
@mullermarian
mullermarian committed Jan 8, 2021
1 parent 120b619 commit 56ee1e6
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 163 deletions.
241 changes: 111 additions & 130 deletions core/src/main/java/studio/core/v1/utils/AudioConversion.java
View file
Expand Up @@ -12,153 +12,134 @@
import javax.sound.sampled.*;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;

public class AudioConversion {

private static final float WAVE_SAMPLE_RATE = 32000.0f;
private static final float OGG_SAMPLE_RATE = 44100.0f;
private static final float MP3_SAMPLE_RATE = 44100.0f;
private static final int BITSIZE = 16;
private static final int MP3_BITSIZE = 32;
private static final int CHANNELS = 1;
public static final float WAVE_SAMPLE_RATE = 32000.0f;
public static final float OGG_SAMPLE_RATE = 44100.0f;
public static final float MP3_SAMPLE_RATE = 44100.0f;
public static final int BITSIZE = 16;
public static final int MP3_BITSIZE = 32;
public static final int CHANNELS = 1;


public static byte[] oggToWave(byte[] oggData) throws IOException {
public static byte[] oggToWave(byte[] oggData) throws Exception {
return anyToWave(oggData);
}

public static byte[] mp3ToWave(byte[] mp3Data) throws IOException {
public static byte[] mp3ToWave(byte[] mp3Data) throws Exception {
return anyToWave(mp3Data);
}

public static byte[] anyToWave(byte[] data) throws IOException {
try {
AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new ByteArrayInputStream(data));

// First, convert to PCM
AudioFormat pcmFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
inputAudio.getFormat().getSampleRate(),
BITSIZE,
inputAudio.getFormat().getChannels(),
inputAudio.getFormat().getChannels()*2,
inputAudio.getFormat().getSampleRate(),
false
);
AudioInputStream pcm = AudioSystem.getAudioInputStream(pcmFormat, inputAudio);

// Then, convert sample rate to 32000Hz, and to mono channel (the only format that is supported by the story teller device)
AudioFormat pcm32000Format = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
WAVE_SAMPLE_RATE,
BITSIZE,
CHANNELS,
CHANNELS *2,
WAVE_SAMPLE_RATE,
false
);
AudioInputStream pcm32000 = AudioSystem.getAudioInputStream(pcm32000Format, pcm);

// Read the whole stream in a byte array because length must be known
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos.writeBytes(pcm32000.readAllBytes());
AudioInputStream waveStream = new AudioInputStream(new ByteArrayInputStream(baos.toByteArray()), pcm32000Format, baos.toByteArray().length);

ByteArrayOutputStream output = new ByteArrayOutputStream();
AudioSystem.write(waveStream, AudioFileFormat.Type.WAVE, output);
return output.toByteArray();
} catch (UnsupportedAudioFileException e) {
e.printStackTrace();
throw new IOException("Unsupported audio format", e);
}
public static byte[] anyToWave(byte[] data) throws Exception {
AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new ByteArrayInputStream(data));

// First, convert to PCM
AudioFormat pcmFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
inputAudio.getFormat().getSampleRate(),
BITSIZE,
inputAudio.getFormat().getChannels(),
inputAudio.getFormat().getChannels()*2,
inputAudio.getFormat().getSampleRate(),
false
);
AudioInputStream pcm = AudioSystem.getAudioInputStream(pcmFormat, inputAudio);

// Then, convert sample rate to 32000Hz, and to mono channel (the only format that is supported by the story teller device)
AudioFormat pcm32000Format = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
WAVE_SAMPLE_RATE,
BITSIZE,
CHANNELS,
CHANNELS *2,
WAVE_SAMPLE_RATE,
false
);
AudioInputStream pcm32000 = AudioSystem.getAudioInputStream(pcm32000Format, pcm);

// Read the whole stream in a byte array because length must be known
ByteArrayOutputStream baos = new ByteArrayOutputStream();
baos.writeBytes(pcm32000.readAllBytes());
AudioInputStream waveStream = new AudioInputStream(new ByteArrayInputStream(baos.toByteArray()), pcm32000Format, baos.toByteArray().length);

ByteArrayOutputStream output = new ByteArrayOutputStream();
AudioSystem.write(waveStream, AudioFileFormat.Type.WAVE, output);
return output.toByteArray();
}

public static byte[] waveToOgg(byte[] waveData) throws IOException {
try {
AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new ByteArrayInputStream(waveData));

// First, convert sample rate to 44100Hz (the only rate that is supported by the vorbis encoding library)
AudioFormat pcm44100Format = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
OGG_SAMPLE_RATE,
BITSIZE,
CHANNELS,
CHANNELS*2,
OGG_SAMPLE_RATE,
false
);
AudioInputStream pcm44100 = AudioSystem.getAudioInputStream(pcm44100Format, inputAudio);

return VorbisEncoder.encode(pcm44100);
} catch (UnsupportedAudioFileException e) {
e.printStackTrace();
throw new IOException("Unsupported audio format", e);
} catch (VorbisEncodingException e) {
e.printStackTrace();
throw new IOException("Audio compression to ogg format failed", e);
}
public static byte[] waveToOgg(byte[] waveData) throws Exception {
AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new ByteArrayInputStream(waveData));

// First, convert sample rate to 44100Hz (the only rate that is supported by the vorbis encoding library)
AudioFormat pcm44100Format = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
OGG_SAMPLE_RATE,
BITSIZE,
CHANNELS,
CHANNELS * 2,
OGG_SAMPLE_RATE,
false
);
AudioInputStream pcm44100 = AudioSystem.getAudioInputStream(pcm44100Format, inputAudio);

return VorbisEncoder.encode(pcm44100);
}

public static byte[] anyToMp3(byte[] data) throws IOException {
try {
public static byte[] anyToMp3(byte[] data) throws Exception {
AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new ByteArrayInputStream(data));

// First, convert to PCM
AudioFormat pcmFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
inputAudio.getFormat().getSampleRate(),
BITSIZE,
inputAudio.getFormat().getChannels(),
inputAudio.getFormat().getChannels()*2,
inputAudio.getFormat().getSampleRate(),
false
);
AudioInputStream pcm = AudioSystem.getAudioInputStream(pcmFormat, inputAudio);

// Then, convert to mono **and oversample** (apparently the input stream in always empty unless the sample rate changes)
AudioFormat pcmOverSampledFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
inputAudio.getFormat().getSampleRate()*2,
BITSIZE,
CHANNELS,
CHANNELS*2,
inputAudio.getFormat().getSampleRate()*2,
false
);
AudioInputStream pcmOverSampled = AudioSystem.getAudioInputStream(pcmOverSampledFormat, pcm);

// Finally, convert sample rate to 44100Hz and sample bitsize to 32 bits
AudioFormat pcm44100Format = new AudioFormat(
AudioFormat.Encoding.PCM_FLOAT,
MP3_SAMPLE_RATE,
MP3_BITSIZE,
CHANNELS,
CHANNELS*4,
MP3_SAMPLE_RATE,
false
);
AudioInputStream pcm44100 = AudioSystem.getAudioInputStream(pcm44100Format, pcmOverSampled);

LameEncoder encoder = new LameEncoder(pcm44100.getFormat(), 64, MPEGMode.MONO.ordinal(), 1, false);

ByteArrayOutputStream mp3 = new ByteArrayOutputStream();
byte[] inputBuffer = new byte[encoder.getPCMBufferSize()];
byte[] outputBuffer = new byte[encoder.getPCMBufferSize()];

int bytesRead;
int bytesWritten;

while (0 < (bytesRead = pcm44100.read(inputBuffer))) {
bytesWritten = encoder.encodeBuffer(inputBuffer, 0, bytesRead, outputBuffer);
mp3.write(outputBuffer, 0, bytesWritten);
}

encoder.close();
return mp3.toByteArray();
} catch (UnsupportedAudioFileException e) {
e.printStackTrace();
throw new IOException("Unsupported audio format", e);
// First, convert to PCM
AudioFormat pcmFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
inputAudio.getFormat().getSampleRate(),
BITSIZE,
inputAudio.getFormat().getChannels(),
inputAudio.getFormat().getChannels() * 2,
inputAudio.getFormat().getSampleRate(),
false
);
AudioInputStream pcm = AudioSystem.getAudioInputStream(pcmFormat, inputAudio);

// Then, convert to mono **and oversample** (apparently the input stream in always empty unless the sample rate changes)
AudioFormat pcmOverSampledFormat = new AudioFormat(
AudioFormat.Encoding.PCM_SIGNED,
inputAudio.getFormat().getSampleRate() * 2,
BITSIZE,
CHANNELS,
CHANNELS * 2,
inputAudio.getFormat().getSampleRate() * 2,
false
);
AudioInputStream pcmOverSampled = AudioSystem.getAudioInputStream(pcmOverSampledFormat, pcm);

// Finally, convert sample rate to 44100Hz and sample bitsize to 32 bits
AudioFormat pcm44100Format = new AudioFormat(
AudioFormat.Encoding.PCM_FLOAT,
MP3_SAMPLE_RATE,
MP3_BITSIZE,
CHANNELS,
CHANNELS * 4,
MP3_SAMPLE_RATE,
false
);
AudioInputStream pcm44100 = AudioSystem.getAudioInputStream(pcm44100Format, pcmOverSampled);

LameEncoder encoder = new LameEncoder(pcm44100.getFormat(), 64, MPEGMode.MONO.ordinal(), 1, false);

ByteArrayOutputStream mp3 = new ByteArrayOutputStream();
byte[] inputBuffer = new byte[encoder.getPCMBufferSize()];
byte[] outputBuffer = new byte[encoder.getPCMBufferSize()];

int bytesRead;
int bytesWritten;

while (0 < (bytesRead = pcm44100.read(inputBuffer))) {
bytesWritten = encoder.encodeBuffer(inputBuffer, 0, bytesRead, outputBuffer);
mp3.write(outputBuffer, 0, bytesWritten);
}

encoder.close();
return mp3.toByteArray();
}
}
30 changes: 26 additions & 4 deletions core/src/main/java/studio/core/v1/utils/PackAssetsCompression.java
View file
Expand Up @@ -10,11 +10,16 @@
import studio.core.v1.model.StageNode;
import studio.core.v1.model.StoryPack;

import java.io.IOException;
import javax.sound.sampled.AudioFileFormat;
import javax.sound.sampled.AudioSystem;
import java.io.ByteArrayInputStream;
import java.util.TreeMap;
import java.util.logging.Logger;

public class PackAssetsCompression {

private static final Logger LOGGER = Logger.getLogger(PackAssetsCompression.class.getName());

public static boolean hasCompressedAssets(StoryPack pack) {
for (int i = 0; i < pack.getStageNodes().size(); i++) {
StageNode node = pack.getStageNodes().get(i);
Expand All @@ -31,7 +36,7 @@ public static boolean hasCompressedAssets(StoryPack pack) {
return false;
}

public static StoryPack withCompressedAssets(StoryPack pack) throws IOException {
public static StoryPack withCompressedAssets(StoryPack pack) throws Exception {
// Store compressed assets bytes
TreeMap<String, byte[]> assets = new TreeMap<>();

Expand All @@ -43,6 +48,7 @@ public static StoryPack withCompressedAssets(StoryPack pack) throws IOException
String assetHash = DigestUtils.sha1Hex(imageData);
if (assets.get(assetHash) == null) {
if ("image/bmp".equals(node.getImage().getMimeType())) {
LOGGER.fine("Compressing BMP image asset into PNG");
imageData = ImageConversion.bitmapToPng(imageData);
}
assets.put(assetHash, imageData);
Expand All @@ -59,6 +65,7 @@ public static StoryPack withCompressedAssets(StoryPack pack) throws IOException
String assetHash = DigestUtils.sha1Hex(audioData);
if (assets.get(assetHash) == null) {
if ("audio/x-wav".equals(node.getAudio().getMimeType())) {
LOGGER.fine("Compressing WAV audio asset into OGG");
audioData = AudioConversion.waveToOgg(audioData);
node.getAudio().setMimeType("audio/ogg");
}
Expand All @@ -75,7 +82,7 @@ public static StoryPack withCompressedAssets(StoryPack pack) throws IOException
return pack;
}

public static StoryPack withUncompressedAssets(StoryPack pack) throws IOException {
public static StoryPack withUncompressedAssets(StoryPack pack) throws Exception {
// Store uncompressed assets bytes
TreeMap<String, byte[]> assets = new TreeMap<>();

Expand All @@ -88,14 +95,17 @@ public static StoryPack withUncompressedAssets(StoryPack pack) throws IOExceptio
if (assets.get(assetHash) == null) {
switch (node.getImage().getMimeType()) {
case "image/png":
LOGGER.fine("Uncompressing PNG image asset into BMP");
imageData = ImageConversion.anyToBitmap(imageData);
break;
case "image/jpeg":
LOGGER.fine("Uncompressing JPG image asset into BMP");
imageData = ImageConversion.anyToBitmap(imageData);
break;
case "image/bmp":
// Convert from 4-bits depth / RLE encoding BMP
if (imageData[28] == 0x04 && imageData[30] == 0x02) {
LOGGER.fine("Uncompressing 4-bits/RLE BMP image asset into BMP");
imageData = ImageConversion.anyToBitmap(imageData);
}
break;
Expand All @@ -114,9 +124,11 @@ public static StoryPack withUncompressedAssets(StoryPack pack) throws IOExceptio
if (!"audio/x-wav".equals(node.getAudio().getMimeType())) {
switch (node.getAudio().getMimeType()) {
case "audio/ogg":
LOGGER.fine("Uncompressing OGG audio asset into WAV");
audioData = AudioConversion.oggToWave(audioData);
break;
case "audio/mpeg":
LOGGER.fine("Uncompressing MP3 audio asset into WAV");
audioData = AudioConversion.mp3ToWave(audioData);
break;
}
Expand All @@ -132,7 +144,7 @@ public static StoryPack withUncompressedAssets(StoryPack pack) throws IOExceptio
return pack;
}

public static StoryPack withPreparedAssetsFirmware2dot4(StoryPack pack) throws IOException {
public static StoryPack withPreparedAssetsFirmware2dot4(StoryPack pack) throws Exception {
// Store prepared assets bytes
TreeMap<String, byte[]> assets = new TreeMap<>();

Expand All @@ -145,6 +157,7 @@ public static StoryPack withPreparedAssetsFirmware2dot4(StoryPack pack) throws I
if (assets.get(assetHash) == null) {
// Convert to 4-bits depth / RLE encoding BMP
if (!"image/bmp".equals(node.getImage().getMimeType()) || imageData[28] != 0x04 || imageData[30] != 0x02) {
LOGGER.fine("Converting image asset into 4-bits/RLE BMP");
imageData = ImageConversion.anyToRLECompressedBitmap(imageData);
}
assets.put(assetHash, imageData);
Expand All @@ -159,7 +172,16 @@ public static StoryPack withPreparedAssetsFirmware2dot4(StoryPack pack) throws I
String assetHash = DigestUtils.sha1Hex(audioData);
if (assets.get(assetHash) == null) {
if (!"audio/mp3".equals(node.getAudio().getMimeType()) && !"audio/mpeg".equals(node.getAudio().getMimeType())) {
LOGGER.fine("Converting audio asset into MP3");
audioData = AudioConversion.anyToMp3(audioData);
} else {
// Check that the file is MONO / 44100Hz
AudioFileFormat audioFileFormat = AudioSystem.getAudioFileFormat(new ByteArrayInputStream(audioData));
if (audioFileFormat.getFormat().getChannels() != AudioConversion.CHANNELS
|| audioFileFormat.getFormat().getSampleRate() != AudioConversion.MP3_SAMPLE_RATE) {
LOGGER.fine("Re-encoding MP3 audio asset");
audioData = AudioConversion.anyToMp3(audioData);
}
}
assets.put(assetHash, audioData);
}
Expand Down

0 comments on commit 56ee1e6

Please sign in to comment.