Skip to content

Commit

Permalink
Voice Gateway V4 and new encryption modes (#651)
Browse files Browse the repository at this point in the history 
* Added support for suffix/lite encryption
* Move to vgw v4
* Use IP given from discord payload rather than original endpoint
* Reduce required allocations for en-/decryption
* Remove un-used createEchoPacket method
* Fix deadlock when changing connection listener in its own methods
* Reduce packet provider to single buffer
  • Loading branch information
@MinnDevelopment
MinnDevelopment committed Jul 11, 2018
1 parent da41772 commit 16437e1
Show file tree
Hide file tree
Showing 11 changed files with 514 additions and 144 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Expand Up @@ -65,7 +65,7 @@ dependencies {
compile 'club.minnced:opus-java:1.0.2'

//Web Connection Support
compile 'com.neovisionaries:nv-websocket-client:2.2'
compile 'com.neovisionaries:nv-websocket-client:2.4'
compile 'com.squareup.okhttp3:okhttp:3.8.1'

//Sets the dependencies for the examples
Expand Down
139 changes: 102 additions & 37 deletions src/main/java/net/dv8tion/jda/core/audio/AudioConnection.java
View file
Expand Up @@ -16,6 +16,7 @@

package net.dv8tion.jda.core.audio;

import com.iwebpp.crypto.TweetNaclFast;
import com.sun.jna.ptr.PointerByReference;
import gnu.trove.map.TIntLongMap;
import gnu.trove.map.TIntObjectMap;
Expand All @@ -39,10 +40,7 @@
import org.slf4j.MDC;
import tomp2p.opuswrapper.Opus;

import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.*;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
Expand All @@ -59,6 +57,8 @@ public class AudioConnection
public static final int OPUS_FRAME_TIME_AMOUNT = 20;//This is 20 milliseconds. We are only dealing with 20ms opus packets.
public static final int OPUS_CHANNEL_COUNT = 2; //We want to use stereo. If the audio given is mono, the encoder promotes it
// to Left and Right mono (stereo that is the same on both sides)
public static final long MAX_UINT_32 = 4294967295L;

private final TIntLongMap ssrcMap = new TIntLongHashMap();
private final TIntObjectMap<Decoder> opusDecoders = new TIntObjectHashMap<>();
private final HashMap<User, Queue<Pair<Long, short[]>>> combinedQueue = new HashMap<>();
Expand Down Expand Up @@ -338,7 +338,7 @@ private synchronized void setupReceiveThread()
couldReceive = true;
sendSilentPackets();
}
AudioPacket decryptedPacket = AudioPacket.decryptAudioPacket(receivedPacket, webSocket.getSecretKey());
AudioPacket decryptedPacket = AudioPacket.decryptAudioPacket(webSocket.encryption, receivedPacket, webSocket.getSecretKey());
if (decryptedPacket == null)
continue;

Expand Down Expand Up @@ -380,12 +380,6 @@ private synchronized void setupReceiveThread()
LOG.warn("Received audio data with a known SSRC, but the userId associate with the SSRC is unknown to JDA!");
continue;
}
// if (decoder.wasPacketLost(decryptedPacket.getSequence()))
// {
// LOG.debug("Packet(s) missed. Using Opus packetloss-compensation.");
// short[] decodedAudio = decoder.decodeFromOpus(null);
// receiveHandler.handleUserAudio(new UserAudio(user, decodedAudio));
// }
short[] decodedAudio = decoder.decodeFromOpus(decryptedPacket);

//If decodedAudio is null, then the Opus decode failed, so throw away the packet.
Expand Down Expand Up @@ -555,8 +549,12 @@ private byte[] encodeToOpus(byte[] rawAudio)
}
((Buffer) nonEncodedBuffer).flip();

//TODO: check for 0 / negative value for error.
int result = Opus.INSTANCE.opus_encode(opusEncoder, nonEncodedBuffer, OPUS_FRAME_SIZE, encoded, encoded.capacity());
if (result <= 0)
{
LOG.error("Received error code from opus_encode(...): {}", result);
return null;
}

//ENCODING STOPS HERE

Expand All @@ -569,7 +567,7 @@ private void setSpeaking(boolean isSpeaking)
{
this.speaking = isSpeaking;
JSONObject obj = new JSONObject()
.put("speaking", isSpeaking)
.put("speaking", isSpeaking ? 1 : 0)
.put("delay", 0);
webSocket.send(VoiceCode.USER_SPEAKING_UPDATE, obj);
if (!isSpeaking)
Expand All @@ -589,7 +587,7 @@ public AudioWebSocket getWebSocket()

@Override
@Deprecated
protected void finalize() throws Throwable
protected void finalize()
{
shutdown();
}
Expand All @@ -598,6 +596,9 @@ private class PacketProvider implements IPacketProvider
{
char seq = 0; //Sequence of audio packets. Used to determine the order of the packets.
int timestamp = 0; //Used to sync up our packets within the same timeframe of other people talking.
private long nonce = 0;
private ByteBuffer buffer = ByteBuffer.allocate(512);
private final byte[] nonceBuffer = new byte[TweetNaclFast.SecretBox.nonceLength];

@Override
public String getIdentifier()
Expand All @@ -617,14 +618,26 @@ public DatagramSocket getUdpSocket()
return AudioConnection.this.udpSocket;
}

@Override
public InetSocketAddress getSocketAddress()
{
return webSocket.getAddress();
}

@Override
public DatagramPacket getNextPacket(boolean changeTalking)
{
DatagramPacket nextPacket = null;
ByteBuffer buffer = getNextPacketRaw(changeTalking);
return buffer == null ? null : getDatagramPacket(buffer);
}

@Override
public ByteBuffer getNextPacketRaw(boolean changeTalking)
{
ByteBuffer nextPacket = null;
try
{
if (sentSilenceOnConnect && sendHandler != null && sendHandler.canProvide())
cond: if (sentSilenceOnConnect && sendHandler != null && sendHandler.canProvide())
{
silenceCounter = -1;
byte[] rawAudio = sendHandler.provide20MsAudio();
Expand All @@ -637,25 +650,14 @@ public DatagramPacket getNextPacket(boolean changeTalking)
{
if (!sendHandler.isOpus())
{
if (opusEncoder == null)
{
if (!AudioNatives.ensureOpus())
{
if (!printedError)
LOG.error("Unable to process PCM audio without opus binaries!");
printedError = true;
return null;
}
IntBuffer error = IntBuffer.allocate(4);
opusEncoder = Opus.INSTANCE.opus_encoder_create(OPUS_SAMPLE_RATE, OPUS_CHANNEL_COUNT, Opus.OPUS_APPLICATION_AUDIO, error);
}
rawAudio = encodeToOpus(rawAudio);
rawAudio = encodeAudio(rawAudio);
if (rawAudio == null)
break cond;
}
AudioPacket packet = new AudioPacket(seq, timestamp, webSocket.getSSRC(), rawAudio);
if (!speaking)
setSpeaking(true);

nextPacket = packet.asEncryptedUdpPacket(webSocket.getAddress(), webSocket.getSecretKey());
nextPacket = getPacketData(rawAudio);
if (!speaking && changeTalking)
setSpeaking(true);

if (seq + 1 > Character.MAX_VALUE)
seq = 0;
Expand All @@ -665,10 +667,7 @@ public DatagramPacket getNextPacket(boolean changeTalking)
}
else if (silenceCounter > -1)
{
AudioPacket packet = new AudioPacket(seq, timestamp, webSocket.getSSRC(), silenceBytes);

nextPacket = packet.asEncryptedUdpPacket(webSocket.getAddress(), webSocket.getSecretKey());

nextPacket = getPacketData(silenceBytes);
if (seq + 1 > Character.MAX_VALUE)
seq = 0;
else
Expand All @@ -681,7 +680,9 @@ else if (silenceCounter > -1)
}
}
else if (speaking && changeTalking)
{
setSpeaking(false);
}
}
catch (Exception e)
{
Expand All @@ -694,6 +695,70 @@ else if (speaking && changeTalking)
return nextPacket;
}

private byte[] encodeAudio(byte[] rawAudio)
{
if (opusEncoder == null)
{
if (!AudioNatives.ensureOpus())
{
if (!printedError)
LOG.error("Unable to process PCM audio without opus binaries!");
printedError = true;
return null;
}
IntBuffer error = IntBuffer.allocate(1);
opusEncoder = Opus.INSTANCE.opus_encoder_create(OPUS_SAMPLE_RATE, OPUS_CHANNEL_COUNT, Opus.OPUS_APPLICATION_AUDIO, error);
if (error.get() != Opus.OPUS_OK && opusEncoder == null)
{
LOG.error("Received error status from opus_encoder_create(...): {}", error.get());
return null;
}
}
return encodeToOpus(rawAudio);
}

private DatagramPacket getDatagramPacket(ByteBuffer b)
{
byte[] data = b.array();
int offset = b.arrayOffset();
int position = b.position();
return new DatagramPacket(data, offset, position - offset, webSocket.getAddress());
}

private ByteBuffer getPacketData(byte[] rawAudio)
{
AudioPacket packet = new AudioPacket(seq, timestamp, webSocket.getSSRC(), rawAudio);
int nlen;
switch (webSocket.encryption)
{
case XSALSA20_POLY1305:
nlen = 0;
break;
case XSALSA20_POLY1305_LITE:
if (nonce >= MAX_UINT_32)
loadNextNonce(nonce = 0);
else
loadNextNonce(++nonce);
nlen = 4;
break;
case XSALSA20_POLY1305_SUFFIX:
ThreadLocalRandom.current().nextBytes(nonceBuffer);
nlen = TweetNaclFast.SecretBox.nonceLength;
break;
default:
throw new IllegalStateException("Encryption mode [" + webSocket.encryption + "] is not supported!");
}
return buffer = packet.asEncryptedPacket(buffer, webSocket.getSecretKey(), nonceBuffer, nlen);
}

private void loadNextNonce(long nonce)
{
nonceBuffer[0] = (byte) ((nonce >>> 24) & 0xFF);
nonceBuffer[1] = (byte) ((nonce >>> 16) & 0xFF);
nonceBuffer[2] = (byte) ((nonce >>> 8) & 0xFF);
nonceBuffer[3] = (byte) ( nonce & 0xFF);
}

@Override
public void onConnectionError(ConnectionStatus status)
{
Expand Down
57 changes: 57 additions & 0 deletions src/main/java/net/dv8tion/jda/core/audio/AudioEncryption.java
View file
@@ -0,0 +1,57 @@
/*
* Copyright 2015-2018 Austin Keener & Michael Ritter & Florian Spieß
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package net.dv8tion.jda.core.audio;

import org.json.JSONArray;

public enum AudioEncryption
{
// these are ordered by priority, lite > suffix > normal
// we prefer lite because it uses only 4 bytes for its nonce while the others use 24 bytes
XSALSA20_POLY1305_LITE,
XSALSA20_POLY1305_SUFFIX,
XSALSA20_POLY1305;

private final String key;

AudioEncryption()
{
this.key = name().toLowerCase();
}

public String getKey()
{
return key;
}

public static AudioEncryption getPreferredMode(JSONArray array)
{
AudioEncryption encryption = null;
for (Object o : array)
{
try
{
String name = ((String) o).toUpperCase();
AudioEncryption e = valueOf(name);
if (encryption == null || e.ordinal() < encryption.ordinal())
encryption = e;
}
catch (IllegalArgumentException ignored) {}
}
return encryption;
}
}

0 comments on commit 16437e1

Please sign in to comment.