From 74865d03602e55eac674567adde5186d110fabb5 Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 31 Dec 2025 02:59:28 +0100 Subject: [PATCH 01/28] Implement DAVE/MLS on the gateway --- .gitignore | 2 + MIGRATION.md | 6 + core/build.gradle.kts | 2 + .../moe/kyokobot/koe/KoeEventListener.java | 1 + .../java/moe/kyokobot/koe/KoeOptions.java | 11 +- .../moe/kyokobot/koe/KoeOptionsBuilder.java | 13 +- .../moe/kyokobot/koe/VoiceServerInfo.java | 31 +- .../AbstractMediaGatewayConnection.java | 29 +- .../koe/gateway/MediaGatewayConnection.java | 15 + .../koe/gateway/MediaGatewayV4Connection.java | 6 + .../koe/gateway/MediaGatewayV5Connection.java | 6 + .../koe/gateway/MediaGatewayV8Connection.java | 165 ++++++++++ .../java/moe/kyokobot/koe/gateway/Op.java | 2 + .../kyokobot/koe/internal/DAVEManager.java | 285 ++++++++++++++++++ .../kyokobot/koe/internal/KoeClientImpl.java | 32 ++ .../koe/internal/MediaConnectionImpl.java | 29 ++ .../handler/DiscordUDPConnection.java | 15 +- settings.gradle | 6 - settings.gradle.kts | 13 + testbot/build.gradle.kts | 1 + .../moe/kyokobot/koe/testbot/TestBot.java | 28 +- 21 files changed, 665 insertions(+), 33 deletions(-) create mode 100644 MIGRATION.md create mode 100644 core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java delete mode 100644 settings.gradle create mode 100644 settings.gradle.kts diff --git a/.gitignore b/.gitignore index 96d4ddd..f66c79e 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ fabric.properties # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser +# JVM crashes +hs_err_pid*.log diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 0000000..f6ee69f --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,6 @@ +# Migrating from Koe 2.x to 3.0 + +## VoiceServerInfo + +1. The public constructor has been removed. Use `VoiceServerInfo#builder()` to create instances of this class instead. +2. It's now required to pass `channelId` (the ID of the voice channel), as it is used as a MLS group identifier while using DAVE E2E encryption. diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 776c7d5..725ac68 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -4,5 +4,7 @@ dependencies { implementation("io.netty:netty-transport-native-epoll:4.1.112.Final:linux-x86_64") implementation("org.slf4j:slf4j-api:1.8.0-beta4") implementation("com.google.crypto.tink:tink:1.14.1") + implementation("moe.kyokobot.libdave:api:1.0-SNAPSHOT") + implementation("moe.kyokobot.libdave:impl-jni:1.0-SNAPSHOT") compileOnly("org.jetbrains:annotations:13.0") } diff --git a/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java b/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java index 7c1c112..d877a2a 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java @@ -12,6 +12,7 @@ public interface KoeEventListener { void gatewayClosed(int code, @Nullable String reason, boolean byRemote); + // TODO: This should actually be called something like userStreamsChanged void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC); void userDisconnected(String id); diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java index cb4a712..05a2281 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java @@ -25,6 +25,7 @@ public class KoeOptions { private final boolean highPacketPriority; private final boolean deafened; private final boolean enableWSSPortOverride; + private final boolean enableDAVE; KoeOptions( @NotNull EventLoopGroup eventLoopGroup, @@ -35,7 +36,8 @@ public class KoeOptions { @NotNull FramePollerFactory framePollerFactory, boolean highPacketPriority, boolean deafened, - boolean enableWSSPortOverride + boolean enableWSSPortOverride, + boolean daveEnabled ) { this.eventLoopGroup = Objects.requireNonNull(eventLoopGroup); this.socketChannelClass = Objects.requireNonNull(socketChannelClass); @@ -46,6 +48,7 @@ public class KoeOptions { this.highPacketPriority = highPacketPriority; this.deafened = deafened; this.enableWSSPortOverride = enableWSSPortOverride; + this.enableDAVE = daveEnabled; } /** @@ -63,7 +66,7 @@ public KoeOptions( boolean deafened ) { this(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, gatewayVersion, - framePollerFactory, highPacketPriority, deafened, true); + framePollerFactory, highPacketPriority, deafened, true, true); } /** @@ -125,6 +128,10 @@ public boolean isEnableWSSPortOverride() { return enableWSSPortOverride; } + public boolean isEnableDAVE() { + return enableDAVE; + } + /** * @return An instance of {@link KoeOptions} with default options. */ diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java index 71ad2d0..e423776 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java @@ -29,6 +29,7 @@ public class KoeOptionsBuilder { private boolean highPacketPriority; private boolean deafened; private boolean enableWSSPortOverride; + private boolean enableDAVE; KoeOptionsBuilder() { boolean epoll = Epoll.isAvailable(); @@ -49,6 +50,7 @@ public class KoeOptionsBuilder { this.highPacketPriority = true; this.deafened = false; this.enableWSSPortOverride = true; + this.enableDAVE = true; } /** @@ -155,8 +157,17 @@ public KoeOptionsBuilder setEnableWSSPortOverride(boolean enableWSSPortOverride) return this; } + /** + * Sets whether End-to-End encryption using Discord's DAVE protocol is enabled. + * Defaults to true. + */ + public KoeOptionsBuilder setDAVEEnabled(boolean enabled) { + this.enableDAVE = enabled; + return this; + } + public KoeOptions create() { return new KoeOptions(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, - gatewayVersion, framePollerFactory, highPacketPriority, deafened, enableWSSPortOverride); + gatewayVersion, framePollerFactory, highPacketPriority, deafened, enableWSSPortOverride, enableDAVE); } } diff --git a/core/src/main/java/moe/kyokobot/koe/VoiceServerInfo.java b/core/src/main/java/moe/kyokobot/koe/VoiceServerInfo.java index c719127..34703a7 100644 --- a/core/src/main/java/moe/kyokobot/koe/VoiceServerInfo.java +++ b/core/src/main/java/moe/kyokobot/koe/VoiceServerInfo.java @@ -15,20 +15,22 @@ public class VoiceServerInfo { private final String sessionId; private final String endpoint; private final String token; + private final long channelId; /** - * @see #builder() Recommended way to create an instance of this class. - * * @param sessionId Session ID from VOICE_STATE_UPDATE payload. - * @param endpoint Voice server endpoint from VOICE_SERVER_UPDATE event (passed as-is in form of "hostname" or "hostname:port"). - * @param token The authentication token from VOICE_SERVER_UPDATE payload. + * @param endpoint Voice server endpoint from VOICE_SERVER_UPDATE event (passed as-is in form of "hostname" or "hostname:port"). + * @param token The authentication token from VOICE_SERVER_UPDATE payload. + * @see #builder() Recommended way to create an instance of this class. */ - public VoiceServerInfo(@NotNull String sessionId, - @NotNull String endpoint, - @NotNull String token) { + private VoiceServerInfo(@NotNull String sessionId, + @NotNull String endpoint, + @NotNull String token, + long channelId) { this.sessionId = Objects.requireNonNull(sessionId); this.endpoint = Objects.requireNonNull(endpoint); this.token = Objects.requireNonNull(token); + this.channelId = channelId; } @NotNull @@ -46,7 +48,9 @@ public String getToken() { return token; } - // TODO: Should we deprecate Builder or the public constructor? + public long getChannelId() { + return channelId; + } @NotNull public static Builder builder() { @@ -57,6 +61,7 @@ public static class Builder { private String sessionId; private String endpoint; private String token; + private long channelId; /** * @param sessionId Session ID from VOICE_STATE_UPDATE payload. @@ -83,8 +88,16 @@ public Builder setToken(String token) { return this; } + /** + * @param channelId An ID of the voice channel. Required for establishing DAVE MLS groups. + */ + public Builder setChannelId(long channelId) { + this.channelId = channelId; + return this; + } + public VoiceServerInfo build() { - return new VoiceServerInfo(sessionId, endpoint, token); + return new VoiceServerInfo(sessionId, endpoint, token, channelId); } } } diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java b/core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java index 328748b..040272f 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java @@ -1,6 +1,8 @@ package moe.kyokobot.koe.gateway; import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; @@ -42,6 +44,7 @@ public abstract class AbstractMediaGatewayConnection implements MediaGatewayConn protected final URI websocketURI; protected final Bootstrap bootstrap; protected final SslContext sslContext; + protected final ByteBufAllocator allocator; protected CompletableFuture connectFuture; protected EventExecutor eventExecutor; @@ -67,6 +70,7 @@ public AbstractMediaGatewayConnection(@NotNull MediaConnectionImpl connection, this.bootstrap = NettyBootstrapFactory.socket(connection.getOptions()) .handler(new WebSocketInitializer()); this.sslContext = SslContextBuilder.forClient().build(); + this.allocator = connection.getOptions().getByteBufAllocator(); this.connectFuture = new CompletableFuture<>(); } catch (SSLException | URISyntaxException e) { throw new IllegalStateException(e); @@ -127,6 +131,8 @@ public boolean isOpen() { protected abstract void handlePayload(JsonObject object); + protected abstract void handlePayload(ByteBuf byteBuf); + protected void onClose(int code, @Nullable String reason, boolean remote) { if (!closed) { closed = true; @@ -167,6 +173,13 @@ public void sendInternalPayload(int op, Object d) { sendRaw(new JsonObject().add("op", op).add("d", d)); } + public void sendInternalBinPayload(int op, byte[] d) { + var buf = this.allocator.buffer(1 + d.length); + buf.writeByte(op); + buf.writeBytes(d); + this.sendRawBin(buf); + } + protected void sendRaw(JsonObject object) { if (channel != null && channel.isOpen()) { var data = object.toString(); @@ -175,6 +188,13 @@ protected void sendRaw(JsonObject object) { } } + protected void sendRawBin(ByteBuf buffer) { + if (channel != null && channel.isOpen()) { + logger.trace("<- ", buffer.readableBytes()); + channel.writeAndFlush(new BinaryWebSocketFrame(buffer)); + } + } + private class WebSocketClientHandler extends SimpleChannelInboundHandler { private final WebSocketClientHandshaker handshaker; @@ -232,11 +252,13 @@ protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Except logger.trace("-> {}", object); frame.release(); handlePayload(object); + } else if (msg instanceof BinaryWebSocketFrame) { + var frame = (BinaryWebSocketFrame) msg; + logger.trace("-> ", frame.content().readableBytes()); + handlePayload(frame.content()); } else if (msg instanceof CloseWebSocketFrame) { var frame = (CloseWebSocketFrame) msg; - if (logger.isDebugEnabled()) { - logger.debug("Websocket closed, code: {}, reason: {}", frame.statusCode(), frame.reasonText()); - } + logger.debug("Websocket closed, code: {}, reason: {}", frame.statusCode(), frame.reasonText()); AbstractMediaGatewayConnection.this.open = false; onClose(frame.statusCode(), frame.reasonText(), true); } @@ -249,6 +271,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { } connection.getDispatcher().gatewayError(cause); + logger.warn("Exception occurred in WebSocket client handler (Guild ID={})", connection.getGuildId(), cause); close(4000, "Internal error"); ctx.close(); diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnection.java index d921a0c..9897d98 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnection.java @@ -22,4 +22,19 @@ public interface MediaGatewayConnection { void reconnect(); void updateSpeaking(int mask); + + /** + * Send a marshalled MLS key package + */ + default void sendMLSKeyPackage(byte[] keyPackage) { + } + + default void sendMLSCommitWelcome(byte[] commitWelcome) { + } + + default void sendMLSInvalidCommitWelcome(int transitionId) { + } + + default void sendSecureFramesReadyForTransition(int transitionId) { + } } diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java index 23ce0f4..f057b13 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java @@ -1,5 +1,6 @@ package moe.kyokobot.koe.gateway; +import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; import moe.kyokobot.koe.codec.OpusCodec; import moe.kyokobot.koe.crypto.EncryptionMode; @@ -136,6 +137,11 @@ protected void handlePayload(JsonObject object) { } } + @Override + protected void handlePayload(ByteBuf byteBuf) { + // no binary messages on this protocol version + } + @Override protected void onClose(int code, @Nullable String reason, boolean remote) { super.onClose(code, reason, remote); diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java index 6cf7474..0df172e 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java @@ -1,5 +1,6 @@ package moe.kyokobot.koe.gateway; +import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; import moe.kyokobot.koe.codec.Codec; import moe.kyokobot.koe.codec.DefaultCodecs; @@ -157,6 +158,11 @@ protected void handlePayload(JsonObject object) { } } + @Override + protected void handlePayload(ByteBuf byteBuf) { + // no binary messages on this protocol version + } + @Override protected void onClose(int code, @Nullable String reason, boolean remote) { super.onClose(code, reason, remote); diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java index e876077..1424121 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java @@ -1,5 +1,6 @@ package moe.kyokobot.koe.gateway; +import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; import moe.kyokobot.koe.codec.Codec; import moe.kyokobot.koe.codec.DefaultCodecs; @@ -48,9 +49,19 @@ public MediaValve getValve() { @Override protected void identify() { logger.debug("Identifying..."); + + int maxDAVEVersion = 0; + var manager = connection.getDAVEManager(); + if (manager != null) { + maxDAVEVersion = manager.getMaxDAVEProtocolVersion(); + } + + logger.debug("Max DAVE Protocol Version: {}", maxDAVEVersion); + sendInternalPayload(Op.IDENTIFY, new JsonObject() .addAsString("server_id", connection.getGuildId()) .addAsString("user_id", connection.getClient().getClientId()) + .add("max_dave_protocol_version", maxDAVEVersion) .add("session_id", voiceServerInfo.getSessionId()) .add("token", voiceServerInfo.getToken()) .add("video", true)); @@ -95,6 +106,7 @@ protected void handlePayload(JsonObject object) { .stream() .map(o -> (String) o) .collect(Collectors.toList()); + address = new InetSocketAddress(ip, port); connection.getDispatcher().gatewayReady((InetSocketAddress) address, ssrc); @@ -116,6 +128,10 @@ protected void handlePayload(JsonObject object) { connection.getDispatcher().sessionDescription(data); connection.getConnectionHandler().handleSessionDescription(data); + var daveManager = connection.getDAVEManager(); + if (daveManager != null) { + daveManager.handleSessionDescription(data, voiceServerInfo.getChannelId()); + } break; } case Op.HEARTBEAT_ACK: { @@ -139,12 +155,29 @@ protected void handlePayload(JsonObject object) { connection.getDispatcher().userConnected(user, audioSsrc, videoSsrc, rtxSsrc); break; } + case Op.CLIENT_CONNECT: { + var data = object.getObject("d"); + var userIds = data.getArray("user_ids"); + + var manager = connection.getDAVEManager(); + if (manager != null) { + userIds.forEach(userId -> manager.addUser((String) userId)); + } + + break; + } case Op.CLIENT_DISCONNECT: { mediaValve.handleEvent(object); var data = object.getObject("d"); var user = data.getString("user_id"); connection.getDispatcher().userDisconnected(user); + + var manager = connection.getDAVEManager(); + if (manager != null) { + manager.removeUser(user); + } + break; } case Op.MEDIA_SINK_WANTS: { @@ -158,7 +191,117 @@ protected void handlePayload(JsonObject object) { break; } + case Op.VOICE_BACKEND_VERSION: { + var data = object.getObject("d"); + logger.debug("Voice backend version: {}", data); + break; + } + case Op.SECURE_FRAMES_PREPARE_PROTOCOL_TRANSITION: { + var data = object.getObject("d"); + logger.debug("Secure frames prepare protocol transition: {}", data); + var transitionId = data.getInt("transition_id"); + var protocolVersion = data.getInt("protocol_version"); + + var manager = connection.getDAVEManager(); + if (manager != null) { + manager.handleSecureFramesPrepareProtocolTransition(transitionId, protocolVersion); + } + + break; + } + case Op.SECURE_FRAMES_EXECUTE_TRANSITION: { + var data = object.getObject("d"); + logger.debug("Secure frames execute transition: {}", data); + var transitionId = data.getInt("transition_id"); + + var manager = connection.getDAVEManager(); + if (manager != null) { + manager.handleSecureFramesExecuteTransition(transitionId); + } + + break; + } + case Op.SECURE_FRAMES_READY_FOR_TRANSITION: { + var data = object.getObject("d"); + logger.debug("Secure frames ready for transition: {}", data); + + break; + } + case Op.SECURE_FRAMES_PREPARE_EPOCH: { + var data = object.getObject("d"); + logger.debug("Secure frames prepare epoch: {}", data); + var epoch = data.getInt("epoch"); + var protocolVersion = data.getInt("protocol_version"); + + var manager = connection.getDAVEManager(); + if (manager != null) { + manager.handleSecureFramesPrepareEpoch(Integer.toString(epoch), protocolVersion); + } + + break; + } + default: + break; + } + } + + @Override + protected void handlePayload(ByteBuf byteBuf) { + var seq = byteBuf.readShort(); + var op = byteBuf.readByte(); + + sequence = seq; + var manager = connection.getDAVEManager(); + if (manager == null) { + return; + } + + switch (op) { + case Op.MLS_WELCOME: { + var transId = byteBuf.readUnsignedShort(); + var payload = new byte[byteBuf.readableBytes()]; + logger.debug("MLS welcome, transId: {} payload: <{} bytes>", transId, payload.length); + byteBuf.readBytes(payload); + manager.handleMLSWelcome(transId, payload); + + break; + } + case Op.MLS_EXTERNAL_SENDER_PACKAGE: { + logger.debug("MLS external sender package"); + + var payload = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(payload); + manager.handleMLSExternalSender(payload); + + break; + } +// case Op.MLS_KEY_PACKAGE: { +// logger.debug("MLS key package"); +// break; +// } + case Op.MLS_PROPOSALS: { + logger.debug("MLS proposals"); + + var payload = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(payload); + manager.handleMLSProposals(payload); + break; + } +// case Op.MLS_COMMIT_WELCOME: { +// logger.debug("MLS commit welcome"); +// break; +// } + case Op.MLS_PREPARE_COMMIT_TRANSITION: { + var transId = byteBuf.readUnsignedShort(); + var payload = new byte[byteBuf.readableBytes()]; + logger.debug("MLS prepare commit transition, transId: {} payload: <{} bytes>", transId, payload.length); + + byteBuf.readBytes(payload); + manager.handleMLSPrepareCommitTransition(transId, payload); + break; + } default: + logger.debug("Received unknown binary payload OP: {}", op); break; } } @@ -184,6 +327,28 @@ public void updateSpeaking(int mask) { .add("ssrc", ssrc)); } + @Override + public void sendMLSKeyPackage(byte[] keyPackage) { + sendInternalBinPayload(Op.MLS_KEY_PACKAGE, keyPackage); + } + + @Override + public void sendMLSCommitWelcome(byte[] commitWelcome) { + sendInternalBinPayload(Op.MLS_COMMIT_WELCOME, commitWelcome); + } + + @Override + public void sendMLSInvalidCommitWelcome(int transitionId) { + sendInternalPayload(Op.MLS_INVALID_COMMIT_WELCOME, new JsonObject() + .add("transition_id", transitionId)); + } + + @Override + public void sendSecureFramesReadyForTransition(int transitionId) { + sendInternalPayload(Op.SECURE_FRAMES_READY_FOR_TRANSITION, new JsonObject() + .add("transition_id", transitionId)); + } + private void setupHeartbeats(int interval) { if (eventExecutor != null) { heartbeatFuture = eventExecutor.scheduleAtFixedRate(this::heartbeat, interval, interval, diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/Op.java b/core/src/main/java/moe/kyokobot/koe/gateway/Op.java index 80eeb99..8d1ff0e 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/Op.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/Op.java @@ -41,4 +41,6 @@ private Op() { public static final int MLS_COMMIT_WELCOME = 28; public static final int MLS_PREPARE_COMMIT_TRANSITION = 29; public static final int MLS_WELCOME = 30; + public static final int MLS_INVALID_COMMIT_WELCOME = 31; + public static final int NO_ROUTE = 32; } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java new file mode 100644 index 0000000..87b59a2 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java @@ -0,0 +1,285 @@ +package moe.kyokobot.koe.internal; + +import io.netty.buffer.ByteBuf; +import moe.kyokobot.koe.codec.OpusCodec; +import moe.kyokobot.koe.internal.json.JsonObject; +import moe.kyokobot.libdave.KeyRatchet; +import moe.kyokobot.libdave.MediaType; +import moe.kyokobot.libdave.Session; +import moe.kyokobot.libdave.netty.NettyDaveFactory; +import moe.kyokobot.libdave.netty.NettyEncryptor; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class DAVEManager implements AutoCloseable { + private static final Logger logger = LoggerFactory.getLogger(DAVEManager.class); + + private static final String MLS_NEW_GROUP_EPOCH = "1"; + private static final int INIT_TRANSITION_ID = 0; + + private final MediaConnectionImpl connection; + private final NettyDaveFactory factory; + private final Session daveSession; + private final Set recognizedUserIds = new HashSet<>(); + private final Map pendingTransitions = new HashMap<>(); + private final int maxProtocolVersion; + + private NettyEncryptor selfEncryptor; + private @Nullable KeyRatchet selfKeyRatchet = null; + private String selfUserIdString; + private long mlsGroupId = 0; + + private int currentProtocolVersion = 0; + + DAVEManager(@NotNull MediaConnectionImpl connection, @NotNull NettyDaveFactory factory) { + this.connection = connection; + this.factory = factory; + this.daveSession = factory.createSession("", "", this::mlsFailureCallback); + this.selfEncryptor = factory.fromEncryptor(factory.createEncryptor()); + this.maxProtocolVersion = factory.maxSupportedProtocolVersion(); + this.selfUserIdString = String.valueOf(connection.getClient().getClientId()); + } + + public int getMaxDAVEProtocolVersion() { + return maxProtocolVersion; + } + + public void addUser(String userId) { + recognizedUserIds.add(userId); + setupKeyRatchetForUser(userId, currentProtocolVersion); + } + + public void removeUser(String userId) { + recognizedUserIds.remove(userId); + } + + public ByteBuf encrypt(MediaType mediaType, int ssrc, ByteBuf output, ByteBuf input, int size) { + if (mediaType == MediaType.AUDIO && size == 3) { + input.markReaderIndex(); + var b1 = input.readByte() == OpusCodec.SILENCE_FRAME[0]; + var b2 = input.readByte() == OpusCodec.SILENCE_FRAME[1]; + var b3 = input.readByte() == OpusCodec.SILENCE_FRAME[2]; + var isSilence = b1 && b2 && b3; + input.resetReaderIndex(); + + if (isSilence) { + return input; + } + } + + output.ensureWritable(this.selfEncryptor.getMaxCiphertextByteSize(mediaType, size)); + var _result = this.selfEncryptor.encrypt(mediaType, ssrc, input, output); + return output; + } + + public void handleSessionDescription(@NotNull JsonObject session, long mlsGroupId) { + int protocolVersion = session.getInt("dave_protocol_version", 0); + this.mlsGroupId = mlsGroupId; + daveProtocolInit(protocolVersion); + } + + public void handleSecureFramesPrepareProtocolTransition(int transitionId, int newProtocolVersion) { + prepareRatchets(transitionId, newProtocolVersion); + if (transitionId != 0) { + sendSecureFramesReadyForTransition(transitionId); + } + } + + public void handleSecureFramesExecuteTransition(int transitionId) { + executeTransition(transitionId); + } + + public void handleSecureFramesPrepareEpoch(String epoch, int protocolVersion) { + prepareEpoch(epoch, protocolVersion); + + if (MLS_NEW_GROUP_EPOCH.equals(epoch)) { + sendMLSKeyPackage(); + } + } + + public void handleMLSExternalSender(byte[] payload) { + daveSession.setExternalSender(payload); + } + + public void handleMLSProposals(byte[] payload) { + var userIds = recognizedUserIdArray(); + + var commitWelcome = daveSession.processProposals(payload, userIds); + if (commitWelcome != null) { + sendMLSCommitWelcome(commitWelcome); + } + } + + public void handleMLSPrepareCommitTransition(int transitionId, byte[] commit) { + var result = daveSession.processCommit(commit); + if (result.isIgnored()) return; + + + if (result.isFailed()) { + sendMLSInvalidCommitWelcome(transitionId); + daveProtocolInit(daveSession.getProtocolVersion()); + return; + } + + prepareRatchets(transitionId, daveSession.getProtocolVersion()); + if (transitionId != 0) { + sendSecureFramesReadyForTransition(transitionId); + } + } + + public void handleMLSWelcome(int transitionId, byte[] welcome) { + var roster = daveSession.processWelcome(welcome, recognizedUserIdArray()); + + if (roster == null) { + sendMLSInvalidCommitWelcome(transitionId); + sendMLSKeyPackage(); + } + + prepareRatchets(transitionId, daveSession.getProtocolVersion()); + if (transitionId != 0) { + sendSecureFramesReadyForTransition(transitionId); + } + } + + private void daveProtocolInit(int protocolVersion) { + logger.debug("DAVE Init - Protocol version={}, MLS Group ID={}", protocolVersion, mlsGroupId); + if (protocolVersion > 0) { + prepareEpoch(MLS_NEW_GROUP_EPOCH, protocolVersion); + sendMLSKeyPackage(); + } else { + prepareRatchets(INIT_TRANSITION_ID, protocolVersion); + executeTransition(INIT_TRANSITION_ID); + } + } + + + private void prepareEpoch(String epoch, int protocolVersion) { + if (MLS_NEW_GROUP_EPOCH.equals(epoch)) { + daveSession.init(protocolVersion, mlsGroupId, selfUserIdString); + } + } + + private void setupKeyRatchetForUser(String uid, int protocolVersion) { + var keyRatchet = makeKeyRatchetForUser(uid, protocolVersion); + setSelfKeyRatchet(keyRatchet); + } + + @Nullable + private KeyRatchet makeKeyRatchetForUser(String uid, int protocolVersion) { + if (protocolVersion == 0) { + return null; + } + + return daveSession.getKeyRatchet(uid); + } + + private void prepareRatchets(int transitionId, int protocolVersion) { + for (var uid : recognizedUserIds) { + if (selfUserIdString.equals(uid)) { + continue; + } + + setupKeyRatchetForUser(uid, protocolVersion); + } + + if (transitionId == INIT_TRANSITION_ID) { + setupKeyRatchetForUser(selfUserIdString, protocolVersion); + } else { + pendingTransitions.put(transitionId, protocolVersion); + } + + currentProtocolVersion = protocolVersion; + } + + private void executeTransition(int transitionId) { + var protocolVersion = pendingTransitions.remove(transitionId); + if (protocolVersion == null) { + return; + } + + if (protocolVersion == 0) { + daveSession.reset(); + } + + setupKeyRatchetForUser(selfUserIdString, protocolVersion); + logger.debug("Transition executed: ID={}, Protocol version={}", transitionId, protocolVersion); + } + + private void sendMLSKeyPackage() { + var gateway = connection.getGatewayConnection(); + if (gateway != null) { + var keyPackage = daveSession.getMarshalledKeyPackage(); + gateway.sendMLSKeyPackage(keyPackage); + } + } + + private void sendMLSCommitWelcome(byte[] commitWelcome) { + var gateway = connection.getGatewayConnection(); + if (gateway != null) { + gateway.sendMLSCommitWelcome(commitWelcome); + } + } + + private void sendMLSInvalidCommitWelcome(int transitionId) { + var gateway = connection.getGatewayConnection(); + if (gateway != null) { + gateway.sendMLSInvalidCommitWelcome(transitionId); + } + } + + private void sendSecureFramesReadyForTransition(int transitionId) { + var gateway = connection.getGatewayConnection(); + if (gateway != null) { + gateway.sendSecureFramesReadyForTransition(transitionId); + } + } + + private void mlsFailureCallback(String source, String reason) { + logger.warn("MLS Failure - Source: {}, Reason: {}", source, reason); + } + + @Override + public void close() throws Exception { + daveSession.close(); + selfEncryptor.close(); + } + + private String[] recognizedUserIdArray() { + var userIds = new String[recognizedUserIds.size() + 1]; + int i = 0; + for (var uid : recognizedUserIds) { + userIds[i++] = uid; + } + userIds[i] = selfUserIdString; + + return userIds; + } + + public void setSelfKeyRatchet(KeyRatchet selfKeyRatchet) { + if (this.selfKeyRatchet != null) { + this.selfKeyRatchet.close(); + } + + this.selfKeyRatchet = selfKeyRatchet; + + if (this.selfKeyRatchet != null) { + this.reinitSelfEncryptor(); + this.selfEncryptor.setKeyRatchet(selfKeyRatchet); + } + } + + private void reinitSelfEncryptor() { + if (this.selfEncryptor != null) { + this.selfEncryptor.close(); + } + + this.selfEncryptor = factory.fromEncryptor(factory.createEncryptor()); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java index c5a444b..26a294a 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java @@ -4,8 +4,13 @@ import moe.kyokobot.koe.KoeOptions; import moe.kyokobot.koe.MediaConnection; import moe.kyokobot.koe.gateway.GatewayVersion; +import moe.kyokobot.libdave.NativeDaveFactory; +import moe.kyokobot.libdave.netty.NativeNettyDaveFactory; +import moe.kyokobot.libdave.netty.NettyDaveFactory; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; @@ -13,15 +18,25 @@ import java.util.concurrent.ConcurrentHashMap; public class KoeClientImpl implements KoeClient { + private static final Logger logger = LoggerFactory.getLogger(KoeClientImpl.class); + private final long clientId; private final KoeOptions options; private final Map connections; + private final @Nullable NettyDaveFactory daveFactory; public KoeClientImpl(long clientId, KoeOptions options) { this.clientId = clientId; this.options = options; this.connections = new ConcurrentHashMap<>(); + + NettyDaveFactory daveFactory = null; + if (options.isEnableDAVE()) { + daveFactory = createDAVEFactory(); + } + + this.daveFactory = daveFactory; } @Override @@ -67,6 +82,11 @@ public MediaConnection newVoiceConnection(long id) { return new MediaConnectionImpl(this, id); } + @Nullable + public NettyDaveFactory getDaveFactory() { + return daveFactory; + } + @Override public long getClientId() { return clientId; @@ -83,4 +103,16 @@ public KoeOptions getOptions() { public GatewayVersion getGatewayVersion() { return options.getGatewayVersion(); } + + private static @Nullable NettyDaveFactory createDAVEFactory() { + // TODO: We have a pure Java implementation planned. + try { + NativeDaveFactory.ensureAvailable(); + return new NativeNettyDaveFactory(); + } catch (RuntimeException e) { + logger.warn("DAVE requested but the native library could not be loaded! Did you forget to include 'moe.kyokobot.libdave:natives-{platform}' dependency in your project?", e); + } + + return null; + } } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java index c5dfd51..e7bc538 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java @@ -33,6 +33,7 @@ public class MediaConnectionImpl implements MediaConnection { private FramePoller videoPoller; private MediaFrameProvider audioSender; private MediaFrameProvider videoSender; + private DAVEManager daveManager; public MediaConnectionImpl(@NotNull KoeClientImpl client, long guildId) { this.client = Objects.requireNonNull(client); @@ -47,6 +48,8 @@ public MediaConnectionImpl(@NotNull KoeClientImpl client, long guildId) { @Override public CompletionStage connect(VoiceServerInfo info) { this.disconnect(); + this.createDAVEManager(); + var conn = client.getGatewayVersion().createConnection(this, info); return conn.start().thenAccept(nothing -> { @@ -75,6 +78,8 @@ public void disconnect() { connectionHandler.close(); connectionHandler = null; } + + this.destroyDAVEManager(); } @Override @@ -265,4 +270,28 @@ public EventDispatcher getDispatcher() { public void setConnectionHandler(ConnectionHandler connectionHandler) { this.connectionHandler = connectionHandler; } + + public DAVEManager getDAVEManager() { + return daveManager; + } + + public void createDAVEManager() { + this.destroyDAVEManager(); + + var daveFactory = client.getDaveFactory(); + if (daveFactory != null) { + daveManager = new DAVEManager(this, daveFactory); + } + } + + public void destroyDAVEManager() { + if (this.daveManager != null) { + try { + this.daveManager.close(); + } catch (Exception e) { + logger.error("Error closing old DAVE manager", e); + + } + } + } } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java b/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java index 7845db9..23bc9fa 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java @@ -5,13 +5,13 @@ import io.netty.buffer.ByteBufAllocator; import io.netty.channel.ChannelInitializer; import io.netty.channel.socket.DatagramChannel; -import moe.kyokobot.koe.MediaConnection; import moe.kyokobot.koe.codec.Codec; import moe.kyokobot.koe.crypto.EncryptionMode; -import moe.kyokobot.koe.internal.util.RTPHeaderWriter; import moe.kyokobot.koe.handler.ConnectionHandler; +import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.NettyBootstrapFactory; import moe.kyokobot.koe.internal.json.JsonObject; +import moe.kyokobot.koe.internal.util.RTPHeaderWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +26,7 @@ public class DiscordUDPConnection implements Closeable, ConnectionHandler { private static final Logger logger = LoggerFactory.getLogger(DiscordUDPConnection.class); - private final MediaConnection connection; + private final MediaConnectionImpl connection; private final ByteBufAllocator allocator; private final SocketAddress serverAddress; private final Bootstrap bootstrap; @@ -38,7 +38,7 @@ public class DiscordUDPConnection implements Closeable, ConnectionHandler playerMap = new ConcurrentHashMap<>(); + private Map vsuChannelMap = new ConcurrentHashMap<>(); public TestBot(String token) { this.token = token; @@ -104,18 +105,31 @@ public void onReady(ReadyEvent event) { public void onVoiceServerUpdate(VoiceServerUpdate voiceServerUpdate) { var conn = koeClient.getConnection(voiceServerUpdate.getGuildIdLong()); if (conn != null) { - var info = new VoiceServerInfo( - voiceServerUpdate.getSessionId(), - voiceServerUpdate.getEndpoint(), - voiceServerUpdate.getToken()); - conn.connect(info).thenAccept(avoid -> logger.info("Koe connection succeeded!")); + var info = VoiceServerInfo.builder() + .setSessionId(voiceServerUpdate.getSessionId()) + .setEndpoint(voiceServerUpdate.getEndpoint()) + .setToken(voiceServerUpdate.getToken()) + .setChannelId(vsuChannelMap.getOrDefault(voiceServerUpdate.getGuildIdLong(), 0L)) + .build(); + conn.connect(info).thenAccept(avoid -> { + logger.info("Koe connection succeeded!"); + this.leakDetect.printAllocStats(); + }); } } @Override public boolean onVoiceStateUpdate(VoiceStateUpdate voiceStateUpdate) { - if (voiceStateUpdate.getVoiceState().getIdLong() == jda.getSelfUser().getIdLong() && voiceStateUpdate.getChannel().getIdLong() == 0) { - koeClient.destroyConnection(voiceStateUpdate.getGuildIdLong()); + if (voiceStateUpdate.getVoiceState().getIdLong() == jda.getSelfUser().getIdLong()) { + logger.info("VSU {} {}", voiceStateUpdate.getGuild(), voiceStateUpdate.getChannel()); + + if (voiceStateUpdate.getChannel() == null) { + koeClient.destroyConnection(voiceStateUpdate.getGuildIdLong()); + vsuChannelMap.remove(voiceStateUpdate.getGuildIdLong()); + return true; + } else { + vsuChannelMap.put(voiceStateUpdate.getGuildIdLong(), voiceStateUpdate.getChannel().getIdLong()); + } } return true; } From 44c11e6ca08031b233ba1e9dca1d4a34fd2f121e Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 31 Dec 2025 02:59:47 +0100 Subject: [PATCH 02/28] Version from 3.0 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 496fcc5..c01ae38 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -37,7 +37,7 @@ subprojects { } else { // Get version from git val gitVersion = getGitVersion() - version = gitVersion.version + version = "3.0+git" + gitVersion.version } configure { From 3a065ff2629783074fefebd364fa8d8d8c08bd2d Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 31 Dec 2025 02:59:58 +0100 Subject: [PATCH 03/28] Add a netty leak detector to the test bot --- .../kyokobot/koe/testbot/NettyLeakDetect.java | 45 +++++++++++++++++++ .../moe/kyokobot/koe/testbot/TestBot.java | 8 +++- 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 testbot/src/main/java/moe/kyokobot/koe/testbot/NettyLeakDetect.java diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/NettyLeakDetect.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/NettyLeakDetect.java new file mode 100644 index 0000000..0cc6984 --- /dev/null +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/NettyLeakDetect.java @@ -0,0 +1,45 @@ +package moe.kyokobot.koe.testbot; + +import io.netty.buffer.PooledByteBufAllocator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NettyLeakDetect { + private static final Logger logger = LoggerFactory.getLogger(NettyLeakDetect.class); + private final PooledByteBufAllocator allocator; + + public NettyLeakDetect() { + allocator = new PooledByteBufAllocator(true, 1, 1, 8192, 11, 0, 0, false); + } + + public PooledByteBufAllocator getAllocator() { + return allocator; + } + + public void printAllocStats() { + long directActive = 0; + long directAlloc = 0; + long directDealloc = 0; + long heapActive = 0; + long heapAlloc = 0; + long heapDealloc = 0; + for (var arena : allocator.metric().directArenas()) { + directActive += arena.numActiveAllocations(); + directAlloc += arena.numAllocations(); + directDealloc += arena.numDeallocations(); + } + + for (var arena : allocator.metric().heapArenas()) { + heapActive += arena.numActiveAllocations(); + heapAlloc += arena.numAllocations(); + heapDealloc += arena.numDeallocations(); + } + logger.info("{}", allocator.dumpStats()); + logger.info("Direct Active: {}", directActive); + logger.info("Direct Alloc: {}", directAlloc); + logger.info("Direct Dealloc: {}", directDealloc); + logger.info("Heap Active: {}", heapActive); + logger.info("Heap Alloc: {}", heapAlloc); + logger.info("Heap Dealloc: {}", heapDealloc); + } +} diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java index 3fd0ada..1628f81 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java @@ -46,6 +46,7 @@ public class TestBot extends ListenerAdapter implements VoiceDispatchInterceptor private static final Logger logger = LoggerFactory.getLogger(TestBot.class); private final String token; + private final NettyLeakDetect leakDetect; private JDA jda; private Koe koe; private KoeClient koeClient; @@ -54,6 +55,7 @@ public class TestBot extends ListenerAdapter implements VoiceDispatchInterceptor private Map vsuChannelMap = new ConcurrentHashMap<>(); public TestBot(String token) { + this.leakDetect = new NettyLeakDetect(); this.token = token; } @@ -61,13 +63,13 @@ public void start() { this.jda = createJDA(); this.koe = createKoe(); this.playerManager = createAudioPlayerManager(); - } public void stop() { try { logger.info("Shutting down..."); koeClient.close(); + this.leakDetect.printAllocStats(); Thread.sleep(250); jda.shutdownNow(); Thread.sleep(500); @@ -77,7 +79,9 @@ public void stop() { } public Koe createKoe() { - return Koe.koe(); + return Koe.koe(KoeOptions.builder() + .setByteBufAllocator(this.leakDetect.getAllocator()) + .create()); } public JDA createJDA() { From 965b16662c95d6a3f6d8042aba430ddd090d4a0f Mon Sep 17 00:00:00 2001 From: Alula Date: Mon, 16 Feb 2026 16:33:50 +0100 Subject: [PATCH 04/28] Refactor codec registry --- MIGRATION.md | 39 ++++++ .../java/moe/kyokobot/koe/KoeOptions.java | 14 +- .../moe/kyokobot/koe/KoeOptionsBuilder.java | 19 ++- .../moe/kyokobot/koe/MediaConnection.java | 37 +++++- .../koe/codec/AbstractFramePoller.java | 5 +- .../java/moe/kyokobot/koe/codec/Codec.java | 115 ---------------- .../moe/kyokobot/koe/codec/CodecInfo.java | 96 ++++++++++++++ .../moe/kyokobot/koe/codec/CodecInstance.java | 73 +++++++++++ .../moe/kyokobot/koe/codec/CodecRegistry.java | 71 ++++++++++ .../koe/codec/DefaultCodecRegistry.java | 123 ++++++++++++++++++ .../moe/kyokobot/koe/codec/DefaultCodecs.java | 25 ---- .../koe/codec/FramePollerFactory.java | 2 +- .../moe/kyokobot/koe/codec/H264Codec.java | 11 -- .../moe/kyokobot/koe/codec/H264CodecInfo.java | 16 +++ .../moe/kyokobot/koe/codec/OpusCodec.java | 12 -- .../moe/kyokobot/koe/codec/OpusCodecInfo.java | 17 +++ .../java/moe/kyokobot/koe/codec/VP8Codec.java | 11 -- .../moe/kyokobot/koe/codec/VP8CodecInfo.java | 16 +++ .../java/moe/kyokobot/koe/codec/VP9Codec.java | 11 -- .../moe/kyokobot/koe/codec/VP9CodecInfo.java | 16 +++ .../codec/netty/NettyFramePollerFactory.java | 14 +- .../koe/codec/netty/NettyH264FramePoller.java | 12 +- .../koe/codec/netty/NettyOpusFramePoller.java | 10 +- .../koe/gateway/MediaGatewayV4Connection.java | 18 +-- .../koe/gateway/MediaGatewayV5Connection.java | 9 +- .../koe/gateway/MediaGatewayV8Connection.java | 9 +- .../koe/handler/ConnectionHandler.java | 4 +- .../kyokobot/koe/internal/DAVEManager.java | 13 +- .../koe/internal/MediaConnectionImpl.java | 16 +-- .../handler/DiscordUDPConnection.java | 17 ++- .../koe/media/MediaFrameProvider.java | 16 +-- .../koe/media/OpusAudioFrameProvider.java | 18 +-- 32 files changed, 618 insertions(+), 267 deletions(-) delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/Codec.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/CodecInstance.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecs.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/H264Codec.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/OpusCodec.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/VP8Codec.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/VP9Codec.java create mode 100644 core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java diff --git a/MIGRATION.md b/MIGRATION.md index f6ee69f..1a36a9a 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -4,3 +4,42 @@ 1. The public constructor has been removed. Use `VoiceServerInfo#builder()` to create instances of this class instead. 2. It's now required to pass `channelId` (the ID of the voice channel), as it is used as a MLS group identifier while using DAVE E2E encryption. + +## Codec API Changes + +### Overview + +The old `Codec` class has been replaced with two new classes: +- **`CodecInfo`**: Immutable codec capabilities (what a codec CAN do) +- **`CodecInstance`**: Session-specific codec instance with negotiated payload types (what a codec IS doing) + +### Codec Class Names + +All codec classes have been renamed with an `Info` suffix: + +| Old (2.x) | New (3.0) | +|----------------------|--------------------------| +| `OpusCodec.INSTANCE` | `OpusCodecInfo.INSTANCE` | +| `H264Codec.INSTANCE` | `H264CodecInfo.INSTANCE` | +| `VP8Codec.INSTANCE` | `VP8CodecInfo.INSTANCE` | +| `VP9Codec.INSTANCE` | `VP9CodecInfo.INSTANCE` | + +## MediaFrameProvider Changes + +The `MediaFrameProvider` interface now uses `CodecInstance` instead of `Codec`: + +**Old (2.x):** +```java +public interface MediaFrameProvider { + boolean canSendFrame(Codec codec); + boolean retrieve(Codec codec, ByteBuf buf, IntReference timestamp); +} +``` + +**New (3.0):** +```java +public interface MediaFrameProvider { + boolean canSendFrame(CodecInstance codec); + boolean retrieve(CodecInstance codec, ByteBuf buf, IntReference timestamp); +} +``` diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java index 05a2281..fbbc979 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java @@ -4,6 +4,8 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.DatagramChannel; import io.netty.channel.socket.SocketChannel; +import moe.kyokobot.koe.codec.CodecRegistry; +import moe.kyokobot.koe.codec.DefaultCodecRegistry; import moe.kyokobot.koe.codec.FramePollerFactory; import moe.kyokobot.koe.gateway.GatewayVersion; import org.jetbrains.annotations.NotNull; @@ -22,6 +24,7 @@ public class KoeOptions { private final ByteBufAllocator byteBufAllocator; private final GatewayVersion gatewayVersion; private final FramePollerFactory framePollerFactory; + private final CodecRegistry codecRegistry; private final boolean highPacketPriority; private final boolean deafened; private final boolean enableWSSPortOverride; @@ -34,6 +37,7 @@ public class KoeOptions { @NotNull ByteBufAllocator byteBufAllocator, @NotNull GatewayVersion gatewayVersion, @NotNull FramePollerFactory framePollerFactory, + @NotNull CodecRegistry codecRegistry, boolean highPacketPriority, boolean deafened, boolean enableWSSPortOverride, @@ -45,6 +49,7 @@ public class KoeOptions { this.byteBufAllocator = Objects.requireNonNull(byteBufAllocator); this.gatewayVersion = Objects.requireNonNull(gatewayVersion); this.framePollerFactory = Objects.requireNonNull(framePollerFactory); + this.codecRegistry = Objects.requireNonNull(codecRegistry); this.highPacketPriority = highPacketPriority; this.deafened = deafened; this.enableWSSPortOverride = enableWSSPortOverride; @@ -66,7 +71,7 @@ public KoeOptions( boolean deafened ) { this(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, gatewayVersion, - framePollerFactory, highPacketPriority, deafened, true, true); + framePollerFactory, new DefaultCodecRegistry(), highPacketPriority, deafened, true, true); } /** @@ -83,7 +88,7 @@ public KoeOptions( boolean highPacketPriority ) { this(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, gatewayVersion, - framePollerFactory, highPacketPriority, false); + framePollerFactory, new DefaultCodecRegistry(), highPacketPriority, false, true, true); } @NotNull @@ -116,6 +121,11 @@ public FramePollerFactory getFramePollerFactory() { return framePollerFactory; } + @NotNull + public CodecRegistry getCodecRegistry() { + return codecRegistry; + } + public boolean isHighPacketPriority() { return highPacketPriority; } diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java index e423776..419da7d 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java @@ -12,6 +12,8 @@ import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioDatagramChannel; import io.netty.channel.socket.nio.NioSocketChannel; +import moe.kyokobot.koe.codec.CodecRegistry; +import moe.kyokobot.koe.codec.DefaultCodecRegistry; import moe.kyokobot.koe.codec.FramePollerFactory; import moe.kyokobot.koe.codec.netty.NettyFramePollerFactory; import moe.kyokobot.koe.gateway.GatewayVersion; @@ -26,6 +28,7 @@ public class KoeOptionsBuilder { private ByteBufAllocator byteBufAllocator; private GatewayVersion gatewayVersion; private FramePollerFactory framePollerFactory; + private CodecRegistry codecRegistry; private boolean highPacketPriority; private boolean deafened; private boolean enableWSSPortOverride; @@ -47,6 +50,7 @@ public class KoeOptionsBuilder { this.byteBufAllocator = new PooledByteBufAllocator(); this.gatewayVersion = GatewayVersion.V8; this.framePollerFactory = new NettyFramePollerFactory(); + this.codecRegistry = new DefaultCodecRegistry(); this.highPacketPriority = true; this.deafened = false; this.enableWSSPortOverride = true; @@ -131,6 +135,19 @@ public KoeOptionsBuilder setFramePollerFactory(@NotNull FramePollerFactory frame return this; } + /** + * Sets a custom codec registry. + * Useful for registering custom codecs or removing built-in ones. + * Defaults to {@link DefaultCodecRegistry} with built-in codecs. + * + * @param codecRegistry An instance of {@link CodecRegistry} to use for the Koe client. + */ + public KoeOptionsBuilder setCodecRegistry(@NotNull CodecRegistry codecRegistry) { + Objects.requireNonNull(codecRegistry, "codecRegistry cannot be null"); + this.codecRegistry = codecRegistry; + return this; + } + /** * Sets whether to set IP_TOS flags to request high priority/low-delay for sent RTP packets. * Defaults to true. @@ -168,6 +185,6 @@ public KoeOptionsBuilder setDAVEEnabled(boolean enabled) { public KoeOptions create() { return new KoeOptions(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, - gatewayVersion, framePollerFactory, highPacketPriority, deafened, enableWSSPortOverride, enableDAVE); + gatewayVersion, framePollerFactory, codecRegistry, highPacketPriority, deafened, enableWSSPortOverride, enableDAVE); } } diff --git a/core/src/main/java/moe/kyokobot/koe/MediaConnection.java b/core/src/main/java/moe/kyokobot/koe/MediaConnection.java index e48c8d0..0ecb629 100644 --- a/core/src/main/java/moe/kyokobot/koe/MediaConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/MediaConnection.java @@ -1,7 +1,8 @@ package moe.kyokobot.koe; import moe.kyokobot.koe.media.MediaFrameProvider; -import moe.kyokobot.koe.codec.Codec; +import moe.kyokobot.koe.codec.CodecInfo; +import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.handler.ConnectionHandler; import org.jetbrains.annotations.NotNull; @@ -52,7 +53,22 @@ public interface MediaConnection extends Closeable { void setAudioSender(@Nullable MediaFrameProvider sender); - void setAudioCodec(@NotNull Codec audioCodec); + /** + * Sets the audio codec instance for this connection. + * + * @param audioCodec the codec instance to use + */ + void setAudioCodec(@NotNull CodecInstance audioCodec); + + /** + * Sets the audio codec using a codec info (convenience method). + * Creates a codec instance with default payload types. + * + * @param info the codec info to use + */ + default void setAudioCodec(@NotNull CodecInfo info) { + setAudioCodec(info.instantiate()); + } /** * Starts polling audio frames. Called automatically after connecting, you don't have to. @@ -67,7 +83,22 @@ public interface MediaConnection extends Closeable { void setVideoSender(@Nullable MediaFrameProvider sender); - void setVideoCodec(@Nullable Codec videoCodec); + /** + * Sets the video codec instance for this connection. + * + * @param videoCodec the codec instance to use, or null to disable video + */ + void setVideoCodec(@Nullable CodecInstance videoCodec); + + /** + * Sets the video codec using a codec info (convenience method). + * Creates a codec instance with default payload types. + * + * @param info the codec info to use + */ + default void setVideoCodec(@NotNull CodecInfo info) { + setVideoCodec(info.instantiate()); + } /** * Starts polling video frames. Called automatically after connecting if codec has been set. diff --git a/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java index 08a3203..ac6dc8c 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java @@ -3,14 +3,17 @@ import io.netty.buffer.ByteBufAllocator; import io.netty.channel.EventLoopGroup; import moe.kyokobot.koe.MediaConnection; +import org.jetbrains.annotations.NotNull; public abstract class AbstractFramePoller implements FramePoller { protected final MediaConnection connection; + protected final CodecInstance codec; protected final ByteBufAllocator allocator; protected final EventLoopGroup eventLoop; protected boolean polling = false; - public AbstractFramePoller(MediaConnection connection) { + public AbstractFramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { + this.codec = codec; this.connection = connection; this.allocator = connection.getOptions().getByteBufAllocator(); this.eventLoop = connection.getOptions().getEventLoopGroup(); diff --git a/core/src/main/java/moe/kyokobot/koe/codec/Codec.java b/core/src/main/java/moe/kyokobot/koe/codec/Codec.java deleted file mode 100644 index 06e326c..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/Codec.java +++ /dev/null @@ -1,115 +0,0 @@ -package moe.kyokobot.koe.codec; - -import moe.kyokobot.koe.internal.json.JsonObject; -import org.jetbrains.annotations.Nullable; - -import java.util.Objects; - -public abstract class Codec { - protected final String name; - protected final byte payloadType; - protected final byte rtxPayloadType; - protected final int priority; - protected final CodecType type; - protected final JsonObject jsonDescription; - - protected Codec(String name, byte payloadType, int priority, CodecType type) { - this(name, payloadType, (byte) 0, priority, type); - } - - protected Codec(String name, byte payloadType, byte rtxPayloadType, int priority, CodecType type) { - this.name = name; - this.payloadType = payloadType; - this.rtxPayloadType = rtxPayloadType; - this.priority = priority; - this.type = type; - - this.jsonDescription = new JsonObject() - .add("name", name) - .add("payload_type", payloadType) - .add("priority", priority) - .add("type", type.name().toLowerCase()); - } - - public String getName() { - return name; - } - - public byte getPayloadType() { - return payloadType; - } - - public byte getRetransmissionPayloadType() { - return rtxPayloadType; - } - - public int getPriority() { - return priority; - } - - public CodecType getType() { - return type; - } - - public JsonObject getJsonDescription() { - return jsonDescription; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Codec that = (Codec) o; - return payloadType == that.payloadType && type == that.type; - } - - @Override - public int hashCode() { - return Objects.hash(payloadType, type); - } - - /** - * Gets audio codec description by name. - * - * @param name the codec name - * @return Codec instance or null if the codec is not found/supported by Koe. - */ - @Nullable - public static Codec getAudio(String name) { - return DefaultCodecs.audioCodecs.get(name); - } - - /** - * Gets video codec description by name. - * - * @param name the codec name - * @return Codec instance or null if the codec is not found/supported by Koe. - */ - @Nullable - public static Codec getVideo(String name) { - return DefaultCodecs.audioCodecs.get(name); - } - - /** - * Gets audio or video codec by payload type. - * - * @param payloadType the payload type - * @return Codec instance or null if the codec is not found/supported by Koe. - */ - @Nullable - public static Codec getByPayload(byte payloadType) { - for (var codec : DefaultCodecs.audioCodecs.values()) { - if (codec.getPayloadType() == payloadType) { - return codec; - } - } - - for (var codec : DefaultCodecs.videoCodecs.values()) { - if (codec.getPayloadType() == payloadType) { - return codec; - } - } - - return null; - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java new file mode 100644 index 0000000..7bda349 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java @@ -0,0 +1,96 @@ +package moe.kyokobot.koe.codec; + +import moe.kyokobot.koe.internal.json.JsonObject; +import org.jetbrains.annotations.NotNull; + +/** + * Immutable codec capabilities and metadata. + * Represents what a codec CAN do, not what it IS doing in a session. + */ +public abstract class CodecInfo { + protected final String name; + protected final CodecType type; + protected final int defaultPriority; + protected final byte defaultPayloadType; + protected final byte defaultRtxPayloadType; + + protected CodecInfo(@NotNull String name, @NotNull CodecType type, byte defaultPayloadType, + byte defaultRtxPayloadType, int defaultPriority) { + this.name = name; + this.type = type; + this.defaultPayloadType = defaultPayloadType; + this.defaultRtxPayloadType = defaultRtxPayloadType; + this.defaultPriority = defaultPriority; + } + + @NotNull + public String getName() { + return name; + } + + @NotNull + public CodecType getType() { + return type; + } + + public byte getDefaultPayloadType() { + return defaultPayloadType; + } + + public byte getDefaultRtxPayloadType() { + return defaultRtxPayloadType; + } + + public int getDefaultPriority() { + return defaultPriority; + } + + /** + * Creates a codec instance with negotiated payload types. + * + * @param negotiatedPayloadType the negotiated payload type for this session + * @param negotiatedRtxPayloadType the negotiated RTX payload type for this session + * @return a new CodecInstance with the negotiated payload types + */ + @NotNull + public CodecInstance instantiate(byte negotiatedPayloadType, byte negotiatedRtxPayloadType) { + return new CodecInstance(this, negotiatedPayloadType, negotiatedRtxPayloadType); + } + + /** + * Creates a codec instance using default payload types (for Discord's simple protocol). + * + * @return a new CodecInstance with default payload types + */ + @NotNull + public CodecInstance instantiate() { + return instantiate(defaultPayloadType, defaultRtxPayloadType); + } + + /** + * Converts this codec info to JSON format for gateway protocol. + * + * @return JSON representation of this codec + */ + @NotNull + public JsonObject toJson() { + return new JsonObject() + .add("name", name) + .add("payload_type", defaultPayloadType) + .add("priority", defaultPriority) + .add("type", type.name().toLowerCase()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CodecInfo codecInfo = (CodecInfo) o; + return name.equals(codecInfo.name) && type == codecInfo.type; + } + + @Override + public int hashCode() { + return Objects.hash(name, type); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/CodecInstance.java b/core/src/main/java/moe/kyokobot/koe/codec/CodecInstance.java new file mode 100644 index 0000000..7ef680d --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/CodecInstance.java @@ -0,0 +1,73 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; + +import java.util.Objects; + +/** + * A codec instance in an active media session. + * Contains negotiated payload types that may differ from defaults. + */ +public class CodecInstance { + private final CodecInfo info; + private final byte payloadType; + private final byte rtxPayloadType; + + CodecInstance(@NotNull CodecInfo info, byte payloadType, byte rtxPayloadType) { + this.info = Objects.requireNonNull(info, "info cannot be null"); + this.payloadType = payloadType; + this.rtxPayloadType = rtxPayloadType; + } + + @NotNull + public String getName() { + return info.getName(); + } + + @NotNull + public CodecType getType() { + return info.getType(); + } + + public byte getPayloadType() { + return payloadType; + } + + public byte getRetransmissionPayloadType() { + return rtxPayloadType; + } + + public int getPriority() { + return info.getDefaultPriority(); + } + + @NotNull + public CodecInfo getInfo() { + return info; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CodecInstance that = (CodecInstance) o; + return payloadType == that.payloadType && + rtxPayloadType == that.rtxPayloadType && + info.equals(that.info); + } + + @Override + public int hashCode() { + return Objects.hash(info, payloadType, rtxPayloadType); + } + + @Override + public String toString() { + return "CodecInstance{" + + "name='" + getName() + '\'' + + ", type=" + getType() + + ", payloadType=" + payloadType + + ", rtxPayloadType=" + rtxPayloadType + + '}'; + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java b/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java new file mode 100644 index 0000000..ce08a4e --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java @@ -0,0 +1,71 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; + +/** + * Registry for managing available codec capabilities. + * Allows users to register custom codecs. + */ +public interface CodecRegistry { + /** + * Registers a codec capability. + * + * @param codec the codec info to register + * @throws IllegalArgumentException if codec with same name already exists + */ + void register(@NotNull CodecInfo codec); + + /** + * Unregisters a codec by name. + * + * @param codecName the name of the codec to unregister + * @return true if codec was registered, false if not found + */ + boolean unregister(@NotNull String codecName); + + /** + * Gets codec info by name. + * + * @param name the codec name (case-insensitive) + * @return CodecInfo or null if not found + */ + @Nullable + CodecInfo getByName(@NotNull String name); + + /** + * Gets codec info by payload type. + * Useful for decoding incoming packets. + * + * @param payloadType the payload type + * @return CodecInfo or null if not found + */ + @Nullable + CodecInfo getByPayloadType(byte payloadType); + + /** + * Gets all registered audio codecs. + * + * @return collection of audio codec infos + */ + @NotNull + Collection getAudioCodecs(); + + /** + * Gets all registered video codecs. + * + * @return collection of video codec infos + */ + @NotNull + Collection getVideoCodecs(); + + /** + * Gets all registered codecs (audio + video). + * + * @return collection of all codec infos + */ + @NotNull + Collection getAllCodecs(); +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java b/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java new file mode 100644 index 0000000..05d13bc --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java @@ -0,0 +1,123 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Default implementation of CodecRegistry with built-in codecs pre-registered. + */ +public class DefaultCodecRegistry implements CodecRegistry { + protected final ConcurrentHashMap codecsByName = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap codecsByPayloadType = new ConcurrentHashMap<>(); + + /** + * Creates a registry with all built-in codecs pre-registered. + */ + public DefaultCodecRegistry() { + registerBuiltInCodecs(); + } + + /** + * Creates an empty registry with no codecs registered. + * Use this if you want to register only specific codecs. + * + * @return an empty CodecRegistry + */ + public static DefaultCodecRegistry empty() { + return new EmptyCodecRegistry(); + } + + /** + * Registers all built-in codecs. Can be overridden to skip registration. + */ + protected void registerBuiltInCodecs() { + register(OpusCodecInfo.INSTANCE); + register(H264CodecInfo.INSTANCE); + register(VP8CodecInfo.INSTANCE); + register(VP9CodecInfo.INSTANCE); + } + + /** + * Private subclass for empty registries to avoid registering built-ins. + */ + private static class EmptyCodecRegistry extends DefaultCodecRegistry { + EmptyCodecRegistry() { + super(); // Initialize maps + } + + @Override + protected void registerBuiltInCodecs() { + // Skip built-in codec registration + } + } + + @Override + public void register(@NotNull CodecInfo codec) { + String name = codec.getName().toLowerCase(); + + if (codecsByName.containsKey(name)) { + throw new IllegalArgumentException("Codec already registered: " + name); + } + + codecsByName.put(name, codec); + codecsByPayloadType.put(codec.getDefaultPayloadType(), codec); + + if (codec.getDefaultRtxPayloadType() != 0) { + codecsByPayloadType.put(codec.getDefaultRtxPayloadType(), codec); + } + } + + @Override + public boolean unregister(@NotNull String codecName) { + CodecInfo removed = codecsByName.remove(codecName.toLowerCase()); + if (removed != null) { + codecsByPayloadType.remove(removed.getDefaultPayloadType()); + if (removed.getDefaultRtxPayloadType() != 0) { + codecsByPayloadType.remove(removed.getDefaultRtxPayloadType()); + } + return true; + } + return false; + } + + @Override + @Nullable + public CodecInfo getByName(@NotNull String name) { + return codecsByName.get(name.toLowerCase()); + } + + @Override + @Nullable + public CodecInfo getByPayloadType(byte payloadType) { + return codecsByPayloadType.get(payloadType); + } + + @Override + @NotNull + public Collection getAudioCodecs() { + return codecsByName.values().stream() + .filter(c -> c.getType() == CodecType.AUDIO) + .collect(Collectors.toList()); + } + + @Override + @NotNull + public Collection getVideoCodecs() { + return codecsByName.values().stream() + .filter(c -> c.getType() == CodecType.VIDEO) + .collect(Collectors.toList()); + } + + @Override + @NotNull + public Collection getAllCodecs() { + return new ArrayList<>(codecsByName.values()); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecs.java b/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecs.java deleted file mode 100644 index bfcd84d..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecs.java +++ /dev/null @@ -1,25 +0,0 @@ -package moe.kyokobot.koe.codec; - -import java.util.Map; - -// todo: migrate to codec registry or something -public class DefaultCodecs { - private DefaultCodecs() { - // - } - - public static final Map audioCodecs; - public static final Map videoCodecs; - - static { - audioCodecs = Map.of( - "opus", OpusCodec.INSTANCE - ); - - videoCodecs = Map.of( - "H264", H264Codec.INSTANCE, - "VP8", VP8Codec.INSTANCE, - "VP9", VP9Codec.INSTANCE - ); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java index a6456db..da5d78e 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java @@ -4,5 +4,5 @@ @FunctionalInterface public interface FramePollerFactory { - FramePoller createFramePoller(Codec codec, MediaConnection connection); + FramePoller createFramePoller(CodecInstance codec, MediaConnection connection); } diff --git a/core/src/main/java/moe/kyokobot/koe/codec/H264Codec.java b/core/src/main/java/moe/kyokobot/koe/codec/H264Codec.java deleted file mode 100644 index 419a997..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/H264Codec.java +++ /dev/null @@ -1,11 +0,0 @@ -package moe.kyokobot.koe.codec; - -public class H264Codec extends Codec { - public static final byte PAYLOAD_TYPE = (byte) 101; - public static final byte RTX_PAYLOAD_TYPE = (byte) 102; - public static final H264Codec INSTANCE = new H264Codec(); - - public H264Codec() { - super("H264", PAYLOAD_TYPE, RTX_PAYLOAD_TYPE, 1000, CodecType.VIDEO); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java new file mode 100644 index 0000000..c0d13da --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java @@ -0,0 +1,16 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; + +/** + * H.264 video codec information. + */ +public class H264CodecInfo extends CodecInfo { + public static final H264CodecInfo INSTANCE = new H264CodecInfo(); + public static final byte DEFAULT_PAYLOAD_TYPE = (byte) 101; + public static final byte DEFAULT_RTX_PAYLOAD_TYPE = (byte) 102; + + private H264CodecInfo() { + super("H264", CodecType.VIDEO, DEFAULT_PAYLOAD_TYPE, DEFAULT_RTX_PAYLOAD_TYPE, 1000); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/OpusCodec.java b/core/src/main/java/moe/kyokobot/koe/codec/OpusCodec.java deleted file mode 100644 index 0a6313e..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/OpusCodec.java +++ /dev/null @@ -1,12 +0,0 @@ -package moe.kyokobot.koe.codec; - -public class OpusCodec extends Codec { - public static final byte PAYLOAD_TYPE = (byte) 120; - public static final int FRAME_DURATION = 20; - public static final OpusCodec INSTANCE = new OpusCodec(); - public static final byte[] SILENCE_FRAME = new byte[] {(byte)0xF8, (byte)0xFF, (byte)0xFE}; - - public OpusCodec() { - super("opus", PAYLOAD_TYPE, 1000, CodecType.AUDIO); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java new file mode 100644 index 0000000..4f56d28 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java @@ -0,0 +1,17 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; + +/** + * Opus audio codec information. + */ +public class OpusCodecInfo extends CodecInfo { + public static final OpusCodecInfo INSTANCE = new OpusCodecInfo(); + public static final byte DEFAULT_PAYLOAD_TYPE = (byte) 120; + public static final int FRAME_DURATION = 20; + public static final byte[] SILENCE_FRAME = new byte[] {(byte)0xF8, (byte)0xFF, (byte)0xFE}; + + private OpusCodecInfo() { + super("opus", CodecType.AUDIO, DEFAULT_PAYLOAD_TYPE, (byte) 0, 1000); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/VP8Codec.java b/core/src/main/java/moe/kyokobot/koe/codec/VP8Codec.java deleted file mode 100644 index 33ac7ad..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/VP8Codec.java +++ /dev/null @@ -1,11 +0,0 @@ -package moe.kyokobot.koe.codec; - -public class VP8Codec extends Codec { - public static final byte PAYLOAD_TYPE = (byte) 103; - public static final byte RTX_PAYLOAD_TYPE = (byte) 104; - public static final VP8Codec INSTANCE = new VP8Codec(); - - public VP8Codec() { - super("VP8", PAYLOAD_TYPE, RTX_PAYLOAD_TYPE, 2000, CodecType.VIDEO); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java new file mode 100644 index 0000000..fe92a5e --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java @@ -0,0 +1,16 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; + +/** + * VP8 video codec information. + */ +public class VP8CodecInfo extends CodecInfo { + public static final VP8CodecInfo INSTANCE = new VP8CodecInfo(); + public static final byte DEFAULT_PAYLOAD_TYPE = (byte) 103; + public static final byte DEFAULT_RTX_PAYLOAD_TYPE = (byte) 104; + + private VP8CodecInfo() { + super("VP8", CodecType.VIDEO, DEFAULT_PAYLOAD_TYPE, DEFAULT_RTX_PAYLOAD_TYPE, 2000); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/VP9Codec.java b/core/src/main/java/moe/kyokobot/koe/codec/VP9Codec.java deleted file mode 100644 index 40e9623..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/VP9Codec.java +++ /dev/null @@ -1,11 +0,0 @@ -package moe.kyokobot.koe.codec; - -public class VP9Codec extends Codec { - public static final byte PAYLOAD_TYPE = (byte) 105; - public static final byte RTX_PAYLOAD_TYPE = (byte) 106; - public static final VP9Codec INSTANCE = new VP9Codec(); - - public VP9Codec() { - super("VP9", PAYLOAD_TYPE, RTX_PAYLOAD_TYPE, 3000, CodecType.VIDEO); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java new file mode 100644 index 0000000..5bbf816 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java @@ -0,0 +1,16 @@ +package moe.kyokobot.koe.codec; + +import org.jetbrains.annotations.NotNull; + +/** + * VP9 video codec information. + */ +public class VP9CodecInfo extends CodecInfo { + public static final VP9CodecInfo INSTANCE = new VP9CodecInfo(); + public static final byte DEFAULT_PAYLOAD_TYPE = (byte) 105; + public static final byte DEFAULT_RTX_PAYLOAD_TYPE = (byte) 106; + + private VP9CodecInfo() { + super("VP9", CodecType.VIDEO, DEFAULT_PAYLOAD_TYPE, DEFAULT_RTX_PAYLOAD_TYPE, 3000); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java index 7ba2e28..8e1d3e1 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java @@ -6,23 +6,23 @@ import java.util.HashMap; import java.util.Map; -import java.util.function.Function; +import java.util.function.BiFunction; public class NettyFramePollerFactory implements FramePollerFactory { - private final Map> codecMap; + private final Map> codecMap; public NettyFramePollerFactory() { codecMap = new HashMap<>(); - codecMap.put(OpusCodec.INSTANCE, NettyOpusFramePoller::new); - codecMap.put(H264Codec.INSTANCE, NettyH264FramePoller::new); + codecMap.put("opus", NettyOpusFramePoller::new); + codecMap.put("H264", NettyH264FramePoller::new); } @Override @Nullable - public FramePoller createFramePoller(Codec codec, MediaConnection connection) { - var constructor = codecMap.get(codec); + public FramePoller createFramePoller(CodecInstance codec, MediaConnection connection) { + var constructor = codecMap.get(codec.getName()); if (constructor != null) { - return constructor.apply(connection); + return constructor.apply(codec, connection); } return null; } diff --git a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java index a366e5e..6e6faa9 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java @@ -2,8 +2,9 @@ import moe.kyokobot.koe.MediaConnection; import moe.kyokobot.koe.codec.AbstractFramePoller; -import moe.kyokobot.koe.codec.H264Codec; +import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.media.IntReference; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,8 +17,8 @@ public class NettyH264FramePoller extends AbstractFramePoller { */ private static final int FRAME_RATE = 1000 / 30; - public NettyH264FramePoller(MediaConnection connection) { - super(connection); + public NettyH264FramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { + super(codec, connection); } /** @@ -56,8 +57,7 @@ private void pollFrame() { try { do { var handler = connection.getConnectionHandler(); - var sender = connection.getAudioSender(); - var codec = H264Codec.INSTANCE; + var sender = connection.getVideoSender(); if (sender != null && handler != null && sender.canSendFrame(codec)) { var buf = allocator.buffer(); @@ -65,7 +65,7 @@ private void pollFrame() { pollNext = sender.retrieve(codec, buf, timestamp); int len = buf.writerIndex() - start; if (len != 0) { - handler.sendFrame(H264Codec.PAYLOAD_TYPE, timestamp.get(), buf, len, true); + handler.sendFrame(codec.getPayloadType(), timestamp.get(), buf, len, true); } buf.release(); } diff --git a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java index 0b7b812..b1560f0 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java @@ -2,8 +2,9 @@ import moe.kyokobot.koe.MediaConnection; import moe.kyokobot.koe.codec.AbstractFramePoller; -import moe.kyokobot.koe.codec.OpusCodec; +import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.media.IntReference; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,8 +13,8 @@ public class NettyOpusFramePoller extends AbstractFramePoller { private static final Logger logger = LoggerFactory.getLogger(NettyOpusFramePoller.class); - public NettyOpusFramePoller(MediaConnection connection) { - super(connection); + public NettyOpusFramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { + super(codec, connection); } /** @@ -50,7 +51,6 @@ private void pollFrame() { try { var handler = connection.getConnectionHandler(); var sender = connection.getAudioSender(); - var codec = OpusCodec.INSTANCE; // ugly but it's the hottest path in Koe and Java is a shit language. if (sender != null && handler != null && sender.canSendFrame(codec)) { @@ -60,7 +60,7 @@ private void pollFrame() { sender.retrieve(codec, buf, timestamp); int len = buf.writerIndex() - start; if (len != 0) { - handler.sendFrame(OpusCodec.PAYLOAD_TYPE, timestamp.get(), buf, len, false); + handler.sendFrame(codec.getPayloadType(), timestamp.get(), buf, len, false); } buf.release(); } diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java index f057b13..b4970cc 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java @@ -2,7 +2,7 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.codec.OpusCodec; +import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.crypto.EncryptionMode; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; @@ -22,12 +22,6 @@ public class MediaGatewayV4Connection extends AbstractMediaGatewayConnection { private static final Logger logger = LoggerFactory.getLogger(MediaGatewayV4Connection.class); - private static final JsonArray SUPPORTED_CODECS; - - static { - SUPPORTED_CODECS = new JsonArray(); - SUPPORTED_CODECS.add(OpusCodec.INSTANCE.getJsonDescription()); - } private int ssrc; private SocketAddress address; @@ -194,9 +188,17 @@ private void selectProtocol(String protocol) { .add("port", ourAddress.getPort()) .add("mode", mode); + var codecs = new JsonArray(); + // V4 only supports audio codecs + connection.getOptions().getCodecRegistry() + .getAudioCodecs() + .stream() + .map(codecInfo -> codecInfo.toJson()) + .forEach(codecs::add); + sendInternalPayload(Op.SELECT_PROTOCOL, new JsonObject() .add("protocol", "udp") - .add("codecs", SUPPORTED_CODECS) + .add("codecs", codecs) .add("rtc_connection_id", rtcConnectionId.toString()) .add("data", udpInfo) .combine(udpInfo)); diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java index 0df172e..416e35b 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java @@ -2,8 +2,6 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.codec.Codec; -import moe.kyokobot.koe.codec.DefaultCodecs; import moe.kyokobot.koe.crypto.EncryptionMode; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; @@ -20,7 +18,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import java.util.stream.Stream; public class MediaGatewayV5Connection extends AbstractMediaGatewayConnection { private static final Logger logger = LoggerFactory.getLogger(MediaGatewayV5Connection.class); @@ -216,8 +213,10 @@ private void selectProtocol(String protocol) { .add("mode", mode); var codecs = new JsonArray(); - Stream.concat(DefaultCodecs.audioCodecs.values().stream(), DefaultCodecs.videoCodecs.values().stream()) - .map(Codec::getJsonDescription) + connection.getOptions().getCodecRegistry() + .getAllCodecs() + .stream() + .map(codecInfo -> codecInfo.toJson()) .forEach(codecs::add); sendInternalPayload(Op.SELECT_PROTOCOL, new JsonObject() diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java index 1424121..98765bf 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java @@ -2,8 +2,6 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.codec.Codec; -import moe.kyokobot.koe.codec.DefaultCodecs; import moe.kyokobot.koe.crypto.EncryptionMode; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; @@ -20,7 +18,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import java.util.stream.Stream; public class MediaGatewayV8Connection extends AbstractMediaGatewayConnection { private static final Logger logger = LoggerFactory.getLogger(MediaGatewayV8Connection.class); @@ -383,8 +380,10 @@ private void selectProtocol(String protocol) { .add("mode", mode); var codecs = new JsonArray(); - Stream.concat(DefaultCodecs.audioCodecs.values().stream(), DefaultCodecs.videoCodecs.values().stream()) - .map(Codec::getJsonDescription) + connection.getOptions().getCodecRegistry() + .getAllCodecs() + .stream() + .map(codecInfo -> codecInfo.toJson()) .forEach(codecs::add); sendInternalPayload(Op.SELECT_PROTOCOL, new JsonObject() diff --git a/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java b/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java index dd590f3..291c463 100644 --- a/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java +++ b/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java @@ -1,7 +1,7 @@ package moe.kyokobot.koe.handler; import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.codec.Codec; +import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.internal.json.JsonObject; import java.util.concurrent.CompletionStage; @@ -18,7 +18,7 @@ public interface ConnectionHandler { void handleSessionDescription(JsonObject object); - default void sendFrame(Codec codec, int timestamp, ByteBuf data, int start) { + default void sendFrame(CodecInstance codec, int timestamp, ByteBuf data, int start) { sendFrame(codec.getPayloadType(), timestamp, data, start, false); } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java index 87b59a2..d6a8d98 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java @@ -1,7 +1,7 @@ package moe.kyokobot.koe.internal; import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.codec.OpusCodec; +import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.internal.json.JsonObject; import moe.kyokobot.libdave.KeyRatchet; import moe.kyokobot.libdave.MediaType; @@ -17,6 +17,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; public class DAVEManager implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(DAVEManager.class); @@ -27,8 +28,8 @@ public class DAVEManager implements AutoCloseable { private final MediaConnectionImpl connection; private final NettyDaveFactory factory; private final Session daveSession; - private final Set recognizedUserIds = new HashSet<>(); - private final Map pendingTransitions = new HashMap<>(); + private final Set recognizedUserIds = new ConcurrentHashMap<>(); + private final Map pendingTransitions = new ConcurrentHashMap<>(); private final int maxProtocolVersion; private NettyEncryptor selfEncryptor; @@ -63,9 +64,9 @@ public void removeUser(String userId) { public ByteBuf encrypt(MediaType mediaType, int ssrc, ByteBuf output, ByteBuf input, int size) { if (mediaType == MediaType.AUDIO && size == 3) { input.markReaderIndex(); - var b1 = input.readByte() == OpusCodec.SILENCE_FRAME[0]; - var b2 = input.readByte() == OpusCodec.SILENCE_FRAME[1]; - var b3 = input.readByte() == OpusCodec.SILENCE_FRAME[2]; + var b1 = input.readByte() == OpusCodecInfo.SILENCE_FRAME[0]; + var b2 = input.readByte() == OpusCodecInfo.SILENCE_FRAME[1]; + var b3 = input.readByte() == OpusCodecInfo.SILENCE_FRAME[2]; var isSilence = b1 && b2 && b3; input.resetReaderIndex(); diff --git a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java index e7bc538..957fa47 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java @@ -1,10 +1,10 @@ package moe.kyokobot.koe.internal; import moe.kyokobot.koe.*; -import moe.kyokobot.koe.codec.Codec; +import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.codec.CodecType; import moe.kyokobot.koe.codec.FramePoller; -import moe.kyokobot.koe.codec.OpusCodec; +import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.gateway.MediaValve; import moe.kyokobot.koe.handler.ConnectionHandler; @@ -27,8 +27,8 @@ public class MediaConnectionImpl implements MediaConnection { private MediaGatewayConnection gatewayConnection; private ConnectionHandler connectionHandler; private VoiceServerInfo info; - private Codec audioCodec; - private Codec videoCodec; + private CodecInstance audioCodec; + private CodecInstance videoCodec; private FramePoller audioPoller; private FramePoller videoPoller; private MediaFrameProvider audioSender; @@ -39,7 +39,7 @@ public MediaConnectionImpl(@NotNull KoeClientImpl client, long guildId) { this.client = Objects.requireNonNull(client); this.guildId = guildId; this.dispatcher = new EventDispatcher(); - this.audioCodec = OpusCodec.INSTANCE; + this.audioCodec = OpusCodecInfo.INSTANCE.instantiate(); this.audioPoller = client.getOptions().getFramePollerFactory().createFramePoller(this.audioCodec, this); this.videoCodec = null; this.videoPoller = null; @@ -146,7 +146,7 @@ public void setAudioSender(@Nullable MediaFrameProvider sender) { } @Override - public void setAudioCodec(@NotNull Codec audioCodec) { + public void setAudioCodec(@NotNull CodecInstance audioCodec) { if (Objects.requireNonNull(audioCodec).getType() != CodecType.AUDIO) { throw new IllegalArgumentException("Specified codec must be an audio codec!"); } @@ -189,7 +189,7 @@ public void setVideoSender(@Nullable MediaFrameProvider sender) { } @Override - public void setVideoCodec(@Nullable Codec videoCodec) { + public void setVideoCodec(@Nullable CodecInstance videoCodec) { if (videoCodec == null) { this.stopVideoFramePolling(); this.videoCodec = null; @@ -198,7 +198,7 @@ public void setVideoCodec(@Nullable Codec videoCodec) { } if (videoCodec.getType() != CodecType.VIDEO) { - throw new IllegalArgumentException("Specified codec must be an video codec!"); + throw new IllegalArgumentException("Specified codec must be a video codec!"); } boolean wasPolling = videoPoller != null && videoPoller.isPolling(); diff --git a/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java b/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java index 23bc9fa..e4f3c34 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java @@ -5,7 +5,9 @@ import io.netty.buffer.ByteBufAllocator; import io.netty.channel.ChannelInitializer; import io.netty.channel.socket.DatagramChannel; -import moe.kyokobot.koe.codec.Codec; +import moe.kyokobot.koe.codec.CodecInfo; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.CodecRegistry; import moe.kyokobot.koe.crypto.EncryptionMode; import moe.kyokobot.koe.handler.ConnectionHandler; import moe.kyokobot.koe.internal.MediaConnectionImpl; @@ -78,10 +80,15 @@ public void handleSessionDescription(JsonObject object) { var audioCodecName = object.getString("audio_codec"); encryptionMode = EncryptionMode.get(mode); - var audioCodec = Codec.getAudio(audioCodecName); - - if (audioCodecName != null && audioCodec == null) { - logger.warn("Unsupported audio codec type: {}, no audio data will be polled", audioCodecName); + CodecInstance audioCodec = null; + if (audioCodecName != null) { + CodecRegistry registry = connection.getOptions().getCodecRegistry(); + CodecInfo audioCodecInfo = registry.getByName(audioCodecName); + if (audioCodecInfo != null) { + audioCodec = audioCodecInfo.instantiate(); + } else { + logger.warn("Unsupported audio codec type: {}, no audio data will be polled", audioCodecName); + } } if (encryptionMode == null) { diff --git a/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java b/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java index 506fbeb..6317620 100644 --- a/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java +++ b/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java @@ -1,7 +1,7 @@ package moe.kyokobot.koe.media; import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.codec.Codec; +import moe.kyokobot.koe.codec.CodecInstance; /** * Base interface for media frame providers. Note that Koe doesn't handle stuff such as speaking state, silent frames @@ -26,26 +26,26 @@ public interface MediaFrameProvider { void setFrameInterval(int interval); /** - * @return If true, Koe will request media data for given {@link Codec} by - * calling {@link #retrieve(Codec, ByteBuf, moe.kyokobot.koe.media.IntReference)} method. + * @return If true, Koe will request media data for given {@link CodecInstance} by + * calling {@link #retrieve(CodecInstance, ByteBuf, moe.kyokobot.koe.media.IntReference)} method. */ - boolean canSendFrame(Codec codec); + boolean canSendFrame(CodecInstance codec); /** - * If {@link #canSendFrame(Codec)} returns true, Koe will attempt to retrieve an media frame encoded with specified - * {@link Codec} type, by calling this method with target {@link ByteBuf} where the data should be written to. + * If {@link #canSendFrame(CodecInstance)} returns true, Koe will attempt to retrieve an media frame encoded with specified + * {@link CodecInstance} type, by calling this method with target {@link ByteBuf} where the data should be written to. * Do not call {@link ByteBuf#release()} - memory management is already handled by Koe itself. In case if no * data gets written to the buffer, audio packet won't be sent. *

* Do not let this method block - all data should be queued on another thread or pre-loaded in * memory - otherwise it will very likely have significant impact on application performance. * - * @param codec {@link Codec} type this handler was registered with. + * @param codec {@link CodecInstance} type this handler was registered with. * @param buf {@link ByteBuf} the buffer where the media data should be written to. * @param timestamp {@link IntReference} reference to current frame timestamp, which must be updated with * timestamp of written frame. * @return If true, Koe will immediately attempt to poll a next frame, this is meant for video transmissions. */ - boolean retrieve(Codec codec, ByteBuf buf, IntReference timestamp); + boolean retrieve(CodecInstance codec, ByteBuf buf, IntReference timestamp); } diff --git a/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java b/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java index 943cd31..0e93840 100644 --- a/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java +++ b/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java @@ -3,8 +3,8 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.KoeEventAdapter; import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.Codec; -import moe.kyokobot.koe.codec.OpusCodec; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.gateway.SpeakingFlags; import java.util.Objects; @@ -41,7 +41,7 @@ public void setSpeakingMask(int speakingMask) { @Override public int getFrameInterval() { - return OpusCodec.FRAME_DURATION; + return OpusCodecInfo.FRAME_DURATION; } @Override @@ -50,8 +50,8 @@ public void setFrameInterval(int interval) { } @Override - public final boolean canSendFrame(Codec codec) { - if (codec.getPayloadType() != OpusCodec.PAYLOAD_TYPE) { + public final boolean canSendFrame(CodecInstance codec) { + if (!"opus".equalsIgnoreCase(codec.getName())) { return false; } @@ -73,14 +73,14 @@ public final boolean canSendFrame(Codec codec) { } @Override - public final boolean retrieve(Codec codec, ByteBuf buf, IntReference timestamp) { - if (codec.getPayloadType() != OpusCodec.PAYLOAD_TYPE) { + public final boolean retrieve(CodecInstance codec, ByteBuf buf, IntReference timestamp) { + if (!"opus".equalsIgnoreCase(codec.getName())) { return false; } if (counter > 0) { counter--; - buf.writeBytes(OpusCodec.SILENCE_FRAME); + buf.writeBytes(OpusCodecInfo.SILENCE_FRAME); if (speaking) { setSpeaking(false); @@ -103,7 +103,7 @@ public final boolean retrieve(Codec codec, ByteBuf buf, IntReference timestamp) } long now = System.currentTimeMillis(); - boolean changeTalking = (now - lastFramePolled) > OpusCodec.FRAME_DURATION; + boolean changeTalking = (now - lastFramePolled) > OpusCodecInfo.FRAME_DURATION; lastFramePolled = now; if (changeTalking) { setSpeaking(written); From eec5e4e0acb3339ec8a215d46d9ee210ed3b8950 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 17 Feb 2026 23:48:06 +0100 Subject: [PATCH 05/28] Fix DAVEManager compilation error --- core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java index d6a8d98..6a5af53 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java @@ -13,8 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -28,7 +26,7 @@ public class DAVEManager implements AutoCloseable { private final MediaConnectionImpl connection; private final NettyDaveFactory factory; private final Session daveSession; - private final Set recognizedUserIds = new ConcurrentHashMap<>(); + private final Set recognizedUserIds = ConcurrentHashMap.newKeySet(); private final Map pendingTransitions = new ConcurrentHashMap<>(); private final int maxProtocolVersion; From 06612106d3d04ef43e636d07323407b33ef328b7 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 17 Feb 2026 23:50:29 +0100 Subject: [PATCH 06/28] Fix the UDP queue test bot --- testbot/build.gradle.kts | 3 +++ .../moe/kyokobot/koe/testbot/TestBot.java | 19 +++++++++++-------- .../koe/testbot/UdpQueueTestBotLauncher.java | 6 +++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/testbot/build.gradle.kts b/testbot/build.gradle.kts index 72685d8..ea09141 100644 --- a/testbot/build.gradle.kts +++ b/testbot/build.gradle.kts @@ -5,5 +5,8 @@ dependencies { implementation("dev.arbjerg:lavaplayer:2.2.6") implementation("com.github.lavalink-devs:lavaplayer-youtube-source:1.16.0") implementation("ch.qos.logback:logback-classic:1.5.18") + implementation("moe.kyokobot.libdave:natives-linux-x86-64:1.0-SNAPSHOT") + implementation("club.minnced:udpqueue-native-linux-x86-64:0.2.12") + implementation("club.minnced:udpqueue-native-win-x86-64:0.2.12") } diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java index 1628f81..464d748 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java @@ -25,6 +25,7 @@ import net.dv8tion.jda.api.hooks.ListenerAdapter; import net.dv8tion.jda.api.hooks.VoiceDispatchInterceptor; import net.dv8tion.jda.api.requests.GatewayIntent; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,8 +52,8 @@ public class TestBot extends ListenerAdapter implements VoiceDispatchInterceptor private Koe koe; private KoeClient koeClient; private AudioPlayerManager playerManager; - private Map playerMap = new ConcurrentHashMap<>(); - private Map vsuChannelMap = new ConcurrentHashMap<>(); + private final Map playerMap = new ConcurrentHashMap<>(); + private final Map vsuChannelMap = new ConcurrentHashMap<>(); public TestBot(String token) { this.leakDetect = new NettyLeakDetect(); @@ -61,7 +62,11 @@ public TestBot(String token) { public void start() { this.jda = createJDA(); - this.koe = createKoe(); + var options = configureKoe(KoeOptions.builder() + .setByteBufAllocator(this.leakDetect.getAllocator()) + .setDAVEEnabled(false) // TODO: fix segfaults + ); + this.koe = Koe.koe(options); this.playerManager = createAudioPlayerManager(); } @@ -78,10 +83,8 @@ public void stop() { } } - public Koe createKoe() { - return Koe.koe(KoeOptions.builder() - .setByteBufAllocator(this.leakDetect.getAllocator()) - .create()); + public KoeOptions configureKoe(KoeOptionsBuilder builder) { + return builder.create(); } public JDA createJDA() { @@ -101,7 +104,7 @@ public AudioPlayerManager createAudioPlayerManager() { } @Override - public void onReady(ReadyEvent event) { + public void onReady(@NotNull ReadyEvent event) { koeClient = koe.newClient(jda.getSelfUser().getIdLong()); } diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java index 6095cbd..9efc205 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java @@ -8,10 +8,10 @@ public class UdpQueueTestBotLauncher { public static void main(String[] args) { var bot = new TestBot(System.getenv("TOKEN")) { @Override - public Koe createKoe() { - return Koe.koe(KoeOptions.builder() + public KoeOptions configureKoe(KoeOptionsBuilder options) { + return options .setFramePollerFactory(new UdpQueueFramePollerFactory()) - .create()); + .create(); } }; Runtime.getRuntime().addShutdownHook(new Thread(bot::stop)); From 1440bbb1b540e382f2426d4c842e3c1fb0890945 Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 18 Feb 2026 00:52:40 +0100 Subject: [PATCH 07/28] Refactor the poller and provider API --- MIGRATION.md | 121 +++++++++++-- README.md | 21 ++- .../java/moe/kyokobot/koe/KoeOptions.java | 2 +- .../moe/kyokobot/koe/KoeOptionsBuilder.java | 4 +- .../moe/kyokobot/koe/MediaConnection.java | 17 +- .../koe/codec/AbstractFramePoller.java | 26 --- .../moe/kyokobot/koe/codec/CodecInfo.java | 2 + .../moe/kyokobot/koe/codec/FramePoller.java | 9 - .../koe/codec/FramePollerFactory.java | 8 - .../moe/kyokobot/koe/codec/OpusCodecInfo.java | 11 ++ .../codec/netty/NettyFramePollerFactory.java | 29 --- .../koe/codec/netty/NettyH264FramePoller.java | 95 ---------- .../koe/codec/netty/NettyOpusFramePoller.java | 89 ---------- .../media/VideoFrameProvider.java | 51 ++++++ .../koe/internal/MediaConnectionImpl.java | 33 ++-- .../koe/media/AudioFrameProvider.java | 54 ++++++ .../kyokobot/koe/media/MediaFrameHandler.java | 4 - .../koe/media/MediaFrameProvider.java | 51 ------ .../koe/media/OpusAudioFrameProvider.java | 156 ----------------- .../koe/poller/AbstractFramePoller.java | 124 +++++++++++++ .../koe/poller/AbstractOpusFramePoller.java | 165 ++++++++++++++++++ .../koe/poller/FramePollerFactory.java | 9 + .../poller/netty/NettyFramePollerFactory.java | 20 +++ .../poller/netty/NettyOpusFramePoller.java | 28 +++ ext-udpqueue/README.md | 10 +- .../udpqueue/UdpQueueFramePollerFactory.java | 32 ---- .../udpqueue/UdpQueueOpusFramePoller.java | 84 --------- .../udpqueue/QueueManagerPool.java | 12 +- .../udpqueue/UdpQueueFramePollerFactory.java | 28 +++ .../udpqueue/UdpQueueOpusFramePoller.java | 51 ++++++ .../moe/kyokobot/koe/testbot/TestBot.java | 26 ++- .../koe/testbot/UdpQueueTestBotLauncher.java | 14 +- 32 files changed, 746 insertions(+), 640 deletions(-) delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/FramePoller.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/media/VideoFrameProvider.java create mode 100644 core/src/main/java/moe/kyokobot/koe/media/AudioFrameProvider.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/media/MediaFrameHandler.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java delete mode 100644 core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java create mode 100644 core/src/main/java/moe/kyokobot/koe/poller/AbstractFramePoller.java create mode 100644 core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java create mode 100644 core/src/main/java/moe/kyokobot/koe/poller/FramePollerFactory.java create mode 100644 core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java create mode 100644 core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java delete mode 100644 ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueFramePollerFactory.java delete mode 100644 ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueOpusFramePoller.java rename ext-udpqueue/src/main/java/moe/kyokobot/koe/{codec => poller}/udpqueue/QueueManagerPool.java (88%) create mode 100644 ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueFramePollerFactory.java create mode 100644 ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java diff --git a/MIGRATION.md b/MIGRATION.md index 1a36a9a..a314c89 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -5,28 +5,27 @@ 1. The public constructor has been removed. Use `VoiceServerInfo#builder()` to create instances of this class instead. 2. It's now required to pass `channelId` (the ID of the voice channel), as it is used as a MLS group identifier while using DAVE E2E encryption. -## Codec API Changes +## Refactoring of the poller / provider API -### Overview +Frame poller classes moved from `moe.kyokobot.koe.codec` to `moe.kyokobot.koe.poller`. +Transport implementations live under subpackages such as `moe.kyokobot.koe.poller.netty` +and `moe.kyokobot.koe.poller.udpqueue`. -The old `Codec` class has been replaced with two new classes: -- **`CodecInfo`**: Immutable codec capabilities (what a codec CAN do) -- **`CodecInstance`**: Session-specific codec instance with negotiated payload types (what a codec IS doing) +### Class Changes -### Codec Class Names +- `OpusFramePoller` has been replaced by `AbstractOpusFramePoller`. +- Netty and udpqueue opus pollers are now transport adapters on top of `AbstractOpusFramePoller`. +- `FramePollerFactory` moved to `moe.kyokobot.koe.poller.FramePollerFactory`. -All codec classes have been renamed with an `Info` suffix: +### Provider API and Hot-Swap Behavior -| Old (2.x) | New (3.0) | -|----------------------|--------------------------| -| `OpusCodec.INSTANCE` | `OpusCodecInfo.INSTANCE` | -| `H264Codec.INSTANCE` | `H264CodecInfo.INSTANCE` | -| `VP8Codec.INSTANCE` | `VP8CodecInfo.INSTANCE` | -| `VP9Codec.INSTANCE` | `VP9CodecInfo.INSTANCE` | +- Opus pollers resolve providers from `MediaConnection` on each poll iteration. +- Replacing providers via `setAudioSender(...)` / `setVideoSender(...)` does not require poller recreation. +- `OpusAudioFrameProvider` is no longer required to manage silence/speaking transitions for opus transport behavior. -## MediaFrameProvider Changes +### MediaFrameProvider → AudioFrameProvider -The `MediaFrameProvider` interface now uses `CodecInstance` instead of `Codec`: +`MediaFrameProvider` has been replaced by `AudioFrameProvider`. The interface is now focused on audio; codec selection and timestamp handling are handled by the poller. **Old (2.x):** ```java @@ -38,8 +37,94 @@ public interface MediaFrameProvider { **New (3.0):** ```java -public interface MediaFrameProvider { - boolean canSendFrame(CodecInstance codec); - boolean retrieve(CodecInstance codec, ByteBuf buf, IntReference timestamp); +public interface AudioFrameProvider { + void onCodecChanged(CodecInstance codec); + void dispose(); + boolean canProvide(); + boolean provideFrame(ByteBuf buf); } ``` + +- `setAudioSender` / `setVideoSender` and the corresponding getters now use `AudioFrameProvider` (same type for both in 3.0). +- Implementations must provide frames when `canProvide()` is true by writing to the buffer in `provideFrame(ByteBuf)` and returning whether a frame was written. Silence and speaking state are handled by the opus poller. + +### Codec-aware providers: `onCodecChanged(CodecInstance)` + +Providers are notified of the current codec so they can reject unsupported formats without racy startup behavior. + +**Guarantees:** + +- `onCodecChanged(CodecInstance codec)` is called **before** the first `canProvide()` for a provider attached via `setAudioSender` / `setVideoSender`. +- It is called again whenever the codec is changed via `setAudioCodec` / `setVideoCodec` while that provider is attached. + +**Expected behavior:** + +- In `onCodecChanged`, update any internal codec-dependent state (e.g. store the current `CodecInstance`). +- If the provider cannot supply data for the current codec, return `false` from `canProvide()` until the codec changes to a supported one. The poller will not call `provideFrame` when `canProvide()` is false. + +In practice the audio codec is always Opus, so implementing codec checks is not strictly required for audio-only bots. This API is for completeness and for future or experimental video/other codec support (see [Experimental package](#experimental-package)). + +**Example (Opus-only provider):** +```java +private boolean isOpus; + +@Override +public void onCodecChanged(CodecInstance codec) { + this.isOpus = OpusCodecInfo.isInstanceOf(codec); +} + +@Override +public boolean canProvide() { + if (!isOpus) return false; + return /* ... has opus frame ... */; +} +``` + +### ext-udpqueue Updates + +- `UdpQueueFramePollerFactory` moved to `moe.kyokobot.koe.poller.udpqueue`. +- `UdpQueueOpusFramePoller` now follows the shared opus poller hierarchy and uses `CodecInstance`. +- `UdpQueueFramePollerFactory` no longer creates its own `QueueManagerPool`. + You must create and manage `QueueManagerPool` yourself and pass it to the factory. + This is a breaking change. + +**Old (2.x):** +```java +.setFramePollerFactory(new UdpQueueFramePollerFactory()) +``` + +**New (3.x):** +```java +var queuePool = new QueueManagerPool( + Runtime.getRuntime().availableProcessors(), + UdpQueueFramePollerFactory.DEFAULT_BUFFER_DURATION +); + +.setFramePollerFactory(new UdpQueueFramePollerFactory(queuePool)); + +// On shutdown: +queuePool.close(); +``` + +## Codec API Changes + +### Overview + +The old `Codec` class has been replaced with two new classes: +- **`CodecInfo`**: Immutable codec capabilities (what a codec CAN do) +- **`CodecInstance`**: Session-specific codec instance with negotiated payload types (what a codec IS doing) + +### Codec Class Names + +All codec classes have been renamed with an `Info` suffix: + +| Old (2.x) | New (3.0) | +|----------------------|--------------------------| +| `OpusCodec.INSTANCE` | `OpusCodecInfo.INSTANCE` | +| `H264Codec.INSTANCE` | `H264CodecInfo.INSTANCE` | +| `VP8Codec.INSTANCE` | `VP8CodecInfo.INSTANCE` | +| `VP9Codec.INSTANCE` | `VP9CodecInfo.INSTANCE` | + +## Experimental package + +The package `moe.kyokobot.koe.experimental` is reserved for APIs that are not yet stable. Types and members in this package may change or be removed in any **minor** release (e.g. 3.1, 3.2). Do not depend on them if you need binary compatibility across minor versions. The rest of the public API follows the [binary compatibility policy](README.md#binary-compatibility) described in the README. diff --git a/README.md b/README.md index 6312558..f5449be 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Tiny, minimal dependency and embeddable library implementing Discord media serve [Get it on JitPack](https://jitpack.io/#moe.kyokobot.koe/core) +### Versioning and stability policy + +Koe follows [semantic versioning](https://semver.org/). API/ABI stability is defined as follows: + +- 🟢 **Public API** (all packages except `.experimental` and `.internal`) is guaranteed to be stable and is only extended when needed. Minor and patch releases preserve binary compatibility for the public API. +- 🚧 **Experimental API** (`moe.kyokobot.koe.experimental`) - APIs in this package are subject to change or removal in any **minor** release (e.g. 3.1, 3.2). Use only if you can tolerate breaking changes between minor versions. +- 🔒 **Internal API** (`moe.kyokobot.koe.internal`) - For internal use only. Not part of the public API; may change or break in any release, including **patch** releases. + Example: ```groovy @@ -20,29 +28,30 @@ dependencies { `VERSION` can be either a tag or a git commit hash. -#### Dependencies +### Dependencies - Netty - slf4j -- Java 11+ (could be backported to Java 8 with minor code changes) +- Java 11+ -#### Features +### Features - Supports voice gateway v4, v5 and v8. - Easily extendable for stuff such as support for codecs other than Opus or video sending, if Discord ever decides to support it on bots. - Experimental video support. - Basic RTCP support for measuring packet loss and other stuff. -#### Non-goals / won't do +### Non-goals / won't do - Encoding - Koe only implements voice server communication, not voice handling itself, so it only accepts Opus frames and you have set up an encoder yourself, use [lavaplayer](https://github.com/sedmelluq/lavaplayer), libav/ffmpeg or anything else. - Voice receiving support - [it's not supported by Discord anyway](https://github.com/discordapp/discord-api-docs/issues/808#issuecomment-458863743), although someone could implement it by registering hooks. -#### Extensions +### Extensions - [UDP-Queue](https://github.com/KyokoBot/koe/tree/master/ext-udpqueue) -#### Credits +### Credits +[LavaLink team](https://github.com/lavalink-devs) for being the main consumer of Koe and providing most feedback and improvements. [@TheAkio](https://github.com/TheAkio) for name idea. Koe includes modified/stripped-down parts based on following open-source projects: diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java index fbbc979..e41e0fc 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java @@ -6,7 +6,7 @@ import io.netty.channel.socket.SocketChannel; import moe.kyokobot.koe.codec.CodecRegistry; import moe.kyokobot.koe.codec.DefaultCodecRegistry; -import moe.kyokobot.koe.codec.FramePollerFactory; +import moe.kyokobot.koe.poller.FramePollerFactory; import moe.kyokobot.koe.gateway.GatewayVersion; import org.jetbrains.annotations.NotNull; diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java index 419da7d..f9b1ab1 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java @@ -14,8 +14,8 @@ import io.netty.channel.socket.nio.NioSocketChannel; import moe.kyokobot.koe.codec.CodecRegistry; import moe.kyokobot.koe.codec.DefaultCodecRegistry; -import moe.kyokobot.koe.codec.FramePollerFactory; -import moe.kyokobot.koe.codec.netty.NettyFramePollerFactory; +import moe.kyokobot.koe.poller.FramePollerFactory; +import moe.kyokobot.koe.poller.netty.NettyFramePollerFactory; import moe.kyokobot.koe.gateway.GatewayVersion; import org.jetbrains.annotations.NotNull; diff --git a/core/src/main/java/moe/kyokobot/koe/MediaConnection.java b/core/src/main/java/moe/kyokobot/koe/MediaConnection.java index 0ecb629..aa14203 100644 --- a/core/src/main/java/moe/kyokobot/koe/MediaConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/MediaConnection.java @@ -1,20 +1,21 @@ package moe.kyokobot.koe; -import moe.kyokobot.koe.media.MediaFrameProvider; import moe.kyokobot.koe.codec.CodecInfo; import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.experimental.media.VideoFrameProvider; import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.handler.ConnectionHandler; +import moe.kyokobot.koe.media.AudioFrameProvider; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.Closeable; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; public interface MediaConnection extends Closeable { /** * Connects to Discord voice server using specified info. + * * @param info Discord voice server connection information * @return future which completes once Koe is connected to both voice gateway and can successfully * send UDP packets, so you can send audio data. @@ -23,6 +24,7 @@ public interface MediaConnection extends Closeable { /** * Stops polling media frames, disconnects from the gateway and cleans everything up. + * * @see #connect(VoiceServerInfo) */ void disconnect(); @@ -36,10 +38,10 @@ public interface MediaConnection extends Closeable { KoeOptions getOptions(); @Nullable - MediaFrameProvider getAudioSender(); + AudioFrameProvider getAudioSender(); @Nullable - MediaFrameProvider getVideoSender(); + VideoFrameProvider getVideoSender(); long getGuildId(); @@ -51,7 +53,7 @@ public interface MediaConnection extends Closeable { ConnectionHandler getConnectionHandler(); - void setAudioSender(@Nullable MediaFrameProvider sender); + void setAudioSender(@Nullable AudioFrameProvider sender); /** * Sets the audio codec instance for this connection. @@ -77,11 +79,12 @@ default void setAudioCodec(@NotNull CodecInfo info) { /** * Stops polling audio frames. + * * @see MediaConnection#startAudioFramePolling() */ void stopAudioFramePolling(); - void setVideoSender(@Nullable MediaFrameProvider sender); + void setVideoSender(@Nullable VideoFrameProvider sender); /** * Sets the video codec instance for this connection. @@ -107,6 +110,7 @@ default void setVideoCodec(@NotNull CodecInfo info) { /** * Stops polling video frames. + * * @see MediaConnection#startAudioFramePolling() */ void stopVideoFramePolling(); @@ -117,6 +121,7 @@ default void setVideoCodec(@NotNull CodecInfo info) { /** * Sends speaking state notification to the gateway. + * * @param mask new speaking state */ void updateSpeakingState(int mask); diff --git a/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java deleted file mode 100644 index ac6dc8c..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/AbstractFramePoller.java +++ /dev/null @@ -1,26 +0,0 @@ -package moe.kyokobot.koe.codec; - -import io.netty.buffer.ByteBufAllocator; -import io.netty.channel.EventLoopGroup; -import moe.kyokobot.koe.MediaConnection; -import org.jetbrains.annotations.NotNull; - -public abstract class AbstractFramePoller implements FramePoller { - protected final MediaConnection connection; - protected final CodecInstance codec; - protected final ByteBufAllocator allocator; - protected final EventLoopGroup eventLoop; - protected boolean polling = false; - - public AbstractFramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { - this.codec = codec; - this.connection = connection; - this.allocator = connection.getOptions().getByteBufAllocator(); - this.eventLoop = connection.getOptions().getEventLoopGroup(); - } - - @Override - public boolean isPolling() { - return polling; - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java index 7bda349..66b80bf 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/CodecInfo.java @@ -3,6 +3,8 @@ import moe.kyokobot.koe.internal.json.JsonObject; import org.jetbrains.annotations.NotNull; +import java.util.Objects; + /** * Immutable codec capabilities and metadata. * Represents what a codec CAN do, not what it IS doing in a session. diff --git a/core/src/main/java/moe/kyokobot/koe/codec/FramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/FramePoller.java deleted file mode 100644 index 1076c90..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/FramePoller.java +++ /dev/null @@ -1,9 +0,0 @@ -package moe.kyokobot.koe.codec; - -public interface FramePoller { - boolean isPolling(); - - void start(); - - void stop(); -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java deleted file mode 100644 index da5d78e..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/FramePollerFactory.java +++ /dev/null @@ -1,8 +0,0 @@ -package moe.kyokobot.koe.codec; - -import moe.kyokobot.koe.MediaConnection; - -@FunctionalInterface -public interface FramePollerFactory { - FramePoller createFramePoller(CodecInstance codec, MediaConnection connection); -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java b/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java index 4f56d28..a5d8241 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/OpusCodecInfo.java @@ -1,6 +1,7 @@ package moe.kyokobot.koe.codec; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; /** * Opus audio codec information. @@ -14,4 +15,14 @@ public class OpusCodecInfo extends CodecInfo { private OpusCodecInfo() { super("opus", CodecType.AUDIO, DEFAULT_PAYLOAD_TYPE, (byte) 0, 1000); } + + /** + * Returns whether the given codec instance is Opus. + * + * @param instance the codec instance to check, may be null + * @return true if instance is non-null and its codec is Opus + */ + public static boolean isInstanceOf(@Nullable CodecInstance instance) { + return instance != null && INSTANCE.equals(instance.getInfo()); + } } diff --git a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java deleted file mode 100644 index 8e1d3e1..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyFramePollerFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -package moe.kyokobot.koe.codec.netty; - -import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.*; -import org.jetbrains.annotations.Nullable; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.BiFunction; - -public class NettyFramePollerFactory implements FramePollerFactory { - private final Map> codecMap; - - public NettyFramePollerFactory() { - codecMap = new HashMap<>(); - codecMap.put("opus", NettyOpusFramePoller::new); - codecMap.put("H264", NettyH264FramePoller::new); - } - - @Override - @Nullable - public FramePoller createFramePoller(CodecInstance codec, MediaConnection connection) { - var constructor = codecMap.get(codec.getName()); - if (constructor != null) { - return constructor.apply(codec, connection); - } - return null; - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java deleted file mode 100644 index 6e6faa9..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyH264FramePoller.java +++ /dev/null @@ -1,95 +0,0 @@ -package moe.kyokobot.koe.codec.netty; - -import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.AbstractFramePoller; -import moe.kyokobot.koe.codec.CodecInstance; -import moe.kyokobot.koe.media.IntReference; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -public class NettyH264FramePoller extends AbstractFramePoller { - private static final Logger logger = LoggerFactory.getLogger(NettyH264FramePoller.class); - /** - * Delay between frame polling attempts. - */ - private static final int FRAME_RATE = 1000 / 30; - - public NettyH264FramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { - super(codec, connection); - } - - /** - * Last frame time in ms. - */ - private long lastFrame = 0; - - - /** - * Current frame timestamp. - */ - private final IntReference timestamp = new IntReference(); - - @Override - public void start() { - if (this.polling) { - throw new IllegalStateException("Polling already started!"); - } - - this.polling = true; - this.lastFrame = System.currentTimeMillis(); - eventLoop.execute(this::pollFrame); - } - - @Override - public void stop() { - this.polling = false; - } - - private void pollFrame() { - if (!this.polling) { - return; - } - - boolean pollNext = false; - try { - do { - var handler = connection.getConnectionHandler(); - var sender = connection.getVideoSender(); - - if (sender != null && handler != null && sender.canSendFrame(codec)) { - var buf = allocator.buffer(); - int start = buf.writerIndex(); - pollNext = sender.retrieve(codec, buf, timestamp); - int len = buf.writerIndex() - start; - if (len != 0) { - handler.sendFrame(codec.getPayloadType(), timestamp.get(), buf, len, true); - } - buf.release(); - } - } while (pollNext); - } catch (Exception e) { - logger.error("Sending frame failed", e); - } - - long frameDelay = FRAME_RATE - (System.currentTimeMillis() - lastFrame); - - if (frameDelay > 0) { - eventLoop.schedule(this::loop, frameDelay, TimeUnit.MILLISECONDS); - } else { - loop(); - } - } - - private void loop() { - if (System.currentTimeMillis() < lastFrame + 60) { - lastFrame += FRAME_RATE; - } else { - lastFrame = System.currentTimeMillis(); - } - - pollFrame(); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java b/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java deleted file mode 100644 index b1560f0..0000000 --- a/core/src/main/java/moe/kyokobot/koe/codec/netty/NettyOpusFramePoller.java +++ /dev/null @@ -1,89 +0,0 @@ -package moe.kyokobot.koe.codec.netty; - -import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.AbstractFramePoller; -import moe.kyokobot.koe.codec.CodecInstance; -import moe.kyokobot.koe.media.IntReference; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -public class NettyOpusFramePoller extends AbstractFramePoller { - private static final Logger logger = LoggerFactory.getLogger(NettyOpusFramePoller.class); - - public NettyOpusFramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { - super(codec, connection); - } - - /** - * Last frame time in ms. - */ - private long lastFrame = 0; - - /** - * Current frame timestamp. - */ - private final IntReference timestamp = new IntReference(); - - @Override - public void start() { - if (this.polling) { - throw new IllegalStateException("Polling already started!"); - } - - this.polling = true; - this.lastFrame = System.currentTimeMillis(); - eventLoop.execute(this::pollFrame); - } - - @Override - public void stop() { - this.polling = false; - } - - private void pollFrame() { - if (!this.polling) { - return; - } - - try { - var handler = connection.getConnectionHandler(); - var sender = connection.getAudioSender(); - - // ugly but it's the hottest path in Koe and Java is a shit language. - if (sender != null && handler != null && sender.canSendFrame(codec)) { - var buf = allocator.buffer(); - int start = buf.writerIndex(); - // opus codec doesn't need framing, we don't handle multiple packet cases. - sender.retrieve(codec, buf, timestamp); - int len = buf.writerIndex() - start; - if (len != 0) { - handler.sendFrame(codec.getPayloadType(), timestamp.get(), buf, len, false); - } - buf.release(); - } - } catch (Exception e) { // get rid of somehow? - logger.error("Sending frame failed", e); - } - - long frameDelay = 20 - (System.currentTimeMillis() - lastFrame); - - if (frameDelay > 0) { - eventLoop.schedule(this::loop, frameDelay, TimeUnit.MILLISECONDS); - } else { - loop(); - } - } - - private void loop() { - if (System.currentTimeMillis() < lastFrame + 60) { - lastFrame += 20; - } else { - lastFrame = System.currentTimeMillis(); - } - - pollFrame(); - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/media/VideoFrameProvider.java b/core/src/main/java/moe/kyokobot/koe/experimental/media/VideoFrameProvider.java new file mode 100644 index 0000000..dd857ed --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/media/VideoFrameProvider.java @@ -0,0 +1,51 @@ +package moe.kyokobot.koe.experimental.media; + +import io.netty.buffer.ByteBuf; +import moe.kyokobot.koe.codec.CodecInstance; + +import org.jetbrains.annotations.NotNull; + +/** + * Base interface for video frame providers. + */ +public interface VideoFrameProvider { + /** + * Notifies this provider of the current codec for the connection. + *

+ * Guarantees: + *

    + *
  • This method is called before the first {@link #canProvide()} invocation for a provider + * instance attached to a connection (e.g. when set via {@code setVideoSender}).
    • + *
  • It is called again whenever the codec is changed via {@code setVideoCodec} + * while this provider is attached.
    • + *
+ * Implementations should update internal codec-dependent state in this callback. If the provider + * cannot supply data in the requested format, it must return {@code false} from {@link #canProvide()} + * until the codec changes to a supported one. + * + * @param codec the current codec instance for this connection + */ + void onCodecChanged(@NotNull CodecInstance codec); + + /** + * Called when this {@link VideoFrameProvider} should clean up it's event handlers and etc. + */ + void dispose(); + + /** + * @return If true, this provider has media data available and can provide frames. This is used by the + * frame poller to determine if it should attempt to retrieve frames from this provider. + * Must return false if the provider cannot supply data for the current codec (see {@link #onCodecChanged}). + */ + boolean canProvide(); + + /** + * Retrieves a media frame and writes it to the provided {@link ByteBuf}. The buffer is guaranteed to be empty + * and have enough capacity for the frame. The provider should write the frame data to the buffer and return true + * if a frame was provided, or return false if no frame is available (e.g. end of stream). + * + * @param buf The buffer to write the frame data to. + * @return true if a frame was provided and written to the buffer, false if no frame is available. + */ + boolean provideFrame(ByteBuf buf); +} diff --git a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java index 957fa47..c5ed91d 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java @@ -3,12 +3,13 @@ import moe.kyokobot.koe.*; import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.codec.CodecType; -import moe.kyokobot.koe.codec.FramePoller; import moe.kyokobot.koe.codec.OpusCodecInfo; +import moe.kyokobot.koe.experimental.media.VideoFrameProvider; import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.gateway.MediaValve; import moe.kyokobot.koe.handler.ConnectionHandler; -import moe.kyokobot.koe.media.MediaFrameProvider; +import moe.kyokobot.koe.media.AudioFrameProvider; +import moe.kyokobot.koe.poller.AbstractFramePoller; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -29,10 +30,10 @@ public class MediaConnectionImpl implements MediaConnection { private VoiceServerInfo info; private CodecInstance audioCodec; private CodecInstance videoCodec; - private FramePoller audioPoller; - private FramePoller videoPoller; - private MediaFrameProvider audioSender; - private MediaFrameProvider videoSender; + private AbstractFramePoller audioPoller; + private AbstractFramePoller videoPoller; + private AudioFrameProvider audioSender; + private VideoFrameProvider videoSender; private DAVEManager daveManager; public MediaConnectionImpl(@NotNull KoeClientImpl client, long guildId) { @@ -105,13 +106,13 @@ public KoeOptions getOptions() { @Override @Nullable - public MediaFrameProvider getAudioSender() { + public AudioFrameProvider getAudioSender() { return audioSender; } @Override @Nullable - public MediaFrameProvider getVideoSender() { + public VideoFrameProvider getVideoSender() { return videoSender; } @@ -138,11 +139,14 @@ public ConnectionHandler getConnectionHandler() { } @Override - public void setAudioSender(@Nullable MediaFrameProvider sender) { + public void setAudioSender(@Nullable AudioFrameProvider sender) { if (this.audioSender != null) { this.audioSender.dispose(); } this.audioSender = sender; + if (sender != null && this.audioCodec != null) { + sender.onCodecChanged(this.audioCodec); + } } @Override @@ -156,6 +160,9 @@ public void setAudioCodec(@NotNull CodecInstance audioCodec) { this.audioCodec = audioCodec; this.audioPoller = client.getOptions().getFramePollerFactory().createFramePoller(audioCodec, this); + if (this.audioSender != null) { + this.audioSender.onCodecChanged(audioCodec); + } if (wasPolling) { this.startAudioFramePolling(); @@ -181,11 +188,14 @@ public void stopAudioFramePolling() { } @Override - public void setVideoSender(@Nullable MediaFrameProvider sender) { + public void setVideoSender(@Nullable VideoFrameProvider sender) { if (this.videoSender != null) { this.videoSender.dispose(); } this.videoSender = sender; + if (sender != null && this.videoCodec != null) { + sender.onCodecChanged(this.videoCodec); + } } @Override @@ -206,6 +216,9 @@ public void setVideoCodec(@Nullable CodecInstance videoCodec) { this.videoCodec = videoCodec; this.videoPoller = client.getOptions().getFramePollerFactory().createFramePoller(videoCodec, this); + if (this.videoSender != null) { + this.videoSender.onCodecChanged(videoCodec); + } if (wasPolling) { this.startVideoFramePolling(); diff --git a/core/src/main/java/moe/kyokobot/koe/media/AudioFrameProvider.java b/core/src/main/java/moe/kyokobot/koe/media/AudioFrameProvider.java new file mode 100644 index 0000000..eac2a73 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/media/AudioFrameProvider.java @@ -0,0 +1,54 @@ +package moe.kyokobot.koe.media; + +import io.netty.buffer.ByteBuf; +import moe.kyokobot.koe.codec.CodecInstance; + +import org.jetbrains.annotations.NotNull; + +/** + * Base interface for audio frame providers. + *

+ * For audio-only usage with the default Opus codec, implementations may omit codec checks. + * This is provided for completeness and for future or experimental codec support. + */ +public interface AudioFrameProvider { + /** + * Notifies this provider of the current codec for the connection. + *

+ * Guarantees: + *

    + *
  • This method is called before the first {@link #canProvide()} invocation for a provider + * instance attached to a connection (e.g. when set via {@code setAudioSender}).
    • + *
  • It is called again whenever the codec is changed via {@code setAudioCodec} + * while this provider is attached.
    • + *
+ * Implementations should update internal codec-dependent state in this callback. If the provider + * cannot supply data in the requested format, it must return {@code false} from {@link #canProvide()} + * until the codec changes to a supported one. + * + * @param codec the current codec instance for this connection + */ + void onCodecChanged(@NotNull CodecInstance codec); + + /** + * Called when this {@link AudioFrameProvider} should clean up it's event handlers and etc. + */ + void dispose(); + + /** + * @return If true, this provider has media data available and can provide frames. This is used by the + * frame poller to determine if it should attempt to retrieve frames from this provider. + * Must return false if the provider cannot supply data for the current codec (see {@link #onCodecChanged}). + */ + boolean canProvide(); + + /** + * Retrieves a media frame and writes it to the provided {@link ByteBuf}. The buffer is guaranteed to be empty + * and have enough capacity for the frame. The provider should write the frame data to the buffer and return true + * if a frame was provided, or return false if no frame is available (e.g. end of stream). + * + * @param buf The buffer to write the frame data to. + * @return true if a frame was provided and written to the buffer, false if no frame is available. + */ + boolean provideFrame(ByteBuf buf); +} diff --git a/core/src/main/java/moe/kyokobot/koe/media/MediaFrameHandler.java b/core/src/main/java/moe/kyokobot/koe/media/MediaFrameHandler.java deleted file mode 100644 index c379aa2..0000000 --- a/core/src/main/java/moe/kyokobot/koe/media/MediaFrameHandler.java +++ /dev/null @@ -1,4 +0,0 @@ -package moe.kyokobot.koe.media; - -public interface MediaFrameHandler { -} diff --git a/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java b/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java deleted file mode 100644 index 6317620..0000000 --- a/core/src/main/java/moe/kyokobot/koe/media/MediaFrameProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -package moe.kyokobot.koe.media; - -import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.codec.CodecInstance; - -/** - * Base interface for media frame providers. Note that Koe doesn't handle stuff such as speaking state, silent frames - * or etc., these are implemented by codec-specific frame provider classes. - * - * @see OpusAudioFrameProvider for Opus audio codec specific implementation that handles speaking state and etc. - */ -public interface MediaFrameProvider { - /** - * Called when this {@link MediaFrameProvider} should clean up it's event handlers and etc. - */ - void dispose(); - - /** - * @return Frame interval between polling attempts. - */ - int getFrameInterval(); - - /** - * Sets delay between polling attempts by frame poller. - */ - void setFrameInterval(int interval); - - /** - * @return If true, Koe will request media data for given {@link CodecInstance} by - * calling {@link #retrieve(CodecInstance, ByteBuf, moe.kyokobot.koe.media.IntReference)} method. - */ - boolean canSendFrame(CodecInstance codec); - - /** - * If {@link #canSendFrame(CodecInstance)} returns true, Koe will attempt to retrieve an media frame encoded with specified - * {@link CodecInstance} type, by calling this method with target {@link ByteBuf} where the data should be written to. - * Do not call {@link ByteBuf#release()} - memory management is already handled by Koe itself. In case if no - * data gets written to the buffer, audio packet won't be sent. - *

- * Do not let this method block - all data should be queued on another thread or pre-loaded in - * memory - otherwise it will very likely have significant impact on application performance. - * - * @param codec {@link CodecInstance} type this handler was registered with. - * @param buf {@link ByteBuf} the buffer where the media data should be written to. - * @param timestamp {@link IntReference} reference to current frame timestamp, which must be updated with - * timestamp of written frame. - * @return If true, Koe will immediately attempt to poll a next frame, this is meant for video transmissions. - */ - boolean retrieve(CodecInstance codec, ByteBuf buf, IntReference timestamp); - -} diff --git a/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java b/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java deleted file mode 100644 index 0e93840..0000000 --- a/core/src/main/java/moe/kyokobot/koe/media/OpusAudioFrameProvider.java +++ /dev/null @@ -1,156 +0,0 @@ -package moe.kyokobot.koe.media; - -import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.KoeEventAdapter; -import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.CodecInstance; -import moe.kyokobot.koe.codec.OpusCodecInfo; -import moe.kyokobot.koe.gateway.SpeakingFlags; - -import java.util.Objects; - -/** - * Implementation of {@link MediaFrameProvider} which automatically takes care of - * checking codec type, sending silent frames and updating speaking state. - */ -public abstract class OpusAudioFrameProvider implements MediaFrameProvider { - private static final int SILENCE_FRAME_COUNT = 5; - private final MediaConnection connection; - private final Op12HackListener hackListener; - - private int counter; - private long lastFramePolled = 0; - private boolean lastProvide = false; - private boolean lastSpeaking = false; - private boolean speaking = false; - private int speakingMask = SpeakingFlags.NORMAL; - - public OpusAudioFrameProvider(MediaConnection connection) { - this.connection = Objects.requireNonNull(connection); - this.hackListener = new Op12HackListener(); - this.connection.registerListener(this.hackListener); - } - - public int getSpeakingMask() { - return speakingMask; - } - - public void setSpeakingMask(int speakingMask) { - this.speakingMask = speakingMask; - } - - @Override - public int getFrameInterval() { - return OpusCodecInfo.FRAME_DURATION; - } - - @Override - public void setFrameInterval(int interval) { - throw new UnsupportedOperationException("Only 20ms frames are supported."); - } - - @Override - public final boolean canSendFrame(CodecInstance codec) { - if (!"opus".equalsIgnoreCase(codec.getName())) { - return false; - } - - if (counter > 0) { - return true; - } - - boolean provide = canProvide(); - - if (lastProvide != provide) { - lastProvide = provide; - if (!provide) { - counter = SILENCE_FRAME_COUNT; - return true; - } - } - - return provide; - } - - @Override - public final boolean retrieve(CodecInstance codec, ByteBuf buf, IntReference timestamp) { - if (!"opus".equalsIgnoreCase(codec.getName())) { - return false; - } - - if (counter > 0) { - counter--; - buf.writeBytes(OpusCodecInfo.SILENCE_FRAME); - - if (speaking) { - setSpeaking(false); - } - - timestamp.add(960); - return false; - } - - int startIndex = buf.writerIndex(); - retrieveOpusFrame(buf); - boolean written = buf.writerIndex() != startIndex; - - if (written && !speaking) { - setSpeaking(true); - } - - if (!written) { - counter = SILENCE_FRAME_COUNT; - } - - long now = System.currentTimeMillis(); - boolean changeTalking = (now - lastFramePolled) > OpusCodecInfo.FRAME_DURATION; - lastFramePolled = now; - if (changeTalking) { - setSpeaking(written); - } - - timestamp.add(960); - return false; - } - - private void setSpeaking(boolean state) { - this.speaking = state; - if (this.speaking != this.lastSpeaking) { - this.lastSpeaking = state; - - connection.updateSpeakingState(state ? this.speakingMask : 0); - } - } - - private class Op12HackListener extends KoeEventAdapter { - @Override - public void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { - if (speaking) { - connection.updateSpeakingState(speakingMask); - } - } - } - - @Override - public void dispose() { - this.connection.unregisterListener(this.hackListener); - } - - /** - * Called every time Opus frame poller tries to retrieve an Opus audio frame. - * - * @return If this method returns true, Koe will attempt to retrieve an Opus audio frame. - */ - public abstract boolean canProvide(); - - /** - * If {@link #canProvide()} returns true, this method will attempt to retrieve an Opus audio frame. - *

- * This method must not block, otherwise it might cause severe performance issues, due to event loop thread - * getting blocked, therefore it's recommended to load all data before or in parallel, not when Koe frame poller - * calls this method. If no data gets written, the frame won't be sent. - * - * @param targetBuffer the target {@link ByteBuf} audio data should be written to. - */ - public abstract void retrieveOpusFrame(ByteBuf targetBuffer); -} diff --git a/core/src/main/java/moe/kyokobot/koe/poller/AbstractFramePoller.java b/core/src/main/java/moe/kyokobot/koe/poller/AbstractFramePoller.java new file mode 100644 index 0000000..eed547f --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/poller/AbstractFramePoller.java @@ -0,0 +1,124 @@ +package moe.kyokobot.koe.poller; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.channel.EventLoopGroup; +import moe.kyokobot.koe.MediaConnection; +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractFramePoller implements AutoCloseable { + protected final MediaConnection connection; + protected final EventLoopGroup eventLoop; + protected final ByteBufAllocator allocator; + protected final AtomicBoolean polling = new AtomicBoolean(false); + + private volatile long lastFrameTime = 0L; + private volatile long totalFrames = 0L; + private volatile long droppedFrames = 0L; + + public AbstractFramePoller(@NotNull MediaConnection connection) { + this.connection = connection; + this.allocator = connection.getOptions().getByteBufAllocator(); + this.eventLoop = connection.getOptions().getEventLoopGroup(); + } + + public boolean isPolling() { + return polling.get(); + } + + public void start() { + if (!polling.compareAndSet(false, true)) { + throw new IllegalStateException("Polling already started!"); + } + lastFrameTime = System.nanoTime(); + scheduleNext(); + } + + public void stop() { + polling.set(false); + } + + protected abstract long getFrameIntervalNanos(); + + protected abstract boolean pollAndSend(); + + // Thread-safety note: poll() is always scheduled from within a running poll() (or from start() exactly once), + // and all scheduling targets the same single-threaded Netty EventLoop. This means poll() executes strictly + // sequentially — there is never more than one poll() task enqueued at a time, so no concurrent execution + // is possible and no additional locking is needed here. + private void scheduleNext() { + if (!polling.get()) { + return; + } + + long now = System.nanoTime(); + long elapsed = now - lastFrameTime; + long frameInterval = getFrameIntervalNanos(); + long delay = frameInterval - elapsed; + + if (delay < 0L) { + long missedFrames = -(delay / frameInterval); + if (missedFrames > 3L) { + lastFrameTime = now; + } else { + lastFrameTime = safeAdd(lastFrameTime, frameInterval); + } + delay = 0L; + droppedFrames = saturatingAdd(droppedFrames, missedFrames); + } else { + lastFrameTime = safeAdd(lastFrameTime, frameInterval); + } + + if (delay > 0L) { + eventLoop.schedule(this::poll, delay, TimeUnit.NANOSECONDS); + } else { + eventLoop.execute(this::poll); + } + } + + private void poll() { + if (!polling.get()) { + return; + } + + try { + boolean sent = pollAndSend(); + if (sent) { + totalFrames = saturatingAdd(totalFrames, 1L); + } + } catch (Exception ignored) { + // Keep polling even when transport/provider raises. + } + + scheduleNext(); + } + + private static long saturatingAdd(long left, long right) { + if (right > 0L && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + if (right < 0L && left < Long.MIN_VALUE - right) { + return Long.MIN_VALUE; + } + return left + right; + } + + private static long safeAdd(long left, long right) { + return saturatingAdd(left, right); + } + + public long getTotalFrames() { + return totalFrames; + } + + public long getDroppedFrames() { + return droppedFrames; + } + + @Override + public void close() { + // Optional override by subclasses. + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java b/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java new file mode 100644 index 0000000..0b63b59 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java @@ -0,0 +1,165 @@ +package moe.kyokobot.koe.poller; + +import io.netty.buffer.ByteBuf; +import moe.kyokobot.koe.KoeEventAdapter; +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.OpusCodecInfo; +import moe.kyokobot.koe.gateway.SpeakingFlags; +import moe.kyokobot.koe.media.AudioFrameProvider; +import moe.kyokobot.koe.media.IntReference; + +public abstract class AbstractOpusFramePoller extends AbstractFramePoller { + private static final long FRAME_INTERVAL_NANOS = 20_000_000L; + private static final int RTP_TIMESTAMP_INCREMENT = 960; + private static final int SILENCE_FRAME_COUNT = 5; + + protected final CodecInstance codec; + + private final IntReference timestamp = new IntReference(); + private final Op12HackListener hackListener; + + private int silenceCounter = 0; + private boolean lastProvide = false; + private boolean lastSpeaking = false; + private boolean speaking = false; + private int speakingMask = SpeakingFlags.NORMAL; + + protected AbstractOpusFramePoller(MediaConnection connection, CodecInstance codec) { + super(connection); + + if (!(codec.getInfo() instanceof OpusCodecInfo)) { + throw new IllegalArgumentException("Expected an Opus codec, got " + codec.getName()); + } + + this.codec = codec; + this.hackListener = new Op12HackListener(); + this.connection.registerListener(this.hackListener); + } + + public int getSpeakingMask() { + return speakingMask; + } + + public void setSpeakingMask(int speakingMask) { + this.speakingMask = speakingMask; + } + + @Override + protected long getFrameIntervalNanos() { + return FRAME_INTERVAL_NANOS; + } + + @Override + protected boolean pollAndSend() { + int attempts = getPollsPerTick(); + if (attempts <= 0) { + return false; + } + + boolean sentAny = false; + for (int i = 0; i < attempts; i++) { + if (!pollSingleFrame()) { + break; + } + sentAny = true; + } + return sentAny; + } + + protected int getPollsPerTick() { + return 1; + } + + protected AudioFrameProvider resolveProvider() { + return connection.getAudioSender(); + } + + protected abstract boolean canSendFrame(); + + protected abstract void sendFramePayload(ByteBuf buf, int len, int timestamp); + + private boolean pollSingleFrame() { + if (!canSendFrame()) { + return false; + } + + AudioFrameProvider provider = resolveProvider(); + if (provider == null) { + return false; + } + + if (silenceCounter > 0) { + return sendSilenceFrame(); + } + + boolean canProvide = provider.canProvide(); + if (lastProvide && !canProvide) { + silenceCounter = SILENCE_FRAME_COUNT; + } + lastProvide = canProvide; + + if (silenceCounter > 0) { + return sendSilenceFrame(); + } + if (!canProvide) { + return false; + } + + ByteBuf buf = allocator.buffer(); + try { + int start = buf.writerIndex(); + boolean wrote = provider.provideFrame(buf); + int len = buf.writerIndex() - start; + if (wrote && len > 0) { + sendFramePayload(buf, len, timestamp.get()); + timestamp.add(RTP_TIMESTAMP_INCREMENT); + setSpeaking(true); + return true; + } + } finally { + buf.release(); + } + + silenceCounter = SILENCE_FRAME_COUNT; + return false; + } + + private boolean sendSilenceFrame() { + silenceCounter--; + + ByteBuf buf = allocator.buffer(OpusCodecInfo.SILENCE_FRAME.length); + try { + buf.writeBytes(OpusCodecInfo.SILENCE_FRAME); + int len = buf.readableBytes(); + sendFramePayload(buf, len, timestamp.get()); + timestamp.add(RTP_TIMESTAMP_INCREMENT); + setSpeaking(false); + return true; + } finally { + buf.release(); + } + } + + private void setSpeaking(boolean state) { + this.speaking = state; + if (this.speaking != this.lastSpeaking) { + this.lastSpeaking = state; + connection.updateSpeakingState(state ? this.speakingMask : 0); + } + } + + private class Op12HackListener extends KoeEventAdapter { + @Override + public void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { + if (speaking) { + connection.updateSpeakingState(speakingMask); + } + } + } + + @Override + public void close() { + connection.unregisterListener(hackListener); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/poller/FramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/poller/FramePollerFactory.java new file mode 100644 index 0000000..d502c3f --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/poller/FramePollerFactory.java @@ -0,0 +1,9 @@ +package moe.kyokobot.koe.poller; + +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInstance; + +@FunctionalInterface +public interface FramePollerFactory { + AbstractFramePoller createFramePoller(CodecInstance codec, MediaConnection connection); +} diff --git a/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java new file mode 100644 index 0000000..e985fb6 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java @@ -0,0 +1,20 @@ +package moe.kyokobot.koe.poller.netty; + +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.H264CodecInfo; +import moe.kyokobot.koe.codec.OpusCodecInfo; +import moe.kyokobot.koe.poller.AbstractFramePoller; +import moe.kyokobot.koe.poller.FramePollerFactory; +import org.jetbrains.annotations.Nullable; + +public class NettyFramePollerFactory implements FramePollerFactory { + @Override + @Nullable + public AbstractFramePoller createFramePoller(CodecInstance codec, MediaConnection connection) { + if (codec.getInfo() instanceof OpusCodecInfo) { + return new NettyOpusFramePoller(codec, connection); + } + return null; + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java new file mode 100644 index 0000000..17e3963 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java @@ -0,0 +1,28 @@ +package moe.kyokobot.koe.poller.netty; + +import io.netty.buffer.ByteBuf; +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.handler.ConnectionHandler; +import moe.kyokobot.koe.poller.AbstractOpusFramePoller; +import org.jetbrains.annotations.NotNull; + +public class NettyOpusFramePoller extends AbstractOpusFramePoller { + public NettyOpusFramePoller(@NotNull CodecInstance codec, @NotNull MediaConnection connection) { + super(connection, codec); + } + + @Override + protected boolean canSendFrame() { + return connection.getConnectionHandler() != null; + } + + @Override + protected void sendFramePayload(ByteBuf buf, int len, int timestamp) { + var handler = connection.getConnectionHandler(); + if (handler == null) { + return; + } + handler.sendFrame(codec.getPayloadType(), timestamp, buf, len, false); + } +} diff --git a/ext-udpqueue/README.md b/ext-udpqueue/README.md index 9ae186a..336f4d6 100644 --- a/ext-udpqueue/README.md +++ b/ext-udpqueue/README.md @@ -11,7 +11,15 @@ limits GC pressure because of much smaller number of allocations. Just add it to KoeOptions :^) ```java +var queuePool = new QueueManagerPool( + Runtime.getRuntime().availableProcessors(), + UdpQueueFramePollerFactory.DEFAULT_BUFFER_DURATION +); + var Koe = Koe.koe(KoeOptions.builder() - .setFramePollerFactory(new UdpQueueFramePollerFactory()) + .setFramePollerFactory(new UdpQueueFramePollerFactory(queuePool)) .create()); + +// close on shutdown +queuePool.close(); ``` diff --git a/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueFramePollerFactory.java b/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueFramePollerFactory.java deleted file mode 100644 index 3577b48..0000000 --- a/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueFramePollerFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -package moe.kyokobot.koe.codec.udpqueue; - -import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.Codec; -import moe.kyokobot.koe.codec.FramePoller; -import moe.kyokobot.koe.codec.FramePollerFactory; -import moe.kyokobot.koe.codec.OpusCodec; -import org.jetbrains.annotations.Nullable; - -public class UdpQueueFramePollerFactory implements FramePollerFactory { - public static final int DEFAULT_BUFFER_DURATION = 400; - public static final int MAXIMUM_PACKET_SIZE = 4096; - - private final QueueManagerPool pool; - - public UdpQueueFramePollerFactory() { - this(DEFAULT_BUFFER_DURATION, Runtime.getRuntime().availableProcessors()); - } - - public UdpQueueFramePollerFactory(int bufferDuration, int poolSize) { - this.pool = new QueueManagerPool(poolSize, bufferDuration); - } - - @Override - @Nullable - public FramePoller createFramePoller(Codec codec, MediaConnection connection) { - if (OpusCodec.INSTANCE.equals(codec)) { - return new UdpQueueOpusFramePoller(this.pool.getNextWrapper(), connection); - } - return null; - } -} diff --git a/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueOpusFramePoller.java b/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueOpusFramePoller.java deleted file mode 100644 index 92038ab..0000000 --- a/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/UdpQueueOpusFramePoller.java +++ /dev/null @@ -1,84 +0,0 @@ -package moe.kyokobot.koe.codec.udpqueue; - -import moe.kyokobot.koe.MediaConnection; -import moe.kyokobot.koe.codec.AbstractFramePoller; -import moe.kyokobot.koe.codec.OpusCodec; -import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; -import moe.kyokobot.koe.media.IntReference; - -import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; - -public class UdpQueueOpusFramePoller extends AbstractFramePoller { - private final QueueManagerPool.UdpQueueWrapper manager; - private final IntReference timestamp = new IntReference(); - private long lastFrame; - - public UdpQueueOpusFramePoller(QueueManagerPool.UdpQueueWrapper manager, MediaConnection connection) { - super(connection); - this.manager = manager; - } - - @Override - public void start() { - if (this.polling) { - throw new IllegalStateException("Polling already started!"); - } - - this.polling = true; - this.lastFrame = System.currentTimeMillis(); - eventLoop.execute(this::populateQueue); - } - - @Override - public void stop() { - if (this.polling) { - this.polling = false; - } - } - - void populateQueue() { - if (!this.polling || manager == null) { - return; - } - - int remaining = manager.getRemainingCapacity(); - - var handler = (DiscordUDPConnection) connection.getConnectionHandler(); - var sender = connection.getAudioSender(); - var codec = OpusCodec.INSTANCE; - - for (int i = 0; i < remaining; i++) { - if (sender != null && handler != null && sender.canSendFrame(codec)) { - var buf = allocator.buffer(); - int start = buf.writerIndex(); - sender.retrieve(codec, buf, timestamp); - int len = buf.writerIndex() - start; - var packet = handler.createPacket(OpusCodec.PAYLOAD_TYPE, timestamp.get(), buf, len, false); - if (packet != null) { - manager.queuePacket(packet.nioBuffer(), (InetSocketAddress) handler.getServerAddress()); - packet.release(); - } - buf.release(); - } - } - - long frameDelay = 40 - (System.currentTimeMillis() - lastFrame); - - if (frameDelay > 0) { - eventLoop.schedule(this::loop, frameDelay, TimeUnit.MILLISECONDS); - } else { - loop(); - } - } - - private void loop() { - if (System.currentTimeMillis() < lastFrame + 60) { - lastFrame += 40; - } else { - lastFrame = System.currentTimeMillis(); - } - - populateQueue(); - } -} diff --git a/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/QueueManagerPool.java b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/QueueManagerPool.java similarity index 88% rename from ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/QueueManagerPool.java rename to ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/QueueManagerPool.java index de856fb..55e653a 100644 --- a/ext-udpqueue/src/main/java/moe/kyokobot/koe/codec/udpqueue/QueueManagerPool.java +++ b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/QueueManagerPool.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.codec.udpqueue; +package moe.kyokobot.koe.poller.udpqueue; import com.sedmelluq.discord.lavaplayer.udpqueue.natives.UdpQueueManager; @@ -7,10 +7,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static moe.kyokobot.koe.codec.OpusCodec.FRAME_DURATION; -import static moe.kyokobot.koe.codec.udpqueue.UdpQueueFramePollerFactory.MAXIMUM_PACKET_SIZE; +import static moe.kyokobot.koe.codec.OpusCodecInfo.FRAME_DURATION; public class QueueManagerPool { + public static final int DEFAULT_BUFFER_DURATION = 400; + public static final int MAXIMUM_PACKET_SIZE = 4096; + private final AtomicLong queueKeySeq; private final UdpQueueManager[] managers; private boolean closed; @@ -33,6 +35,10 @@ public QueueManagerPool(int size, int bufferDuration) { } } + public QueueManagerPool() { + this(Runtime.getRuntime().availableProcessors(), DEFAULT_BUFFER_DURATION); + } + public void close() { if (closed || this.managers == null) { return; diff --git a/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueFramePollerFactory.java b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueFramePollerFactory.java new file mode 100644 index 0000000..05a9afc --- /dev/null +++ b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueFramePollerFactory.java @@ -0,0 +1,28 @@ +package moe.kyokobot.koe.poller.udpqueue; + +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.OpusCodecInfo; +import moe.kyokobot.koe.poller.AbstractFramePoller; +import moe.kyokobot.koe.poller.FramePollerFactory; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Objects; + +public class UdpQueueFramePollerFactory implements FramePollerFactory { + private final QueueManagerPool pool; + + public UdpQueueFramePollerFactory(@NotNull QueueManagerPool pool) { + this.pool = Objects.requireNonNull(pool, "pool"); + } + + @Override + @Nullable + public AbstractFramePoller createFramePoller(CodecInstance codec, MediaConnection connection) { + if (codec.getInfo() instanceof OpusCodecInfo) { + return new UdpQueueOpusFramePoller(this.pool.getNextWrapper(), codec, connection); + } + return null; + } +} diff --git a/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java new file mode 100644 index 0000000..9b7180e --- /dev/null +++ b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java @@ -0,0 +1,51 @@ +package moe.kyokobot.koe.poller.udpqueue; + +import io.netty.buffer.ByteBuf; +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; +import moe.kyokobot.koe.poller.AbstractOpusFramePoller; +import org.jetbrains.annotations.NotNull; + +import java.net.InetSocketAddress; + +public class UdpQueueOpusFramePoller extends AbstractOpusFramePoller { + private final QueueManagerPool.UdpQueueWrapper manager; + + public UdpQueueOpusFramePoller(QueueManagerPool.UdpQueueWrapper manager, + @NotNull CodecInstance codec, + @NotNull MediaConnection connection) { + super(connection, codec); + this.manager = manager; + } + + @Override + protected int getPollsPerTick() { + return manager == null ? 0 : manager.getRemainingCapacity(); + } + + @Override + protected boolean canSendFrame() { + return manager != null && connection.getConnectionHandler() instanceof DiscordUDPConnection; + } + + @Override + protected void sendFramePayload(ByteBuf buf, int len, int timestamp) { + var connectionHandler = connection.getConnectionHandler(); + if (!(connectionHandler instanceof DiscordUDPConnection)) { + return; + } + var handler = (DiscordUDPConnection) connectionHandler; + + var packet = handler.createPacket(codec.getPayloadType(), timestamp, buf, len, false); + if (packet == null) { + return; + } + + try { + manager.queuePacket(packet.nioBuffer(), (InetSocketAddress) handler.getServerAddress()); + } finally { + packet.release(); + } + } +} diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java index 464d748..564cbcc 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java @@ -13,7 +13,9 @@ import dev.lavalink.youtube.YoutubeAudioSourceManager; import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.*; -import moe.kyokobot.koe.media.OpusAudioFrameProvider; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.OpusCodecInfo; +import moe.kyokobot.koe.media.AudioFrameProvider; import net.dv8tion.jda.api.JDA; import net.dv8tion.jda.api.JDABuilder; import net.dv8tion.jda.api.Permission; @@ -172,7 +174,7 @@ public void onMessageReceived(MessageReceivedEvent event) { if (koeClient.getConnection(voiceState.getGuild().getIdLong()) == null) { var conn = koeClient.createConnection(voiceState.getGuild().getIdLong()); var player = playerMap.computeIfAbsent(event.getGuild(), n -> playerManager.createPlayer()); - conn.setAudioSender(new AudioSender(player, conn)); + conn.setAudioSender(new OpusProvider(player)); conn.registerListener(new ExampleListener()); connect(channel); event.getChannel().sendMessage("Joined channel `" + channel.getName() + "`!").queue(); @@ -222,13 +224,13 @@ public void loadFailed(FriendlyException exception) { }); } - private static class AudioSender extends OpusAudioFrameProvider { + private static class OpusProvider implements AudioFrameProvider { private final AudioPlayer player; private final MutableAudioFrame frame; private final ByteBuffer frameBuffer; + private boolean isOpus; - AudioSender(AudioPlayer player, MediaConnection connection) { - super(connection); + OpusProvider(AudioPlayer player) { this.player = player; this.frame = new MutableAudioFrame(); this.frameBuffer = ByteBuffer.allocate(DISCORD_OPUS.maximumChunkSize()); @@ -236,14 +238,26 @@ private static class AudioSender extends OpusAudioFrameProvider { frame.setFormat(DISCORD_OPUS); } + @Override + public void onCodecChanged(@NotNull CodecInstance codec) { + this.isOpus = OpusCodecInfo.isInstanceOf(codec); + } + @Override public boolean canProvide() { + if (!isOpus) return false; return player.provide(frame); } @Override - public void retrieveOpusFrame(ByteBuf targetBuffer) { + public boolean provideFrame(ByteBuf targetBuffer) { targetBuffer.writeBytes(frameBuffer.array(), 0, frame.getDataLength()); + return true; + } + + @Override + public void dispose() { + } } diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java index 9efc205..10dfc73 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/UdpQueueTestBotLauncher.java @@ -1,20 +1,26 @@ package moe.kyokobot.koe.testbot; -import moe.kyokobot.koe.Koe; import moe.kyokobot.koe.KoeOptions; -import moe.kyokobot.koe.codec.udpqueue.UdpQueueFramePollerFactory; +import moe.kyokobot.koe.KoeOptionsBuilder; +import moe.kyokobot.koe.poller.udpqueue.QueueManagerPool; +import moe.kyokobot.koe.poller.udpqueue.UdpQueueFramePollerFactory; public class UdpQueueTestBotLauncher { public static void main(String[] args) { + var queuePool = new QueueManagerPool(); + var bot = new TestBot(System.getenv("TOKEN")) { @Override public KoeOptions configureKoe(KoeOptionsBuilder options) { return options - .setFramePollerFactory(new UdpQueueFramePollerFactory()) + .setFramePollerFactory(new UdpQueueFramePollerFactory(queuePool)) .create(); } }; - Runtime.getRuntime().addShutdownHook(new Thread(bot::stop)); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bot.stop(); + queuePool.close(); + })); bot.start(); } } From eb1c4c66e31f75632e6523ebc8c8bcea4f946a93 Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 18 Feb 2026 01:08:48 +0100 Subject: [PATCH 08/28] Remove deprecated KoeOptions constructors --- .../java/moe/kyokobot/koe/KoeOptions.java | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java index e41e0fc..a1b4f81 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java @@ -56,41 +56,6 @@ public class KoeOptions { this.enableDAVE = daveEnabled; } - /** - * @deprecated Use {@link KoeOptionsBuilder} instead. Provided for binary compatibility with older versions. - */ - @Deprecated(forRemoval = true, since = "2.2.0") - public KoeOptions( - @NotNull EventLoopGroup eventLoopGroup, - @NotNull Class socketChannelClass, - @NotNull Class datagramChannelClass, - @NotNull ByteBufAllocator byteBufAllocator, - @NotNull GatewayVersion gatewayVersion, - @NotNull FramePollerFactory framePollerFactory, - boolean highPacketPriority, - boolean deafened - ) { - this(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, gatewayVersion, - framePollerFactory, new DefaultCodecRegistry(), highPacketPriority, deafened, true, true); - } - - /** - * @deprecated Use {@link KoeOptionsBuilder} instead. Provided for binary compatibility with older versions. - */ - @Deprecated(forRemoval = true, since = "2.2.0") - public KoeOptions( - @NotNull EventLoopGroup eventLoopGroup, - @NotNull Class socketChannelClass, - @NotNull Class datagramChannelClass, - @NotNull ByteBufAllocator byteBufAllocator, - @NotNull GatewayVersion gatewayVersion, - @NotNull FramePollerFactory framePollerFactory, - boolean highPacketPriority - ) { - this(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, gatewayVersion, - framePollerFactory, new DefaultCodecRegistry(), highPacketPriority, false, true, true); - } - @NotNull public EventLoopGroup getEventLoopGroup() { return eventLoopGroup; From e01eb1a83a526c6b330c11206e4ea3233664d555 Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 18 Feb 2026 01:39:47 +0100 Subject: [PATCH 09/28] Isolate the WIP stuff in .experimental package and add separate constructors --- core/src/main/java/moe/kyokobot/koe/Koe.java | 28 ++-- .../java/moe/kyokobot/koe/KoeOptions.java | 118 +++------------- .../moe/kyokobot/koe/KoeOptionsBuilder.java | 36 ++--- .../moe/kyokobot/koe/MediaConnection.java | 35 ----- .../moe/kyokobot/koe/codec/CodecRegistry.java | 10 -- .../koe/codec/DefaultCodecRegistry.java | 35 ++--- .../experimental/KoeClientExperimental.java | 22 +++ .../koe/experimental/KoeExperimental.java | 56 ++++++++ .../KoeOptionsBuilderExperimental.java | 16 +++ .../experimental/KoeOptionsExperimental.java | 15 ++ .../MediaConnectionExperimental.java | 45 ++++++ .../codec/ExperimentalCodecRegistry.java | 14 ++ .../codec/H264CodecInfo.java | 5 +- .../codec/VP8CodecInfo.java | 5 +- .../codec/VP9CodecInfo.java | 5 +- .../kyokobot/koe/internal/KoeClientImpl.java | 16 ++- .../moe/kyokobot/koe/internal/KoeImpl.java | 28 ---- .../kyokobot/koe/internal/KoeOptionsImpl.java | 130 ++++++++++++++++++ .../koe/internal/MediaConnectionImpl.java | 6 +- .../poller/netty/NettyFramePollerFactory.java | 1 - 20 files changed, 388 insertions(+), 238 deletions(-) create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/KoeClientExperimental.java create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/KoeExperimental.java create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsBuilderExperimental.java create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsExperimental.java create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/MediaConnectionExperimental.java create mode 100644 core/src/main/java/moe/kyokobot/koe/experimental/codec/ExperimentalCodecRegistry.java rename core/src/main/java/moe/kyokobot/koe/{ => experimental}/codec/H264CodecInfo.java (77%) rename core/src/main/java/moe/kyokobot/koe/{ => experimental}/codec/VP8CodecInfo.java (76%) rename core/src/main/java/moe/kyokobot/koe/{ => experimental}/codec/VP9CodecInfo.java (76%) delete mode 100644 core/src/main/java/moe/kyokobot/koe/internal/KoeImpl.java create mode 100644 core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java diff --git a/core/src/main/java/moe/kyokobot/koe/Koe.java b/core/src/main/java/moe/kyokobot/koe/Koe.java index 533dfa5..5a3d50b 100644 --- a/core/src/main/java/moe/kyokobot/koe/Koe.java +++ b/core/src/main/java/moe/kyokobot/koe/Koe.java @@ -1,21 +1,33 @@ package moe.kyokobot.koe; -import moe.kyokobot.koe.internal.KoeImpl; +import moe.kyokobot.koe.internal.KoeClientImpl; import org.jetbrains.annotations.NotNull; -public interface Koe { +import java.util.Objects; + +public class Koe { + private final KoeOptions options; + + private Koe(@NotNull KoeOptions options) { + this.options = Objects.requireNonNull(options); + } + /** * @param clientId the ID of the user or bot which will connect to Discord voice servers. * @return a new Koe client */ @NotNull - KoeClient newClient(long clientId); + public KoeClient newClient(long clientId) { + return new KoeClientImpl(clientId, options); + } /** * @return Options of current Koe instance */ @NotNull - KoeOptions getOptions(); + public KoeOptions getOptions() { + return options; + } /** * Create a new Koe instance with given options. @@ -24,8 +36,8 @@ public interface Koe { * @return A new Koe instance. */ @NotNull - static Koe koe(@NotNull KoeOptions options) { - return new KoeImpl(options); + public static Koe koe(@NotNull KoeOptions options) { + return new Koe(options); } /** @@ -34,7 +46,7 @@ static Koe koe(@NotNull KoeOptions options) { * @return A new Koe instance. */ @NotNull - static Koe koe() { - return new KoeImpl(KoeOptions.defaultOptions()); + public static Koe koe() { + return new Koe(KoeOptions.builder().create()); } } diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java index a1b4f81..f1231eb 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java @@ -5,117 +5,41 @@ import io.netty.channel.socket.DatagramChannel; import io.netty.channel.socket.SocketChannel; import moe.kyokobot.koe.codec.CodecRegistry; -import moe.kyokobot.koe.codec.DefaultCodecRegistry; -import moe.kyokobot.koe.poller.FramePollerFactory; import moe.kyokobot.koe.gateway.GatewayVersion; +import moe.kyokobot.koe.poller.FramePollerFactory; import org.jetbrains.annotations.NotNull; -import java.util.Objects; - -/** - * KoeOptions is a class that holds various options for configuring the Koe client. - * - * @see KoeOptionsBuilder The builder class for explanation of all options defined in this class. - */ -public class KoeOptions { - private final EventLoopGroup eventLoopGroup; - private final Class socketChannelClass; - private final Class datagramChannelClass; - private final ByteBufAllocator byteBufAllocator; - private final GatewayVersion gatewayVersion; - private final FramePollerFactory framePollerFactory; - private final CodecRegistry codecRegistry; - private final boolean highPacketPriority; - private final boolean deafened; - private final boolean enableWSSPortOverride; - private final boolean enableDAVE; - - KoeOptions( - @NotNull EventLoopGroup eventLoopGroup, - @NotNull Class socketChannelClass, - @NotNull Class datagramChannelClass, - @NotNull ByteBufAllocator byteBufAllocator, - @NotNull GatewayVersion gatewayVersion, - @NotNull FramePollerFactory framePollerFactory, - @NotNull CodecRegistry codecRegistry, - boolean highPacketPriority, - boolean deafened, - boolean enableWSSPortOverride, - boolean daveEnabled - ) { - this.eventLoopGroup = Objects.requireNonNull(eventLoopGroup); - this.socketChannelClass = Objects.requireNonNull(socketChannelClass); - this.datagramChannelClass = Objects.requireNonNull(datagramChannelClass); - this.byteBufAllocator = Objects.requireNonNull(byteBufAllocator); - this.gatewayVersion = Objects.requireNonNull(gatewayVersion); - this.framePollerFactory = Objects.requireNonNull(framePollerFactory); - this.codecRegistry = Objects.requireNonNull(codecRegistry); - this.highPacketPriority = highPacketPriority; - this.deafened = deafened; - this.enableWSSPortOverride = enableWSSPortOverride; - this.enableDAVE = daveEnabled; - } - - @NotNull - public EventLoopGroup getEventLoopGroup() { - return eventLoopGroup; +public interface KoeOptions { + /** + * Creates a new {@link KoeOptionsBuilder} instance with default options. + * + * @return A new {@link KoeOptionsBuilder} instance. + */ + static KoeOptionsBuilder builder() { + return new KoeOptionsBuilder(); } - @NotNull - public Class getSocketChannelClass() { - return socketChannelClass; - } + @NotNull EventLoopGroup getEventLoopGroup(); - @NotNull - public Class getDatagramChannelClass() { - return datagramChannelClass; - } + @NotNull Class getSocketChannelClass(); - @NotNull - public ByteBufAllocator getByteBufAllocator() { - return byteBufAllocator; - } + @NotNull Class getDatagramChannelClass(); - @NotNull - public GatewayVersion getGatewayVersion() { - return gatewayVersion; - } + @NotNull ByteBufAllocator getByteBufAllocator(); - @NotNull - public FramePollerFactory getFramePollerFactory() { - return framePollerFactory; - } + @NotNull GatewayVersion getGatewayVersion(); - @NotNull - public CodecRegistry getCodecRegistry() { - return codecRegistry; - } + @NotNull FramePollerFactory getFramePollerFactory(); - public boolean isHighPacketPriority() { - return highPacketPriority; - } + @NotNull CodecRegistry getCodecRegistry(); - public boolean isDeafened() { - return deafened; - } + boolean isExperimental(); - public boolean isEnableWSSPortOverride() { - return enableWSSPortOverride; - } + boolean isHighPacketPriority(); - public boolean isEnableDAVE() { - return enableDAVE; - } + boolean isDeafened(); - /** - * @return An instance of {@link KoeOptions} with default options. - */ - @NotNull - public static KoeOptions defaultOptions() { - return new KoeOptionsBuilder().create(); - } + boolean isEnableWSSPortOverride(); - public static KoeOptionsBuilder builder() { - return new KoeOptionsBuilder(); - } + boolean isEnableDAVE(); } diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java index f9b1ab1..8ee8250 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java @@ -14,27 +14,29 @@ import io.netty.channel.socket.nio.NioSocketChannel; import moe.kyokobot.koe.codec.CodecRegistry; import moe.kyokobot.koe.codec.DefaultCodecRegistry; +import moe.kyokobot.koe.gateway.GatewayVersion; +import moe.kyokobot.koe.internal.KoeOptionsImpl; import moe.kyokobot.koe.poller.FramePollerFactory; import moe.kyokobot.koe.poller.netty.NettyFramePollerFactory; -import moe.kyokobot.koe.gateway.GatewayVersion; import org.jetbrains.annotations.NotNull; import java.util.Objects; public class KoeOptionsBuilder { - private EventLoopGroup eventLoopGroup; - private Class socketChannelClass; - private Class datagramChannelClass; - private ByteBufAllocator byteBufAllocator; - private GatewayVersion gatewayVersion; - private FramePollerFactory framePollerFactory; - private CodecRegistry codecRegistry; - private boolean highPacketPriority; - private boolean deafened; - private boolean enableWSSPortOverride; - private boolean enableDAVE; - - KoeOptionsBuilder() { + protected EventLoopGroup eventLoopGroup; + protected Class socketChannelClass; + protected Class datagramChannelClass; + protected ByteBufAllocator byteBufAllocator; + protected GatewayVersion gatewayVersion; + protected FramePollerFactory framePollerFactory; + protected CodecRegistry codecRegistry; + protected boolean experimental; + protected boolean highPacketPriority; + protected boolean deafened; + protected boolean enableWSSPortOverride; + protected boolean enableDAVE; + + protected KoeOptionsBuilder() { boolean epoll = Epoll.isAvailable(); this.eventLoopGroup = epoll ? new EpollEventLoopGroup() @@ -51,6 +53,7 @@ public class KoeOptionsBuilder { this.gatewayVersion = GatewayVersion.V8; this.framePollerFactory = new NettyFramePollerFactory(); this.codecRegistry = new DefaultCodecRegistry(); + this.experimental = false; this.highPacketPriority = true; this.deafened = false; this.enableWSSPortOverride = true; @@ -184,7 +187,8 @@ public KoeOptionsBuilder setDAVEEnabled(boolean enabled) { } public KoeOptions create() { - return new KoeOptions(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, - gatewayVersion, framePollerFactory, codecRegistry, highPacketPriority, deafened, enableWSSPortOverride, enableDAVE); + return new KoeOptionsImpl(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, + gatewayVersion, framePollerFactory, codecRegistry, experimental, + highPacketPriority, deafened, enableWSSPortOverride, enableDAVE); } } diff --git a/core/src/main/java/moe/kyokobot/koe/MediaConnection.java b/core/src/main/java/moe/kyokobot/koe/MediaConnection.java index aa14203..fd612f8 100644 --- a/core/src/main/java/moe/kyokobot/koe/MediaConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/MediaConnection.java @@ -2,7 +2,6 @@ import moe.kyokobot.koe.codec.CodecInfo; import moe.kyokobot.koe.codec.CodecInstance; -import moe.kyokobot.koe.experimental.media.VideoFrameProvider; import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.handler.ConnectionHandler; import moe.kyokobot.koe.media.AudioFrameProvider; @@ -40,9 +39,6 @@ public interface MediaConnection extends Closeable { @Nullable AudioFrameProvider getAudioSender(); - @Nullable - VideoFrameProvider getVideoSender(); - long getGuildId(); @Nullable @@ -84,37 +80,6 @@ default void setAudioCodec(@NotNull CodecInfo info) { */ void stopAudioFramePolling(); - void setVideoSender(@Nullable VideoFrameProvider sender); - - /** - * Sets the video codec instance for this connection. - * - * @param videoCodec the codec instance to use, or null to disable video - */ - void setVideoCodec(@Nullable CodecInstance videoCodec); - - /** - * Sets the video codec using a codec info (convenience method). - * Creates a codec instance with default payload types. - * - * @param info the codec info to use - */ - default void setVideoCodec(@NotNull CodecInfo info) { - setVideoCodec(info.instantiate()); - } - - /** - * Starts polling video frames. Called automatically after connecting if codec has been set. - */ - void startVideoFramePolling(); - - /** - * Stops polling video frames. - * - * @see MediaConnection#startAudioFramePolling() - */ - void stopVideoFramePolling(); - void registerListener(KoeEventListener listener); void unregisterListener(KoeEventListener listener); diff --git a/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java b/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java index ce08a4e..2086992 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/CodecRegistry.java @@ -35,16 +35,6 @@ public interface CodecRegistry { @Nullable CodecInfo getByName(@NotNull String name); - /** - * Gets codec info by payload type. - * Useful for decoding incoming packets. - * - * @param payloadType the payload type - * @return CodecInfo or null if not found - */ - @Nullable - CodecInfo getByPayloadType(byte payloadType); - /** * Gets all registered audio codecs. * diff --git a/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java b/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java index 05d13bc..19319bc 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java +++ b/core/src/main/java/moe/kyokobot/koe/codec/DefaultCodecRegistry.java @@ -5,8 +5,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; -import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -17,11 +15,17 @@ public class DefaultCodecRegistry implements CodecRegistry { protected final ConcurrentHashMap codecsByName = new ConcurrentHashMap<>(); protected final ConcurrentHashMap codecsByPayloadType = new ConcurrentHashMap<>(); + public DefaultCodecRegistry(boolean registerBuiltins) { + if (registerBuiltins) { + registerBuiltInCodecs(); + } + } + /** * Creates a registry with all built-in codecs pre-registered. */ public DefaultCodecRegistry() { - registerBuiltInCodecs(); + this(true); } /** @@ -31,7 +35,7 @@ public DefaultCodecRegistry() { * @return an empty CodecRegistry */ public static DefaultCodecRegistry empty() { - return new EmptyCodecRegistry(); + return new DefaultCodecRegistry(false); } /** @@ -39,23 +43,6 @@ public static DefaultCodecRegistry empty() { */ protected void registerBuiltInCodecs() { register(OpusCodecInfo.INSTANCE); - register(H264CodecInfo.INSTANCE); - register(VP8CodecInfo.INSTANCE); - register(VP9CodecInfo.INSTANCE); - } - - /** - * Private subclass for empty registries to avoid registering built-ins. - */ - private static class EmptyCodecRegistry extends DefaultCodecRegistry { - EmptyCodecRegistry() { - super(); // Initialize maps - } - - @Override - protected void registerBuiltInCodecs() { - // Skip built-in codec registration - } } @Override @@ -93,12 +80,6 @@ public CodecInfo getByName(@NotNull String name) { return codecsByName.get(name.toLowerCase()); } - @Override - @Nullable - public CodecInfo getByPayloadType(byte payloadType) { - return codecsByPayloadType.get(payloadType); - } - @Override @NotNull public Collection getAudioCodecs() { diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/KoeClientExperimental.java b/core/src/main/java/moe/kyokobot/koe/experimental/KoeClientExperimental.java new file mode 100644 index 0000000..d3e3b6a --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/KoeClientExperimental.java @@ -0,0 +1,22 @@ +package moe.kyokobot.koe.experimental; + +import moe.kyokobot.koe.KoeClient; +import moe.kyokobot.koe.MediaConnection; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public interface KoeClientExperimental extends KoeClient { + @NotNull + MediaConnectionExperimental createConnection(long guildId); + + @Nullable + MediaConnectionExperimental getConnection(long guildId); + + static MediaConnectionExperimental asExperimental(MediaConnection connection) { + if (connection instanceof MediaConnectionExperimental) { + return (MediaConnectionExperimental) connection; + } else { + throw new IllegalArgumentException("MediaConnection is not an instance of MediaConnectionExperimental"); + } + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/KoeExperimental.java b/core/src/main/java/moe/kyokobot/koe/experimental/KoeExperimental.java new file mode 100644 index 0000000..4c02fb4 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/KoeExperimental.java @@ -0,0 +1,56 @@ +package moe.kyokobot.koe.experimental; + +import moe.kyokobot.koe.internal.KoeClientImpl; +import org.jetbrains.annotations.NotNull; + +import java.util.Objects; + +public class KoeExperimental { + private final KoeOptionsExperimental options; + + private KoeExperimental(@NotNull KoeOptionsExperimental options) { + if (!options.isExperimental()) { + throw new IllegalArgumentException("Provided options are not marked as experimental. Please use KoeOptionsExperimental.builder() to create options for KoeExperimental."); + } + + this.options = Objects.requireNonNull(options); + } + + /** + * @param clientId the ID of the user or bot which will connect to Discord voice servers. + * @return a new Koe client + */ + @NotNull + public KoeClientExperimental newClient(long clientId) { + return new KoeClientImpl(clientId, options); + } + + /** + * @return Options of current Koe instance + */ + @NotNull + public KoeOptionsExperimental getOptions() { + return options; + } + + /** + * Create a new Koe instance with given options. + * + * @param options Options used by new Koe instance. + * @return A new Koe instance. + */ + @NotNull + public static KoeExperimental koe(@NotNull KoeOptionsExperimental options) { + return new KoeExperimental(options); + } + + /** + * Create a new Koe instance with default options. + * + * @return A new Koe instance. + */ + @NotNull + public static KoeExperimental koe() { + return new KoeExperimental(KoeOptionsExperimental.builder().create()); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsBuilderExperimental.java b/core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsBuilderExperimental.java new file mode 100644 index 0000000..7cf9348 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsBuilderExperimental.java @@ -0,0 +1,16 @@ +package moe.kyokobot.koe.experimental; + +import moe.kyokobot.koe.KoeOptionsBuilder; +import moe.kyokobot.koe.experimental.codec.ExperimentalCodecRegistry; + +public class KoeOptionsBuilderExperimental extends KoeOptionsBuilder { + public KoeOptionsBuilderExperimental() { + super(); + this.codecRegistry = new ExperimentalCodecRegistry(); + this.experimental = true; + } + + public KoeOptionsExperimental create() { + return (KoeOptionsExperimental) super.create(); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsExperimental.java b/core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsExperimental.java new file mode 100644 index 0000000..55210de --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/KoeOptionsExperimental.java @@ -0,0 +1,15 @@ +package moe.kyokobot.koe.experimental; + +import moe.kyokobot.koe.KoeOptions; + +public interface KoeOptionsExperimental extends KoeOptions { + /** + * Creates a new {@link KoeOptionsBuilderExperimental} instance with experimental defaults. + * Required to use with Koe created through {@link moe.kyokobot.koe.experimental.KoeExperimental} to access experimental features. + * + * @return A new {@link KoeOptionsBuilderExperimental} instance. + */ + static KoeOptionsBuilderExperimental builder() { + return new KoeOptionsBuilderExperimental(); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/MediaConnectionExperimental.java b/core/src/main/java/moe/kyokobot/koe/experimental/MediaConnectionExperimental.java new file mode 100644 index 0000000..d610ae2 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/MediaConnectionExperimental.java @@ -0,0 +1,45 @@ +package moe.kyokobot.koe.experimental; + +import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.codec.CodecInfo; +import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.experimental.media.VideoFrameProvider; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public interface MediaConnectionExperimental extends MediaConnection { + @Nullable + VideoFrameProvider getVideoSender(); + + void setVideoSender(@Nullable VideoFrameProvider sender); + + /** + * Sets the video codec instance for this connection. + * + * @param videoCodec the codec instance to use, or null to disable video + */ + void setVideoCodec(@Nullable CodecInstance videoCodec); + + /** + * Sets the video codec using a codec info (convenience method). + * Creates a codec instance with default payload types. + * + * @param info the codec info to use + */ + default void setVideoCodec(@NotNull CodecInfo info) { + setVideoCodec(info.instantiate()); + } + + /** + * Starts polling video frames. Called automatically after connecting if codec has been set. + */ + void startVideoFramePolling(); + + /** + * Stops polling video frames. + * + * @see #startVideoFramePolling() + */ + void stopVideoFramePolling(); + +} diff --git a/core/src/main/java/moe/kyokobot/koe/experimental/codec/ExperimentalCodecRegistry.java b/core/src/main/java/moe/kyokobot/koe/experimental/codec/ExperimentalCodecRegistry.java new file mode 100644 index 0000000..2e6dc0e --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/experimental/codec/ExperimentalCodecRegistry.java @@ -0,0 +1,14 @@ +package moe.kyokobot.koe.experimental.codec; + +import moe.kyokobot.koe.codec.DefaultCodecRegistry; + +public class ExperimentalCodecRegistry extends DefaultCodecRegistry { + @Override + protected void registerBuiltInCodecs() { + super.registerBuiltInCodecs(); + + register(H264CodecInfo.INSTANCE); + register(VP8CodecInfo.INSTANCE); + register(VP9CodecInfo.INSTANCE); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/experimental/codec/H264CodecInfo.java similarity index 77% rename from core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java rename to core/src/main/java/moe/kyokobot/koe/experimental/codec/H264CodecInfo.java index c0d13da..fcbfde9 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/H264CodecInfo.java +++ b/core/src/main/java/moe/kyokobot/koe/experimental/codec/H264CodecInfo.java @@ -1,6 +1,7 @@ -package moe.kyokobot.koe.codec; +package moe.kyokobot.koe.experimental.codec; -import org.jetbrains.annotations.NotNull; +import moe.kyokobot.koe.codec.CodecInfo; +import moe.kyokobot.koe.codec.CodecType; /** * H.264 video codec information. diff --git a/core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/experimental/codec/VP8CodecInfo.java similarity index 76% rename from core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java rename to core/src/main/java/moe/kyokobot/koe/experimental/codec/VP8CodecInfo.java index fe92a5e..d3eaead 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/VP8CodecInfo.java +++ b/core/src/main/java/moe/kyokobot/koe/experimental/codec/VP8CodecInfo.java @@ -1,6 +1,7 @@ -package moe.kyokobot.koe.codec; +package moe.kyokobot.koe.experimental.codec; -import org.jetbrains.annotations.NotNull; +import moe.kyokobot.koe.codec.CodecInfo; +import moe.kyokobot.koe.codec.CodecType; /** * VP8 video codec information. diff --git a/core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java b/core/src/main/java/moe/kyokobot/koe/experimental/codec/VP9CodecInfo.java similarity index 76% rename from core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java rename to core/src/main/java/moe/kyokobot/koe/experimental/codec/VP9CodecInfo.java index 5bbf816..a09a9b0 100644 --- a/core/src/main/java/moe/kyokobot/koe/codec/VP9CodecInfo.java +++ b/core/src/main/java/moe/kyokobot/koe/experimental/codec/VP9CodecInfo.java @@ -1,6 +1,7 @@ -package moe.kyokobot.koe.codec; +package moe.kyokobot.koe.experimental.codec; -import org.jetbrains.annotations.NotNull; +import moe.kyokobot.koe.codec.CodecInfo; +import moe.kyokobot.koe.codec.CodecType; /** * VP9 video codec information. diff --git a/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java index 26a294a..7ffde34 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/KoeClientImpl.java @@ -3,6 +3,8 @@ import moe.kyokobot.koe.KoeClient; import moe.kyokobot.koe.KoeOptions; import moe.kyokobot.koe.MediaConnection; +import moe.kyokobot.koe.experimental.KoeClientExperimental; +import moe.kyokobot.koe.experimental.MediaConnectionExperimental; import moe.kyokobot.koe.gateway.GatewayVersion; import moe.kyokobot.libdave.NativeDaveFactory; import moe.kyokobot.libdave.netty.NativeNettyDaveFactory; @@ -17,12 +19,12 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -public class KoeClientImpl implements KoeClient { +public class KoeClientImpl implements KoeClient, KoeClientExperimental { private static final Logger logger = LoggerFactory.getLogger(KoeClientImpl.class); private final long clientId; private final KoeOptions options; - private final Map connections; + private final Map connections; private final @Nullable NettyDaveFactory daveFactory; public KoeClientImpl(long clientId, KoeOptions options) { @@ -41,13 +43,13 @@ public KoeClientImpl(long clientId, KoeOptions options) { @Override @NotNull - public MediaConnection createConnection(long guildId) { + public MediaConnectionExperimental createConnection(long guildId) { return connections.computeIfAbsent(guildId, this::newVoiceConnection); } @Override @Nullable - public MediaConnection getConnection(long guildId) { + public MediaConnectionExperimental getConnection(long guildId) { return connections.get(guildId); } @@ -60,8 +62,8 @@ public void destroyConnection(long guildId) { } } - void removeConnection(long guildId) { - connections.remove(guildId); + void removeClosedConnection(MediaConnectionImpl connection) { + connections.remove(connection.getGuildId(), connection); } @Override @@ -78,7 +80,7 @@ public void close() { } } - public MediaConnection newVoiceConnection(long id) { + public MediaConnectionExperimental newVoiceConnection(long id) { return new MediaConnectionImpl(this, id); } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/KoeImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/KoeImpl.java deleted file mode 100644 index 7a9011a..0000000 --- a/core/src/main/java/moe/kyokobot/koe/internal/KoeImpl.java +++ /dev/null @@ -1,28 +0,0 @@ -package moe.kyokobot.koe.internal; - -import moe.kyokobot.koe.Koe; -import moe.kyokobot.koe.KoeClient; -import moe.kyokobot.koe.KoeOptions; -import org.jetbrains.annotations.NotNull; - -import java.util.Objects; - -public class KoeImpl implements Koe { - private final KoeOptions options; - - public KoeImpl(@NotNull KoeOptions options) { - this.options = Objects.requireNonNull(options); - } - - @NotNull - @Override - public KoeClient newClient(long clientId) { - return new KoeClientImpl(clientId, options); - } - - @NotNull - @Override - public KoeOptions getOptions() { - return options; - } -} diff --git a/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java new file mode 100644 index 0000000..3b95840 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java @@ -0,0 +1,130 @@ +package moe.kyokobot.koe.internal; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.socket.DatagramChannel; +import io.netty.channel.socket.SocketChannel; +import moe.kyokobot.koe.KoeOptionsBuilder; +import moe.kyokobot.koe.codec.CodecRegistry; +import moe.kyokobot.koe.experimental.KoeOptionsExperimental; +import moe.kyokobot.koe.gateway.GatewayVersion; +import moe.kyokobot.koe.poller.FramePollerFactory; +import org.jetbrains.annotations.NotNull; + +import java.util.Objects; + +/** + * KoeOptions is a class that holds various options for configuring the Koe client. + * + * @see KoeOptionsBuilder The builder class for explanation of all options defined in this class. + */ +public class KoeOptionsImpl implements KoeOptionsExperimental { + private final EventLoopGroup eventLoopGroup; + private final Class socketChannelClass; + private final Class datagramChannelClass; + private final ByteBufAllocator byteBufAllocator; + private final GatewayVersion gatewayVersion; + private final FramePollerFactory framePollerFactory; + private final CodecRegistry codecRegistry; + private final boolean experimental; + private final boolean highPacketPriority; + private final boolean deafened; + private final boolean enableWSSPortOverride; + private final boolean enableDAVE; + + public KoeOptionsImpl( + @NotNull EventLoopGroup eventLoopGroup, + @NotNull Class socketChannelClass, + @NotNull Class datagramChannelClass, + @NotNull ByteBufAllocator byteBufAllocator, + @NotNull GatewayVersion gatewayVersion, + @NotNull FramePollerFactory framePollerFactory, + @NotNull CodecRegistry codecRegistry, + boolean experimental, + boolean highPacketPriority, + boolean deafened, + boolean enableWSSPortOverride, + boolean daveEnabled + ) { + this.eventLoopGroup = Objects.requireNonNull(eventLoopGroup); + this.socketChannelClass = Objects.requireNonNull(socketChannelClass); + this.datagramChannelClass = Objects.requireNonNull(datagramChannelClass); + this.byteBufAllocator = Objects.requireNonNull(byteBufAllocator); + this.gatewayVersion = Objects.requireNonNull(gatewayVersion); + this.framePollerFactory = Objects.requireNonNull(framePollerFactory); + this.codecRegistry = Objects.requireNonNull(codecRegistry); + this.experimental = experimental; + this.highPacketPriority = highPacketPriority; + this.deafened = deafened; + this.enableWSSPortOverride = enableWSSPortOverride; + this.enableDAVE = daveEnabled; + } + + @NotNull + @Override + public EventLoopGroup getEventLoopGroup() { + return eventLoopGroup; + } + + @NotNull + @Override + public Class getSocketChannelClass() { + return socketChannelClass; + } + + @NotNull + @Override + public Class getDatagramChannelClass() { + return datagramChannelClass; + } + + @NotNull + @Override + public ByteBufAllocator getByteBufAllocator() { + return byteBufAllocator; + } + + @NotNull + @Override + public GatewayVersion getGatewayVersion() { + return gatewayVersion; + } + + @NotNull + @Override + public FramePollerFactory getFramePollerFactory() { + return framePollerFactory; + } + + @NotNull + @Override + public CodecRegistry getCodecRegistry() { + return codecRegistry; + } + + @Override + public boolean isExperimental() { + return experimental; + } + + @Override + public boolean isHighPacketPriority() { + return highPacketPriority; + } + + @Override + public boolean isDeafened() { + return deafened; + } + + @Override + public boolean isEnableWSSPortOverride() { + return enableWSSPortOverride; + } + + @Override + public boolean isEnableDAVE() { + return enableDAVE; + } + +} diff --git a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java index c5ed91d..d9a7e66 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java @@ -4,6 +4,7 @@ import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.codec.CodecType; import moe.kyokobot.koe.codec.OpusCodecInfo; +import moe.kyokobot.koe.experimental.MediaConnectionExperimental; import moe.kyokobot.koe.experimental.media.VideoFrameProvider; import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.gateway.MediaValve; @@ -18,7 +19,7 @@ import java.util.Objects; import java.util.concurrent.CompletionStage; -public class MediaConnectionImpl implements MediaConnection { +public class MediaConnectionImpl implements MediaConnection, MediaConnectionExperimental { private static final Logger logger = LoggerFactory.getLogger(MediaConnectionImpl.class); private final KoeClientImpl client; @@ -266,7 +267,7 @@ public void close() { } disconnect(); - client.removeConnection(guildId); + client.removeClosedConnection(this); } @Override @@ -303,7 +304,6 @@ public void destroyDAVEManager() { this.daveManager.close(); } catch (Exception e) { logger.error("Error closing old DAVE manager", e); - } } } diff --git a/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java index e985fb6..0be2147 100644 --- a/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java +++ b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyFramePollerFactory.java @@ -2,7 +2,6 @@ import moe.kyokobot.koe.MediaConnection; import moe.kyokobot.koe.codec.CodecInstance; -import moe.kyokobot.koe.codec.H264CodecInfo; import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.poller.AbstractFramePoller; import moe.kyokobot.koe.poller.FramePollerFactory; From ec5e448f8695898ba2677bb447864114f9b02d1f Mon Sep 17 00:00:00 2001 From: Alula Date: Wed, 18 Feb 2026 01:48:07 +0100 Subject: [PATCH 10/28] Remove deprecated VIDEO_SINK_WANTS alias --- MIGRATION.md | 19 ++++++++++++++----- .../java/moe/kyokobot/koe/gateway/Op.java | 5 ----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index a314c89..8db613c 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -3,7 +3,20 @@ ## VoiceServerInfo 1. The public constructor has been removed. Use `VoiceServerInfo#builder()` to create instances of this class instead. -2. It's now required to pass `channelId` (the ID of the voice channel), as it is used as a MLS group identifier while using DAVE E2E encryption. +2. As DAVE E2E encryption is now mandatory, it's required to pass `channelId` (the ID of the voice channel), because it's needed as the MLS group identifier. + +## Minor changes and removals + +- Old `VIDEO_SINK_WANTS` alias for `MEDIA_SINK_WANTS` has been removed. +- Compatibility constructors from `KoeOptions` have been removed. Use `KoeOptions#builder()` instead. + +## Experimental package + +The package `moe.kyokobot.koe.experimental` is reserved for APIs that are not yet stable. Types and members in this package may change or be removed +in any **minor** release (e.g. 3.1, 3.2). Do not depend on them if you need binary compatibility across minor versions. The rest of the public API +follows the [binary compatibility policy](README.md#binary-compatibility) described in the README. + +The unfinished video support has been moved to the experimental package in intent to be finished in future releases. ## Refactoring of the poller / provider API @@ -124,7 +137,3 @@ All codec classes have been renamed with an `Info` suffix: | `H264Codec.INSTANCE` | `H264CodecInfo.INSTANCE` | | `VP8Codec.INSTANCE` | `VP8CodecInfo.INSTANCE` | | `VP9Codec.INSTANCE` | `VP9CodecInfo.INSTANCE` | - -## Experimental package - -The package `moe.kyokobot.koe.experimental` is reserved for APIs that are not yet stable. Types and members in this package may change or be removed in any **minor** release (e.g. 3.1, 3.2). Do not depend on them if you need binary compatibility across minor versions. The rest of the public API follows the [binary compatibility policy](README.md#binary-compatibility) described in the README. diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/Op.java b/core/src/main/java/moe/kyokobot/koe/gateway/Op.java index 8d1ff0e..85d2fb0 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/Op.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/Op.java @@ -21,11 +21,6 @@ private Op() { public static final int CLIENT_DISCONNECT = 13; public static final int CODECS = 14; public static final int MEDIA_SINK_WANTS = 15; - /** - * @deprecated Use {@link #MEDIA_SINK_WANTS} instead. - */ - @Deprecated - public static final int VIDEO_SINK_WANTS = 15; public static final int VOICE_BACKEND_VERSION = 16; public static final int CHANNEL_OPTIONS_UPDATE = 17; public static final int CLIENT_FLAGS = 18; From e12ed717d5f6e4a18165ebfe287fa499eed6bcca Mon Sep 17 00:00:00 2001 From: Alula Date: Sat, 21 Feb 2026 09:45:28 +0100 Subject: [PATCH 11/28] Rename listeners, move gateway impl to internal --- MIGRATION.md | 9 ++ .../moe/kyokobot/koe/KoeEventAdapter.java | 8 +- .../moe/kyokobot/koe/KoeEventListener.java | 35 +++++++- .../AEADAES256GCMRTPSizeEncryptionMode.java | 2 +- ...ChaCha20Poly1305RTPSizeEncryptionMode.java | 2 +- .../kyokobot/koe/crypto/EncryptionMode.java | 2 +- .../kyokobot/koe/gateway/GatewayVersion.java | 8 +- .../moe/kyokobot/koe/gateway/MediaValve.java | 84 ++---------------- .../kyokobot/koe/internal/DAVEManager.java | 6 ++ .../koe/internal/EventDispatcher.java | 14 ++- .../koe/internal/MediaConnectionImpl.java | 3 +- .../AbstractMediaGatewayConnection.java | 5 +- .../MediaGatewayConnectionFactory.java | 3 +- .../gateway/MediaGatewayV4Connection.java | 16 +++- .../gateway/MediaGatewayV5Connection.java | 25 +++++- .../gateway/MediaGatewayV8Connection.java | 15 +++- .../koe/internal/gateway/MediaValveImpl.java | 87 +++++++++++++++++++ .../koe/poller/AbstractOpusFramePoller.java | 2 +- .../moe/kyokobot/koe/testbot/TestBot.java | 2 +- 19 files changed, 224 insertions(+), 104 deletions(-) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/gateway/AbstractMediaGatewayConnection.java (98%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/gateway/MediaGatewayConnectionFactory.java (73%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/gateway/MediaGatewayV4Connection.java (93%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/gateway/MediaGatewayV5Connection.java (90%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/gateway/MediaGatewayV8Connection.java (96%) create mode 100644 core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaValveImpl.java diff --git a/MIGRATION.md b/MIGRATION.md index 8db613c..fc1a506 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -18,6 +18,11 @@ follows the [binary compatibility policy](README.md#binary-compatibility) descri The unfinished video support has been moved to the experimental package in intent to be finished in future releases. +## KoeEventListener + +1. The old `userConnected` method has been renamed to `userStreamsChanged` to reflect the fact that it's not fired when a user joins the voice channel, but when their stream configuration changes (e.g., starts/stops video). +2. A new `usersConnected` method has been added if you actually need to know when an user has joined the voice channel. + ## Refactoring of the poller / provider API Frame poller classes moved from `moe.kyokobot.koe.codec` to `moe.kyokobot.koe.poller`. @@ -137,3 +142,7 @@ All codec classes have been renamed with an `Info` suffix: | `H264Codec.INSTANCE` | `H264CodecInfo.INSTANCE` | | `VP8Codec.INSTANCE` | `VP8CodecInfo.INSTANCE` | | `VP9Codec.INSTANCE` | `VP9CodecInfo.INSTANCE` | + +## Gateway + +The gateway implementation has been moved to the `internal` package until refactoring for an usable public API is complete. diff --git a/core/src/main/java/moe/kyokobot/koe/KoeEventAdapter.java b/core/src/main/java/moe/kyokobot/koe/KoeEventAdapter.java index 1834980..f3f1938 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeEventAdapter.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeEventAdapter.java @@ -3,6 +3,7 @@ import moe.kyokobot.koe.internal.json.JsonObject; import java.net.InetSocketAddress; +import java.util.List; public class KoeEventAdapter implements KoeEventListener { @Override @@ -21,10 +22,15 @@ public void gatewayClosed(int code, String reason, boolean byRemote) { } @Override - public void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { + public void userStreamsChanged(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { // } + @Override + public void usersConnected(List userIds) { + + } + @Override public void userDisconnected(String id) { // diff --git a/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java b/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java index d877a2a..0d86291 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeEventListener.java @@ -4,17 +4,46 @@ import org.jetbrains.annotations.Nullable; import java.net.InetSocketAddress; +import java.util.List; public interface KoeEventListener { void gatewayError(Throwable cause); void gatewayReady(InetSocketAddress target, int ssrc); + /** + * Called when the gateway connection is closed. + * + * @param code the WebSocket close code. + * @param reason the close reason if present, null otherwise. + * @param byRemote true if the connection was closed by the gateway, false if it was closed by Koe. + */ void gatewayClosed(int code, @Nullable String reason, boolean byRemote); - // TODO: This should actually be called something like userStreamsChanged - void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC); - + /** + * Called when the stream information for a user has changed. Not state tracked by Koe, provides data from {@link moe.kyokobot.koe.gateway.Op#USER_SPEAKING} as-is. + * + * @param id the user ID of the user whose stream information has changed. + * @param audioSSRC the SSRC of the user's audio stream, or 0 if not present or just disabled. + * @param videoSSRC the SSRC of the user's video stream, or 0 if not present or just disabled. + * @param rtxSSRC the SSRC of the user's retransmission stream, or 0 if not present or just disabled. + */ + void userStreamsChanged(String id, int audioSSRC, int videoSSRC, int rtxSSRC); + + /** + * Called when one or more users have connected to the voice channel. + * Not state tracked by Koe, provides data from {@link moe.kyokobot.koe.gateway.Op#CLIENT_CONNECT} as-is. + * + * @param userIds the list of user IDs that have connected, as provided by the gateway. + */ + void usersConnected(List userIds); + + /** + * Called when one or more users have disconnected from the voice channel. + * Not state tracked by Koe, provides data from {@link moe.kyokobot.koe.gateway.Op#CLIENT_DISCONNECT} as-is. + * + * @param userIds the list of user IDs that have disconnected, as provided by the gateway. + */ void userDisconnected(String id); void externalIPDiscovered(InetSocketAddress address); diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java index 2e11d96..35f9c20 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java @@ -11,7 +11,7 @@ public class AEADAES256GCMRTPSizeEncryptionMode implements EncryptionMode { private final byte[] extendedNonce = new byte[NONCE_BYTES_LENGTH]; private final byte[] associatedData = new byte[12]; - private int seq = Math.abs(random.nextInt()) % 418 + 1; + private int seq = Math.abs(SECURE_RANDOM.nextInt()) % 418 + 1; @Override @SuppressWarnings("Duplicates") diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java index e8cacfa..97a1b09 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java @@ -13,7 +13,7 @@ public class AEADXChaCha20Poly1305RTPSizeEncryptionMode implements EncryptionMod private final byte[] extendedNonce = new byte[NONCE_BYTES_LENGTH]; private final ByteBuffer c = ByteBuffer.allocate(1276 + TAG_BYTES_LENGTH + NONCE_BYTES_LENGTH); private final byte[] associatedData = new byte[12]; - private int seq = Math.abs(random.nextInt()) % 418 + 1; + private int seq = Math.abs(SECURE_RANDOM.nextInt()) % 418 + 1; @Override @SuppressWarnings("Duplicates") diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java index cb59ad9..424ba9c 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java @@ -6,7 +6,7 @@ import java.util.List; public interface EncryptionMode { - SecureRandom random = new SecureRandom(); + SecureRandom SECURE_RANDOM = new SecureRandom(); int ZERO_BYTES_LENGTH = 32; // For XSalsa20Poly1305 int TAG_BYTES_LENGTH = 16; // For AEAD diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/GatewayVersion.java b/core/src/main/java/moe/kyokobot/koe/gateway/GatewayVersion.java index 096b430..5126e81 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/GatewayVersion.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/GatewayVersion.java @@ -2,6 +2,10 @@ import moe.kyokobot.koe.VoiceServerInfo; import moe.kyokobot.koe.internal.MediaConnectionImpl; +import moe.kyokobot.koe.internal.gateway.MediaGatewayConnectionFactory; +import moe.kyokobot.koe.internal.gateway.MediaGatewayV4Connection; +import moe.kyokobot.koe.internal.gateway.MediaGatewayV5Connection; +import moe.kyokobot.koe.internal.gateway.MediaGatewayV8Connection; public enum GatewayVersion { V4(MediaGatewayV4Connection::new), @@ -10,8 +14,8 @@ public enum GatewayVersion { private final MediaGatewayConnectionFactory factory; - public MediaGatewayConnection createConnection(MediaConnectionImpl connection, VoiceServerInfo voiceServerInfo) { - return factory.create(connection, voiceServerInfo); + public MediaGatewayConnectionFactory getFactory() { + return factory; } GatewayVersion(MediaGatewayConnectionFactory factory) { diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaValve.java b/core/src/main/java/moe/kyokobot/koe/gateway/MediaValve.java index 3accf38..86c1a47 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaValve.java +++ b/core/src/main/java/moe/kyokobot/koe/gateway/MediaValve.java @@ -1,99 +1,29 @@ package moe.kyokobot.koe.gateway; import moe.kyokobot.koe.internal.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * Tracks the media streams of each user connected to the Voice channel and disables their incoming packets. - */ -public class MediaValve { - private static final Logger LOG = LoggerFactory.getLogger(MediaValve.class.getName()); - - private final AbstractMediaGatewayConnection gatewayConnection; - - /** - * A map of `user_id->streams[*].ssrc` - */ - private final Map unwantedStreams = new HashMap<>(); - - private boolean deafen = false; - - public MediaValve(AbstractMediaGatewayConnection gatewayConnection) { - this.gatewayConnection = gatewayConnection; - } +public interface MediaValve { /** * Whether we're deafened, i.e., we are not receiving audio from any user. */ - public boolean isDeafened() { - return deafen; - } + boolean isDeafened(); /** * Set whether to deafen ourselves. *

* You must call {@link #sendToGateway()} after calling this method to have any effect. */ - public void setDeafen(boolean deafen) { - this.deafen = deafen; - } + void setDeafen(boolean deafen); /** * Send a {@link Op#MEDIA_SINK_WANTS} payload to the gateway. */ - public synchronized void sendToGateway() { - JsonObject d = new JsonObject(); - - // disable all incoming audio streams. - d.add("any", deafen ? 0 : 100); - - // add any unwanted streams. - for (int[] streams : unwantedStreams.values()) { - for (int ssrc : streams) d.add(Integer.toString(ssrc), 0); - } - - this.gatewayConnection.sendInternalPayload(Op.MEDIA_SINK_WANTS, d); - } + void sendToGateway(); /** * Handle a voice gateway message. */ - synchronized void handleEvent(JsonObject obj) { - int op = obj.getInt("op"); - if (op == Op.VIDEO) { - JsonObject d = obj.getObject("d"); - String userId = d.getString("user_id"); - - // if `video_ssrc` is 0, it indicates that the user is not showing their camera. - if (d.getInt("video_ssrc") == 0) { - this.removeUser(userId); - return; - } - - // we can skip `audio_ssrc` since "any":0 covers audio. - // instead, all ssrcs listed in "streams". - int[] ssrcs = d.getArray("streams").stream() - .filter(s -> s instanceof JsonObject) - .mapToInt((stream) -> ((JsonObject) stream).getInt("ssrc")) - .toArray(); - - LOG.debug("Received streams for user {}: {}", d, Arrays.toString(ssrcs)); - - this.unwantedStreams.put(userId, ssrcs); - this.sendToGateway(); - } else if (op == Op.CLIENT_DISCONNECT) { - this.removeUser(obj.getObject("d").getString("user_id")); - } - } + void handleEvent(JsonObject obj); - void removeUser(String userId) { - LOG.debug("Removing streams for user {}", userId); - this.unwantedStreams.remove(userId); - this.sendToGateway(); - } -} \ No newline at end of file + void removeUser(String userId); +} diff --git a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java index 6a5af53..592827d 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java @@ -50,6 +50,12 @@ public int getMaxDAVEProtocolVersion() { return maxProtocolVersion; } + public void addUsers(Iterable userIds) { + for (var uid : userIds) { + addUser(uid); + } + } + public void addUser(String userId) { recognizedUserIds.add(userId); setupKeyRatchetForUser(userId, currentProtocolVersion); diff --git a/core/src/main/java/moe/kyokobot/koe/internal/EventDispatcher.java b/core/src/main/java/moe/kyokobot/koe/internal/EventDispatcher.java index f2604c8..328cd7c 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/EventDispatcher.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/EventDispatcher.java @@ -4,6 +4,7 @@ import moe.kyokobot.koe.internal.json.JsonObject; import java.net.InetSocketAddress; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; @@ -19,7 +20,7 @@ void register(KoeEventListener listener) { if (Objects.requireNonNull(listener) == this) { throw new IllegalArgumentException("Are you trying to register the dispatcher, rly?"); } - + listeners.add(listener); } @@ -49,9 +50,16 @@ public void gatewayClosed(int code, String reason, boolean byRemote) { } @Override - public void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { + public void userStreamsChanged(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { + for (var listener : listeners) { + listener.userStreamsChanged(id, audioSSRC, videoSSRC, rtxSSRC); + } + } + + @Override + public void usersConnected(List userIds) { for (var listener : listeners) { - listener.userConnected(id, audioSSRC, videoSSRC, rtxSSRC); + listener.usersConnected(userIds); } } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java index d9a7e66..8426d80 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/MediaConnectionImpl.java @@ -52,7 +52,8 @@ public CompletionStage connect(VoiceServerInfo info) { this.disconnect(); this.createDAVEManager(); - var conn = client.getGatewayVersion().createConnection(this, info); + var gatewayFactory = client.getGatewayVersion().getFactory(); + var conn = gatewayFactory.create(this, info); return conn.start().thenAccept(nothing -> { MediaConnectionImpl.this.info = info; diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java similarity index 98% rename from core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java rename to core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java index 040272f..e1ece8e 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/AbstractMediaGatewayConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.gateway; +package moe.kyokobot.koe.internal.gateway; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; @@ -18,6 +18,9 @@ import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.EventExecutor; import moe.kyokobot.koe.VoiceServerInfo; +import moe.kyokobot.koe.gateway.CloseCode; +import moe.kyokobot.koe.gateway.MediaGatewayConnection; +import moe.kyokobot.koe.gateway.MediaValve; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.NettyBootstrapFactory; import moe.kyokobot.koe.internal.json.JsonObject; diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnectionFactory.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayConnectionFactory.java similarity index 73% rename from core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnectionFactory.java rename to core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayConnectionFactory.java index cd15172..876d92f 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayConnectionFactory.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayConnectionFactory.java @@ -1,6 +1,7 @@ -package moe.kyokobot.koe.gateway; +package moe.kyokobot.koe.internal.gateway; import moe.kyokobot.koe.VoiceServerInfo; +import moe.kyokobot.koe.gateway.MediaGatewayConnection; import moe.kyokobot.koe.internal.MediaConnectionImpl; @FunctionalInterface diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java similarity index 93% rename from core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java rename to core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java index b4970cc..eb2fd45 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV4Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java @@ -1,9 +1,9 @@ -package moe.kyokobot.koe.gateway; +package moe.kyokobot.koe.internal.gateway; import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.gateway.Op; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; import moe.kyokobot.koe.internal.json.JsonArray; @@ -117,7 +117,17 @@ protected void handlePayload(JsonObject object) { var audioSsrc = data.getInt("audio_ssrc", 0); var videoSsrc = data.getInt("video_ssrc", 0); var rtxSsrc = data.getInt("rtx_ssrc", 0); - connection.getDispatcher().userConnected(user, audioSsrc, videoSsrc, rtxSsrc); + connection.getDispatcher().userStreamsChanged(user, audioSsrc, videoSsrc, rtxSsrc); + break; + } + case Op.CLIENT_CONNECT: { + var data = object.getObject("d"); + var userIds = data.getArray("user_ids"); + + List userIdList = userIds.stream() + .map(o -> (String) o) + .collect(Collectors.toList()); + connection.getDispatcher().usersConnected(userIdList); break; } case Op.CLIENT_DISCONNECT: { diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java similarity index 90% rename from core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java rename to core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java index 416e35b..5483d82 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV5Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java @@ -1,8 +1,10 @@ -package moe.kyokobot.koe.gateway; +package moe.kyokobot.koe.internal.gateway; import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.gateway.MediaValve; +import moe.kyokobot.koe.gateway.Op; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; import moe.kyokobot.koe.internal.json.JsonArray; @@ -22,7 +24,7 @@ public class MediaGatewayV5Connection extends AbstractMediaGatewayConnection { private static final Logger logger = LoggerFactory.getLogger(MediaGatewayV5Connection.class); - private final MediaValve mediaValve = new MediaValve(this); + private final MediaValve mediaValve = new MediaValveImpl(this); private int ssrc; private SocketAddress address; private List encryptionModes; @@ -128,7 +130,23 @@ protected void handlePayload(JsonObject object) { var audioSsrc = data.getInt("audio_ssrc", 0); var videoSsrc = data.getInt("video_ssrc", 0); var rtxSsrc = data.getInt("rtx_ssrc", 0); - connection.getDispatcher().userConnected(user, audioSsrc, videoSsrc, rtxSsrc); + connection.getDispatcher().userStreamsChanged(user, audioSsrc, videoSsrc, rtxSsrc); + break; + } + case Op.CLIENT_CONNECT: { + var data = object.getObject("d"); + var userIds = data.getArray("user_ids"); + + List userIdList = userIds.stream() + .map(o -> (String) o) + .collect(Collectors.toList()); + connection.getDispatcher().usersConnected(userIdList); + +// var manager = connection.getDAVEManager(); // TODO: we don't have MLS implemented in v5 +// if (manager != null) { +// manager.addUsers(userIdList); +// } + break; } case Op.CLIENT_DISCONNECT: { @@ -150,6 +168,7 @@ protected void handlePayload(JsonObject object) { break; } + // TODO: MLS default: break; } diff --git a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java similarity index 96% rename from core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java rename to core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java index 98765bf..c486584 100644 --- a/core/src/main/java/moe/kyokobot/koe/gateway/MediaGatewayV8Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java @@ -1,8 +1,10 @@ -package moe.kyokobot.koe.gateway; +package moe.kyokobot.koe.internal.gateway; import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.gateway.MediaValve; +import moe.kyokobot.koe.gateway.Op; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; import moe.kyokobot.koe.internal.json.JsonArray; @@ -22,7 +24,7 @@ public class MediaGatewayV8Connection extends AbstractMediaGatewayConnection { private static final Logger logger = LoggerFactory.getLogger(MediaGatewayV8Connection.class); - private final MediaValve mediaValve = new MediaValve(this); + private final MediaValve mediaValve = new MediaValveImpl(this); private int ssrc; private SocketAddress address; private List encryptionModes; @@ -149,16 +151,21 @@ protected void handlePayload(JsonObject object) { var audioSsrc = data.getInt("audio_ssrc", 0); var videoSsrc = data.getInt("video_ssrc", 0); var rtxSsrc = data.getInt("rtx_ssrc", 0); - connection.getDispatcher().userConnected(user, audioSsrc, videoSsrc, rtxSsrc); + connection.getDispatcher().userStreamsChanged(user, audioSsrc, videoSsrc, rtxSsrc); break; } case Op.CLIENT_CONNECT: { var data = object.getObject("d"); var userIds = data.getArray("user_ids"); + List userIdList = userIds.stream() + .map(o -> (String) o) + .collect(Collectors.toList()); + connection.getDispatcher().usersConnected(userIdList); + var manager = connection.getDAVEManager(); if (manager != null) { - userIds.forEach(userId -> manager.addUser((String) userId)); + manager.addUsers(userIdList); } break; diff --git a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaValveImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaValveImpl.java new file mode 100644 index 0000000..85d54a1 --- /dev/null +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaValveImpl.java @@ -0,0 +1,87 @@ +package moe.kyokobot.koe.internal.gateway; + +import moe.kyokobot.koe.gateway.MediaValve; +import moe.kyokobot.koe.gateway.Op; +import moe.kyokobot.koe.internal.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * Tracks the media streams of each user connected to the Voice channel and disables their incoming packets. + */ +public class MediaValveImpl implements MediaValve { + private static final Logger LOG = LoggerFactory.getLogger(MediaValve.class.getName()); + + private final AbstractMediaGatewayConnection gatewayConnection; + + /** + * A map of `user_id->streams[*].ssrc` + */ + private final Map unwantedStreams = new HashMap<>(); + + private boolean deafen = false; + + public MediaValveImpl(AbstractMediaGatewayConnection gatewayConnection) { + this.gatewayConnection = gatewayConnection; + } + + @Override public boolean isDeafened() { + return deafen; + } + + @Override public void setDeafen(boolean deafen) { + this.deafen = deafen; + } + + @Override public synchronized void sendToGateway() { + JsonObject d = new JsonObject(); + + // disable all incoming audio streams. + d.add("any", deafen ? 0 : 100); + + // add any unwanted streams. + for (int[] streams : unwantedStreams.values()) { + for (int ssrc : streams) d.add(Integer.toString(ssrc), 0); + } + + this.gatewayConnection.sendInternalPayload(Op.MEDIA_SINK_WANTS, d); + } + + @Override public synchronized void handleEvent(JsonObject obj) { + int op = obj.getInt("op"); + if (op == Op.VIDEO) { + JsonObject d = obj.getObject("d"); + String userId = d.getString("user_id"); + + // if `video_ssrc` is 0, it indicates that the user is not showing their camera. + if (d.getInt("video_ssrc") == 0) { + this.removeUser(userId); + return; + } + + // we can skip `audio_ssrc` since "any":0 covers audio. + // instead, all ssrcs listed in "streams". + int[] ssrcs = d.getArray("streams").stream() + .filter(s -> s instanceof JsonObject) + .mapToInt((stream) -> ((JsonObject) stream).getInt("ssrc")) + .toArray(); + + LOG.debug("Received streams for user {}: {}", d, Arrays.toString(ssrcs)); + + this.unwantedStreams.put(userId, ssrcs); + this.sendToGateway(); + } else if (op == Op.CLIENT_DISCONNECT) { + this.removeUser(obj.getObject("d").getString("user_id")); + } + } + + @Override public void removeUser(String userId) { + LOG.debug("Removing streams for user {}", userId); + this.unwantedStreams.remove(userId); + this.sendToGateway(); + } +} diff --git a/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java b/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java index 0b63b59..d1a8f6b 100644 --- a/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java +++ b/core/src/main/java/moe/kyokobot/koe/poller/AbstractOpusFramePoller.java @@ -151,7 +151,7 @@ private void setSpeaking(boolean state) { private class Op12HackListener extends KoeEventAdapter { @Override - public void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { + public void userStreamsChanged(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { if (speaking) { connection.updateSpeakingState(speakingMask); } diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java index 564cbcc..a04da94 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java @@ -263,7 +263,7 @@ public void dispose() { private static class ExampleListener extends KoeEventAdapter { @Override - public void userConnected(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { + public void userStreamsChanged(String id, int audioSSRC, int videoSSRC, int rtxSSRC) { logger.info("An user with id {} joined the channel!", id); } From f0968dec9ce17442382d7babe6791fc40b5852ab Mon Sep 17 00:00:00 2001 From: Alula Date: Sat, 21 Feb 2026 19:59:46 +0100 Subject: [PATCH 12/28] Internalize crypto API + actually make DAVE work --- .../koe/handler/ConnectionHandler.java | 6 ++-- .../kyokobot/koe/internal/DAVEManager.java | 16 ++++++---- .../AEADAES256GCMRTPSizeEncryptionMode.java | 6 ++-- ...ChaCha20Poly1305RTPSizeEncryptionMode.java | 6 ++-- .../crypto/DefaultEncryptionModes.java | 2 +- .../{ => internal}/crypto/EncryptionMode.java | 4 +-- .../crypto/PlainEncryptionMode.java | 8 ++--- .../UnsupportedEncryptionModeException.java | 2 +- .../XSalsa20Poly1305EncryptionMode.java | 7 ++-- .../XSalsa20Poly1305LiteEncryptionMode.java | 7 ++-- .../XSalsa20Poly1305SuffixEncryptionMode.java | 7 ++-- .../gateway/MediaGatewayV4Connection.java | 2 +- .../gateway/MediaGatewayV5Connection.java | 2 +- .../gateway/MediaGatewayV8Connection.java | 2 +- .../handler/DiscordUDPConnection.java | 32 ++++++++++++++++--- .../poller/netty/NettyOpusFramePoller.java | 2 +- .../udpqueue/UdpQueueOpusFramePoller.java | 2 +- testbot/build.gradle.kts | 2 +- .../moe/kyokobot/koe/testbot/TestBot.java | 2 +- 19 files changed, 70 insertions(+), 47 deletions(-) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/AEADAES256GCMRTPSizeEncryptionMode.java (89%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java (90%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/DefaultEncryptionModes.java (97%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/EncryptionMode.java (88%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/PlainEncryptionMode.java (52%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/UnsupportedEncryptionModeException.java (81%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/XSalsa20Poly1305EncryptionMode.java (81%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/XSalsa20Poly1305LiteEncryptionMode.java (84%) rename core/src/main/java/moe/kyokobot/koe/{ => internal}/crypto/XSalsa20Poly1305SuffixEncryptionMode.java (83%) diff --git a/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java b/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java index 291c463..cf5a4cc 100644 --- a/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java +++ b/core/src/main/java/moe/kyokobot/koe/handler/ConnectionHandler.java @@ -2,7 +2,9 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.codec.CodecInstance; +import moe.kyokobot.koe.codec.CodecType; import moe.kyokobot.koe.internal.json.JsonObject; +import moe.kyokobot.libdave.MediaType; import java.util.concurrent.CompletionStage; @@ -19,10 +21,10 @@ public interface ConnectionHandler { void handleSessionDescription(JsonObject object); default void sendFrame(CodecInstance codec, int timestamp, ByteBuf data, int start) { - sendFrame(codec.getPayloadType(), timestamp, data, start, false); + sendFrame(codec.getType(), codec.getPayloadType(), timestamp, data, start, false); } CompletionStage connect(); - void sendFrame(byte payloadType, int timestamp, ByteBuf data, int start, boolean extension); + void sendFrame(CodecType codecType, byte payloadType, int timestamp, ByteBuf data, int start, boolean extension); } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java index 592827d..fd911c3 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java @@ -3,9 +3,7 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.codec.OpusCodecInfo; import moe.kyokobot.koe.internal.json.JsonObject; -import moe.kyokobot.libdave.KeyRatchet; -import moe.kyokobot.libdave.MediaType; -import moe.kyokobot.libdave.Session; +import moe.kyokobot.libdave.*; import moe.kyokobot.libdave.netty.NettyDaveFactory; import moe.kyokobot.libdave.netty.NettyEncryptor; import org.jetbrains.annotations.NotNull; @@ -50,6 +48,10 @@ public int getMaxDAVEProtocolVersion() { return maxProtocolVersion; } + public int getMaxCiphertextByteSize(MediaType mediaType, int frameSize) { + return selfEncryptor.getMaxCiphertextByteSize(mediaType, frameSize); + } + public void addUsers(Iterable userIds) { for (var uid : userIds) { addUser(uid); @@ -65,7 +67,7 @@ public void removeUser(String userId) { recognizedUserIds.remove(userId); } - public ByteBuf encrypt(MediaType mediaType, int ssrc, ByteBuf output, ByteBuf input, int size) { + public int encrypt(MediaType mediaType, int ssrc, ByteBuf output, ByteBuf input, int size) { if (mediaType == MediaType.AUDIO && size == 3) { input.markReaderIndex(); var b1 = input.readByte() == OpusCodecInfo.SILENCE_FRAME[0]; @@ -75,13 +77,13 @@ public ByteBuf encrypt(MediaType mediaType, int ssrc, ByteBuf output, ByteBuf in input.resetReaderIndex(); if (isSilence) { - return input; + output.writeBytes(OpusCodecInfo.SILENCE_FRAME); + return EncryptorResultCode.SUCCESS.getValue(); } } output.ensureWritable(this.selfEncryptor.getMaxCiphertextByteSize(mediaType, size)); - var _result = this.selfEncryptor.encrypt(mediaType, ssrc, input, output); - return output; + return this.selfEncryptor.encrypt(mediaType, ssrc, input, output); } public void handleSessionDescription(@NotNull JsonObject session, long mlsGroupId) { diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/AEADAES256GCMRTPSizeEncryptionMode.java similarity index 89% rename from core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/AEADAES256GCMRTPSizeEncryptionMode.java index 35f9c20..cb2406f 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/AEADAES256GCMRTPSizeEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/AEADAES256GCMRTPSizeEncryptionMode.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import com.google.crypto.tink.aead.internal.InsecureNonceAesGcmJce; import io.netty.buffer.ByteBuf; @@ -15,9 +15,9 @@ public class AEADAES256GCMRTPSizeEncryptionMode implements EncryptionMode { @Override @SuppressWarnings("Duplicates") - public boolean box(ByteBuf packet, int len, ByteBuf output, byte[] secretKey) { + public boolean box(ByteBuf plain, int len, ByteBuf output, byte[] secretKey) { var m = new byte[len]; - packet.readBytes(m, 0, len); + plain.readBytes(m, 0, len); var s = this.seq++; extendedNonce[0] = (byte) (s & 0xff); diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java similarity index 90% rename from core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java index 97a1b09..091ce28 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/AEADXChaCha20Poly1305RTPSizeEncryptionMode.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import com.google.crypto.tink.aead.internal.InsecureNonceXChaCha20Poly1305; import io.netty.buffer.ByteBuf; @@ -17,9 +17,9 @@ public class AEADXChaCha20Poly1305RTPSizeEncryptionMode implements EncryptionMod @Override @SuppressWarnings("Duplicates") - public boolean box(ByteBuf packet, int len, ByteBuf output, byte[] secretKey) { + public boolean box(ByteBuf plain, int len, ByteBuf output, byte[] secretKey) { var m = new byte[len]; - packet.readBytes(m, 0, len); + plain.readBytes(m, 0, len); var s = this.seq++; extendedNonce[0] = (byte) (s & 0xff); diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/DefaultEncryptionModes.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/DefaultEncryptionModes.java similarity index 97% rename from core/src/main/java/moe/kyokobot/koe/crypto/DefaultEncryptionModes.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/DefaultEncryptionModes.java index ec59135..4aa5f26 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/DefaultEncryptionModes.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/DefaultEncryptionModes.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import java.security.Security; import java.util.Collections; diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/EncryptionMode.java similarity index 88% rename from core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/EncryptionMode.java index 424ba9c..b41eda8 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/EncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/EncryptionMode.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import io.netty.buffer.ByteBuf; @@ -11,7 +11,7 @@ public interface EncryptionMode { int ZERO_BYTES_LENGTH = 32; // For XSalsa20Poly1305 int TAG_BYTES_LENGTH = 16; // For AEAD - boolean box(ByteBuf opus, int start, ByteBuf output, byte[] secretKey); + boolean box(ByteBuf plain, int start, ByteBuf output, byte[] secretKey); String getName(); diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/PlainEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/PlainEncryptionMode.java similarity index 52% rename from core/src/main/java/moe/kyokobot/koe/crypto/PlainEncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/PlainEncryptionMode.java index a842078..fe6ec4d 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/PlainEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/PlainEncryptionMode.java @@ -1,12 +1,12 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import io.netty.buffer.ByteBuf; public class PlainEncryptionMode implements EncryptionMode { @Override - public boolean box(ByteBuf opus, int start, ByteBuf output, byte[] secretKey) { - opus.readerIndex(start); - output.writeBytes(opus); + public boolean box(ByteBuf plain, int start, ByteBuf output, byte[] secretKey) { + plain.readerIndex(start); + output.writeBytes(plain); return true; } diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/UnsupportedEncryptionModeException.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/UnsupportedEncryptionModeException.java similarity index 81% rename from core/src/main/java/moe/kyokobot/koe/crypto/UnsupportedEncryptionModeException.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/UnsupportedEncryptionModeException.java index 842b24f..092385c 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/UnsupportedEncryptionModeException.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/UnsupportedEncryptionModeException.java @@ -1,4 +1,4 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; public class UnsupportedEncryptionModeException extends IllegalArgumentException { public UnsupportedEncryptionModeException(String message) { diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305EncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305EncryptionMode.java similarity index 81% rename from core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305EncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305EncryptionMode.java index ccd53d8..6ebb2c6 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305EncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305EncryptionMode.java @@ -1,7 +1,6 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.internal.crypto.TweetNaclFastInstanced; public class XSalsa20Poly1305EncryptionMode implements EncryptionMode { private final byte[] extendedNonce = new byte[24]; @@ -11,14 +10,14 @@ public class XSalsa20Poly1305EncryptionMode implements EncryptionMode { @Override @SuppressWarnings("Duplicates") - public boolean box(ByteBuf packet, int len, ByteBuf output, byte[] secretKey) { + public boolean box(ByteBuf plain, int len, ByteBuf output, byte[] secretKey) { for (int i = 0; i < c.length; i++) { m[i] = 0; c[i] = 0; } for (int i = 0; i < len; i++) { - m[i + 32] = packet.readByte(); + m[i + 32] = plain.readByte(); } output.getBytes(0, extendedNonce, 0, 12); diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305LiteEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305LiteEncryptionMode.java similarity index 84% rename from core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305LiteEncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305LiteEncryptionMode.java index 6dfd0f1..3250c0f 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305LiteEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305LiteEncryptionMode.java @@ -1,7 +1,6 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.internal.crypto.TweetNaclFastInstanced; public class XSalsa20Poly1305LiteEncryptionMode implements EncryptionMode { private final byte[] extendedNonce = new byte[24]; @@ -12,14 +11,14 @@ public class XSalsa20Poly1305LiteEncryptionMode implements EncryptionMode { @Override @SuppressWarnings("Duplicates") - public boolean box(ByteBuf packet, int len, ByteBuf output, byte[] secretKey) { + public boolean box(ByteBuf plain, int len, ByteBuf output, byte[] secretKey) { for (int i = 0; i < c.length; i++) { m[i] = 0; c[i] = 0; } for (int i = 0; i < len; i++) { - m[i + 32] = packet.readByte(); + m[i + 32] = plain.readByte(); } int s = this.seq++; diff --git a/core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305SuffixEncryptionMode.java b/core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305SuffixEncryptionMode.java similarity index 83% rename from core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305SuffixEncryptionMode.java rename to core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305SuffixEncryptionMode.java index db7fb95..5cb82a4 100644 --- a/core/src/main/java/moe/kyokobot/koe/crypto/XSalsa20Poly1305SuffixEncryptionMode.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/crypto/XSalsa20Poly1305SuffixEncryptionMode.java @@ -1,7 +1,6 @@ -package moe.kyokobot.koe.crypto; +package moe.kyokobot.koe.internal.crypto; import io.netty.buffer.ByteBuf; -import moe.kyokobot.koe.internal.crypto.TweetNaclFastInstanced; import java.util.concurrent.ThreadLocalRandom; @@ -13,14 +12,14 @@ public class XSalsa20Poly1305SuffixEncryptionMode implements EncryptionMode { @Override @SuppressWarnings("Duplicates") - public boolean box(ByteBuf packet, int len, ByteBuf output, byte[] secretKey) { + public boolean box(ByteBuf plain, int len, ByteBuf output, byte[] secretKey) { for (int i = 0; i < c.length; i++) { m[i] = 0; c[i] = 0; } for (int i = 0; i < len; i++) { - m[i + 32] = packet.readByte(); + m[i + 32] = plain.readByte(); } ThreadLocalRandom.current().nextBytes(extendedNonce); diff --git a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java index eb2fd45..667b260 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV4Connection.java @@ -2,7 +2,7 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.internal.crypto.EncryptionMode; import moe.kyokobot.koe.gateway.Op; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.handler.DiscordUDPConnection; diff --git a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java index 5483d82..387e137 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV5Connection.java @@ -2,7 +2,7 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.internal.crypto.EncryptionMode; import moe.kyokobot.koe.gateway.MediaValve; import moe.kyokobot.koe.gateway.Op; import moe.kyokobot.koe.internal.MediaConnectionImpl; diff --git a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java index c486584..4dd32e8 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/MediaGatewayV8Connection.java @@ -2,7 +2,7 @@ import io.netty.buffer.ByteBuf; import moe.kyokobot.koe.VoiceServerInfo; -import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.internal.crypto.EncryptionMode; import moe.kyokobot.koe.gateway.MediaValve; import moe.kyokobot.koe.gateway.Op; import moe.kyokobot.koe.internal.MediaConnectionImpl; diff --git a/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java b/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java index e4f3c34..3761b0f 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/handler/DiscordUDPConnection.java @@ -8,12 +8,15 @@ import moe.kyokobot.koe.codec.CodecInfo; import moe.kyokobot.koe.codec.CodecInstance; import moe.kyokobot.koe.codec.CodecRegistry; -import moe.kyokobot.koe.crypto.EncryptionMode; +import moe.kyokobot.koe.codec.CodecType; import moe.kyokobot.koe.handler.ConnectionHandler; import moe.kyokobot.koe.internal.MediaConnectionImpl; import moe.kyokobot.koe.internal.NettyBootstrapFactory; +import moe.kyokobot.koe.internal.crypto.EncryptionMode; import moe.kyokobot.koe.internal.json.JsonObject; import moe.kyokobot.koe.internal.util.RTPHeaderWriter; +import moe.kyokobot.libdave.EncryptorResultCode; +import moe.kyokobot.libdave.MediaType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,31 +111,50 @@ public void handleSessionDescription(JsonObject object) { } @Override - public void sendFrame(byte payloadType, int timestamp, ByteBuf data, int len, boolean extension) { - var buf = createPacket(payloadType, timestamp, data, len, extension); + public void sendFrame(CodecType codecType, byte payloadType, int timestamp, ByteBuf data, int len, boolean extension) { + var buf = createPacket(codecType, payloadType, timestamp, data, len, extension); if (buf != null) { channel.writeAndFlush(buf); } } - public ByteBuf createPacket(byte payloadType, int timestamp, ByteBuf data, int len, boolean extension) { + public ByteBuf createPacket(CodecType codecType, byte payloadType, int timestamp, ByteBuf data, int len, boolean extension) { if (secretKey == null) { return null; } + var mediaType = codecType == CodecType.AUDIO ? MediaType.AUDIO : MediaType.VIDEO; + + var inputBuffer = data; + int inputLen = len; + var buf = allocator.buffer(); buf.clear(); var dave = connection.getDAVEManager(); if (dave != null) { + inputBuffer = allocator.buffer(); + var result = dave.encrypt(mediaType, ssrc, inputBuffer, data, len); + inputLen = inputBuffer.readableBytes(); + + if (result < 0) { + logger.debug("DAVE encryption failed with code {}", result); + buf.release(); + inputBuffer.release(); + return null; + } + } else { + inputBuffer.retain(); } RTPHeaderWriter.writeV2(buf, payloadType, nextSeq(), timestamp, ssrc, extension); - if (encryptionMode.box(data, len, buf, secretKey)) { + if (encryptionMode.box(inputBuffer, inputLen, buf, secretKey)) { + inputBuffer.release(); return buf; } else { logger.debug("Encryption failed!"); buf.release(); + inputBuffer.release(); // TODO: handle failed encryption? } diff --git a/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java index 17e3963..442a64d 100644 --- a/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java +++ b/core/src/main/java/moe/kyokobot/koe/poller/netty/NettyOpusFramePoller.java @@ -23,6 +23,6 @@ protected void sendFramePayload(ByteBuf buf, int len, int timestamp) { if (handler == null) { return; } - handler.sendFrame(codec.getPayloadType(), timestamp, buf, len, false); + handler.sendFrame(codec, timestamp, buf, len); } } diff --git a/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java index 9b7180e..b783eab 100644 --- a/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java +++ b/ext-udpqueue/src/main/java/moe/kyokobot/koe/poller/udpqueue/UdpQueueOpusFramePoller.java @@ -37,7 +37,7 @@ protected void sendFramePayload(ByteBuf buf, int len, int timestamp) { } var handler = (DiscordUDPConnection) connectionHandler; - var packet = handler.createPacket(codec.getPayloadType(), timestamp, buf, len, false); + var packet = handler.createPacket(codec.getType(), codec.getPayloadType(), timestamp, buf, len, false); if (packet == null) { return; } diff --git a/testbot/build.gradle.kts b/testbot/build.gradle.kts index ea09141..4616b07 100644 --- a/testbot/build.gradle.kts +++ b/testbot/build.gradle.kts @@ -5,7 +5,7 @@ dependencies { implementation("dev.arbjerg:lavaplayer:2.2.6") implementation("com.github.lavalink-devs:lavaplayer-youtube-source:1.16.0") implementation("ch.qos.logback:logback-classic:1.5.18") - + implementation("moe.kyokobot.libdave:natives-linux-x86-64:1.0-SNAPSHOT") implementation("club.minnced:udpqueue-native-linux-x86-64:0.2.12") implementation("club.minnced:udpqueue-native-win-x86-64:0.2.12") diff --git a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java index a04da94..5521cca 100644 --- a/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java +++ b/testbot/src/main/java/moe/kyokobot/koe/testbot/TestBot.java @@ -66,7 +66,7 @@ public void start() { this.jda = createJDA(); var options = configureKoe(KoeOptions.builder() .setByteBufAllocator(this.leakDetect.getAllocator()) - .setDAVEEnabled(false) // TODO: fix segfaults +// .setDAVEEnabled(false) // TODO: fix segfaults ); this.koe = Koe.koe(options); this.playerManager = createAudioPlayerManager(); From 45fea56e545546fa27726a9639987dde8469e235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?To=CF=80?= Date: Sun, 22 Feb 2026 01:37:42 +0100 Subject: [PATCH 13/28] add github actions to publish koe (#42) --- .github/workflows/ci.yml | 47 +++++++++++ build.gradle.kts | 149 ++++++++++++++++++++++------------ core/build.gradle.kts | 27 ++++-- ext-udpqueue/build.gradle.kts | 14 +++- jitpack.yml | 1 - settings.gradle.kts | 34 +++++++- testbot/build.gradle.kts | 20 +++-- 7 files changed, 217 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 jitpack.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..888744d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: Publish +on: + push: + branches: [ '**' ] + paths-ignore: + - '**.md' + release: + types: [ published ] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Java + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: 17 + cache: gradle + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v5 + + - name: Build and Publish + run: | + ./gradlew build publish \ + --no-daemon \ + -PMAVEN_USERNAME=${{ secrets.MAVEN_USERNAME }} \ + -PMAVEN_PASSWORD=${{ secrets.MAVEN_PASSWORD }} \ + -PMAVEN_CENTRAL_USERNAME=${{ secrets.MAVEN_CENTRAL_USERNAME }} \ + -PMAVEN_CENTRAL_PASSWORD=${{ secrets.MAVEN_CENTRAL_PASSWORD }} + + - name: Upload core Artifact + uses: actions/upload-artifact@v6 + with: + name: core + path: core/build/libs/core-*.jar + + - name: Upload ext-udpqueue Artifact + uses: actions/upload-artifact@v6 + with: + name: ext-udpqueue + path: ext-udpqueue/build/libs/ext-udpqueue-*.jar diff --git a/build.gradle.kts b/build.gradle.kts index c01ae38..34a472e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,49 +1,25 @@ +import com.vanniktech.maven.publish.MavenPublishBaseExtension +import com.vanniktech.maven.publish.SonatypeHost import java.io.ByteArrayOutputStream -data class VersionInfo(val version: String, val isCommitHash: Boolean) - -fun getGitVersion(): VersionInfo { - var versionStr = ByteArrayOutputStream() - val result = exec { - standardOutput = versionStr - errorOutput = versionStr - isIgnoreExitValue = true - commandLine("git", "describe", "--exact-match", "--tags") - } - if (result.exitValue == 0) { - return VersionInfo(versionStr.toString().trim(), false) - } - - - versionStr = ByteArrayOutputStream() - exec { - standardOutput = versionStr - errorOutput = versionStr - commandLine("git", "rev-parse", "--short", "HEAD") - } - - return VersionInfo(versionStr.toString().trim(), true) +plugins { + id("com.vanniktech.maven.publish") version "0.32.0" apply false } +val gitVersionInfo = getGitVersion() +logger.lifecycle("Version: ${gitVersionInfo.version} (isCommitHash: ${gitVersionInfo.isCommitHash})") + subprojects { - apply(plugin = "maven-publish") apply(plugin = "java-library") group = "moe.kyokobot.koe" - if (project.hasProperty("version")) { - // Used by JitPack - version = project.version - } else { - // Get version from git - val gitVersion = getGitVersion() - version = "3.0+git" + gitVersion.version - } + version = gitVersionInfo.version configure { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 - + withSourcesJar() withJavadocJar() } @@ -55,37 +31,108 @@ subprojects { maven { url = uri("https://maven.lavalink.dev/releases") } + maven { + url = uri("https://maven.lavalink.dev/snapshots") + } maven { url = uri("https://jitpack.io/") } } + if (name != "testbot") { + apply(plugin = "com.vanniktech.maven.publish") + + afterEvaluate { + plugins.withId("com.vanniktech.maven.publish.base") { + configure { + val mavenUsername = findProperty("MAVEN_USERNAME") as String? + val mavenPassword = findProperty("MAVEN_PASSWORD") as String? + if (!mavenUsername.isNullOrEmpty() && !mavenPassword.isNullOrEmpty()) { + repositories { + val snapshots = "https://maven.lavalink.dev/snapshots" + val releases = "https://maven.lavalink.dev/releases" - configure { - publications { - create("mavenJava") { - from(components["java"]) - pom { - url.set("https://github.com/KyokoBot/koe") - licenses { - license { - name.set("The MIT License") - url.set("http://www.opensource.org/licenses/mit-license.php") + maven(if (gitVersionInfo.isCommitHash) snapshots else releases) { + credentials { + username = mavenUsername + password = mavenPassword + } + } } + } else { + logger.lifecycle("Not publishing to maven.lavalink.dev because credentials are not set") } - developers { - developer { - id.set("alula") - name.set("Alula") - email.set("git@alula.me") + } + + configure { + coordinates(group.toString(), project.the().archivesName.get(), version.toString()) + val mavenCentralUsername = findProperty("MAVEN_CENTRAL_USERNAME") as String? + val mavenCentralPassword = findProperty("MAVEN_CENTRAL_PASSWORD") as String? + if (!mavenCentralUsername.isNullOrEmpty() && !mavenCentralPassword.isNullOrEmpty()) { + publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL, false) + if (!gitVersionInfo.isCommitHash) { + signAllPublications() } + } else { + logger.lifecycle("Not publishing to OSSRH due to missing credentials") } - scm { - connection.set("scm:git:https://github.com/KyokoBot/koe.git") - developerConnection.set("scm:git:ssh://github.com:KyokoBot/koe.git") + + pom { url.set("https://github.com/KyokoBot/koe") + licenses { + license { + name.set("The MIT License") + url.set("http://www.opensource.org/licenses/mit-license.php") + } + } + developers { + developer { + id.set("alula") + name.set("Alula") + email.set("git@alula.me") + } + } + scm { + connection.set("scm:git:https://github.com/KyokoBot/koe.git") + developerConnection.set("scm:git:ssh://github.com:KyokoBot/koe.git") + url.set("https://github.com/KyokoBot/koe") + } } } } } + + configure { + publications { + create("mavenJava") { + from(components["java"]) + + } + } + } + } +} + +data class VersionInfo(val version: String, val isCommitHash: Boolean) + +fun getGitVersion(): VersionInfo { + var versionStr = ByteArrayOutputStream() + val result = exec { + standardOutput = versionStr + errorOutput = versionStr + isIgnoreExitValue = true + commandLine("git", "describe", "--exact-match", "--tags") + } + if (result.exitValue == 0) { + return VersionInfo(versionStr.toString().trim(), false) + } + + + versionStr = ByteArrayOutputStream() + exec { + standardOutput = versionStr + errorOutput = versionStr + commandLine("git", "rev-parse", "--short", "HEAD") } + + return VersionInfo(versionStr.toString().trim(), true) } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 725ac68..662d084 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -1,10 +1,21 @@ dependencies { - api("io.netty:netty-transport:4.1.112.Final") - implementation("io.netty:netty-codec-http:4.1.112.Final") - implementation("io.netty:netty-transport-native-epoll:4.1.112.Final:linux-x86_64") - implementation("org.slf4j:slf4j-api:1.8.0-beta4") - implementation("com.google.crypto.tink:tink:1.14.1") - implementation("moe.kyokobot.libdave:api:1.0-SNAPSHOT") - implementation("moe.kyokobot.libdave:impl-jni:1.0-SNAPSHOT") - compileOnly("org.jetbrains:annotations:13.0") + api(libs.netty.transport) + implementation(libs.netty.codec.http) + implementation(libs.netty.transport.native.epoll.linux) { + artifact { + classifier = "linux-x86_64" + } + } + + implementation(libs.slf4j.api) + implementation(libs.tink) + implementation(libs.libdave.api) + implementation(libs.libdave.impl.jni) + implementation(libs.jetbrains.annotations) +} + +mavenPublishing { + pom { + name = "core" + } } diff --git a/ext-udpqueue/build.gradle.kts b/ext-udpqueue/build.gradle.kts index ed9ac3e..e42a7cf 100644 --- a/ext-udpqueue/build.gradle.kts +++ b/ext-udpqueue/build.gradle.kts @@ -1,6 +1,12 @@ dependencies { - compileOnly(project(":core")) - implementation("dev.arbjerg:lava-common:2.2.1") - implementation("club.minnced:udpqueue-api:0.2.9") - compileOnly("org.jetbrains:annotations:13.0") + compileOnly(projects.core) + implementation(libs.lava.common) + implementation(libs.udpqueue.api) + implementation(libs.jetbrains.annotations) +} + +mavenPublishing { + pom { + name = "ext-udpqueue" + } } diff --git a/jitpack.yml b/jitpack.yml deleted file mode 100644 index 88cfba7..0000000 --- a/jitpack.yml +++ /dev/null @@ -1 +0,0 @@ -jdk: openjdk11 diff --git a/settings.gradle.kts b/settings.gradle.kts index 0cc67d7..46fc158 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,10 +4,40 @@ include(":core") include(":ext-udpqueue") include(":testbot") +enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") + dependencyResolutionManagement { versionCatalogs { - create("netty") { - version("netty", "4.2.9.Final") + create("libs") { + version("netty", "4.1.112.Final") + library("netty-transport", "io.netty", "netty-transport").versionRef("netty") + library("netty-codec-http", "io.netty", "netty-codec-http").versionRef("netty") + library("netty-transport-native-epoll-linux", "io.netty", "netty-transport-native-epoll").versionRef("netty") + + library("tink", "com.google.crypto.tink", "tink").version("1.14.1") + + library("jetbrains-annotations", "org.jetbrains", "annotations").version("13.0") + + library("slf4j-api", "org.slf4j", "slf4j-api").version("1.8.0-beta4") + library("logback-classic", "ch.qos.logback", "logback-classic").version("1.5.18") + + version("lavaplayer", "2.2.6") + library("lava-common", "dev.arbjerg", "lava-common").versionRef("lavaplayer") + library("lavaplayer", "dev.arbjerg", "lavaplayer").versionRef("lavaplayer") + library("lavaplayer-youtube", "com.github.lavalink-devs", "lavaplayer-youtube-source").version("1.16.0") + + library("jda", "net.dv8tion", "JDA").version("5.0.2") + + version("libdave", "cfa4755") + library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") + library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") + library("libdave-natives-linux", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") + library("libdave-natives-windows", "moe.kyokobot.libdave", "natives-win-x86-64").versionRef("libdave") + + version("udpqueue", "0.2.12") + library("udpqueue-api", "club.minnced", "udpqueue-api").versionRef("udpqueue") + library("udpqueue-native-linux", "club.minnced", "udpqueue-native-linux-x86-64").versionRef("udpqueue") + library("udpqueue-native-win", "club.minnced", "udpqueue-native-win-x86-64").versionRef("udpqueue") } } } diff --git a/testbot/build.gradle.kts b/testbot/build.gradle.kts index 4616b07..9fef593 100644 --- a/testbot/build.gradle.kts +++ b/testbot/build.gradle.kts @@ -1,12 +1,14 @@ dependencies { - implementation(project(":core")) - implementation(project(":ext-udpqueue")) - implementation("net.dv8tion:JDA:5.0.2") - implementation("dev.arbjerg:lavaplayer:2.2.6") - implementation("com.github.lavalink-devs:lavaplayer-youtube-source:1.16.0") - implementation("ch.qos.logback:logback-classic:1.5.18") + implementation(projects.core) + implementation(projects.extUdpqueue) + implementation(libs.jda) + implementation(libs.lavaplayer) + implementation(libs.lavaplayer.youtube) + implementation(libs.logback.classic) - implementation("moe.kyokobot.libdave:natives-linux-x86-64:1.0-SNAPSHOT") - implementation("club.minnced:udpqueue-native-linux-x86-64:0.2.12") - implementation("club.minnced:udpqueue-native-win-x86-64:0.2.12") + implementation(libs.libdave.natives.linux) + implementation(libs.libdave.natives.windows) + + implementation(libs.udpqueue.native.linux) + implementation(libs.udpqueue.native.win) } From 3a5613e0a5c9051fa7ec54a548a7484a502ff8b7 Mon Sep 17 00:00:00 2001 From: topi314 Date: Sun, 22 Feb 2026 01:42:43 +0100 Subject: [PATCH 14/28] remove duplicate publication --- build.gradle.kts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 34a472e..1f6f057 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -100,15 +100,6 @@ subprojects { } } } - - configure { - publications { - create("mavenJava") { - from(components["java"]) - - } - } - } } } From 81c80bdaca706fb303421dc907a15964aa7cf117 Mon Sep 17 00:00:00 2001 From: topi314 Date: Sun, 22 Feb 2026 01:46:46 +0100 Subject: [PATCH 15/28] remove explicit sources & javadoc jar --- build.gradle.kts | 3 --- 1 file changed, 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 1f6f057..90bf7cd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,9 +19,6 @@ subprojects { configure { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 - - withSourcesJar() - withJavadocJar() } repositories { From a82691e3fee11f5bead33bacd9e6664705d7f883 Mon Sep 17 00:00:00 2001 From: Alula Date: Mon, 23 Feb 2026 01:02:45 +0100 Subject: [PATCH 16/28] Fix passthrough mode --- .../kyokobot/koe/internal/DAVEManager.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java index fd911c3..badfef0 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/DAVEManager.java @@ -40,6 +40,7 @@ public class DAVEManager implements AutoCloseable { this.factory = factory; this.daveSession = factory.createSession("", "", this::mlsFailureCallback); this.selfEncryptor = factory.fromEncryptor(factory.createEncryptor()); + this.selfEncryptor.setPassthroughMode(true); this.maxProtocolVersion = factory.maxSupportedProtocolVersion(); this.selfUserIdString = String.valueOf(connection.getClient().getClientId()); } @@ -147,6 +148,7 @@ public void handleMLSWelcome(int transitionId, byte[] welcome) { if (roster == null) { sendMLSInvalidCommitWelcome(transitionId); sendMLSKeyPackage(); + return; } prepareRatchets(transitionId, daveSession.getProtocolVersion()); @@ -175,7 +177,11 @@ private void prepareEpoch(String epoch, int protocolVersion) { private void setupKeyRatchetForUser(String uid, int protocolVersion) { var keyRatchet = makeKeyRatchetForUser(uid, protocolVersion); - setSelfKeyRatchet(keyRatchet); + if (selfUserIdString.equals(uid)) { + setSelfKeyRatchet(keyRatchet); + } else if (keyRatchet != null) { + keyRatchet.close(); + } } @Nullable @@ -255,6 +261,10 @@ private void mlsFailureCallback(String source, String reason) { @Override public void close() throws Exception { daveSession.close(); + if (selfKeyRatchet != null) { + selfKeyRatchet.close(); + selfKeyRatchet = null; + } selfEncryptor.close(); } @@ -275,10 +285,13 @@ public void setSelfKeyRatchet(KeyRatchet selfKeyRatchet) { } this.selfKeyRatchet = selfKeyRatchet; + this.reinitSelfEncryptor(); - if (this.selfKeyRatchet != null) { - this.reinitSelfEncryptor(); + if (this.selfKeyRatchet == null) { + this.selfEncryptor.setPassthroughMode(true); + } else { this.selfEncryptor.setKeyRatchet(selfKeyRatchet); + this.selfEncryptor.setPassthroughMode(false); } } From d45bccda94e21675c309b072ee2304258bcd1ffd Mon Sep 17 00:00:00 2001 From: Alula Date: Mon, 23 Feb 2026 01:03:20 +0100 Subject: [PATCH 17/28] Update libdave-jvm --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 46fc158..412bb43 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,7 +28,7 @@ dependencyResolutionManagement { library("jda", "net.dv8tion", "JDA").version("5.0.2") - version("libdave", "cfa4755") + version("libdave", "32a0ce5") library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") library("libdave-natives-linux", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") From b75d1e985d445460d35be24ac50a757b6865a3d9 Mon Sep 17 00:00:00 2001 From: Alula Date: Mon, 23 Feb 2026 01:08:55 +0100 Subject: [PATCH 18/28] Screw you, git --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 412bb43..588bcd4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,7 +28,7 @@ dependencyResolutionManagement { library("jda", "net.dv8tion", "JDA").version("5.0.2") - version("libdave", "32a0ce5") + version("libdave", "32a0ce524") library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") library("libdave-natives-linux", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") From 05c80a686e3b36fd27dc33031b9f64c7c43311a4 Mon Sep 17 00:00:00 2001 From: Alula Date: Mon, 23 Feb 2026 01:09:54 +0100 Subject: [PATCH 19/28] Implement -dirty + use 9 char revs --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 90bf7cd..eca76f8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -119,7 +119,7 @@ fun getGitVersion(): VersionInfo { exec { standardOutput = versionStr errorOutput = versionStr - commandLine("git", "rev-parse", "--short", "HEAD") + commandLine("git", "describe", "--match=NeVeRmAtCh", "--always", "--abbrev=9", "--dirty") } return VersionInfo(versionStr.toString().trim(), true) From 220bb70292e153a2b4518abf948a31275669ce4e Mon Sep 17 00:00:00 2001 From: topi314 Date: Mon, 23 Feb 2026 01:27:21 +0100 Subject: [PATCH 20/28] add application plugin to testbot --- settings.gradle.kts | 2 +- testbot/build.gradle.kts | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 588bcd4..097588e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -32,7 +32,7 @@ dependencyResolutionManagement { library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") library("libdave-natives-linux", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") - library("libdave-natives-windows", "moe.kyokobot.libdave", "natives-win-x86-64").versionRef("libdave") + library("libdave-natives-win", "moe.kyokobot.libdave", "natives-win-x86-64").versionRef("libdave") version("udpqueue", "0.2.12") library("udpqueue-api", "club.minnced", "udpqueue-api").versionRef("udpqueue") diff --git a/testbot/build.gradle.kts b/testbot/build.gradle.kts index 9fef593..7302450 100644 --- a/testbot/build.gradle.kts +++ b/testbot/build.gradle.kts @@ -1,3 +1,11 @@ +plugins { + id("application") +} + +application { + mainClass = "moe.kyokobot.koe.testbot.KoeTestBotLauncher" +} + dependencies { implementation(projects.core) implementation(projects.extUdpqueue) @@ -7,7 +15,7 @@ dependencies { implementation(libs.logback.classic) implementation(libs.libdave.natives.linux) - implementation(libs.libdave.natives.windows) + implementation(libs.libdave.natives.win) implementation(libs.udpqueue.native.linux) implementation(libs.udpqueue.native.win) From 3f101a400e551401e959eea3305b8a46d1f20bf5 Mon Sep 17 00:00:00 2001 From: topi314 Date: Mon, 23 Feb 2026 21:32:10 +0100 Subject: [PATCH 21/28] mix migration docs DEFAULT_BUFFER_DURATION --- MIGRATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MIGRATION.md b/MIGRATION.md index fc1a506..b9cb4e5 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -115,7 +115,7 @@ public boolean canProvide() { ```java var queuePool = new QueueManagerPool( Runtime.getRuntime().availableProcessors(), - UdpQueueFramePollerFactory.DEFAULT_BUFFER_DURATION + QueueManagerPool.DEFAULT_BUFFER_DURATION ); .setFramePollerFactory(new UdpQueueFramePollerFactory(queuePool)); From 281670e62751b809a27129164aa8d8c5c0d49dd8 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 03:00:46 +0100 Subject: [PATCH 22/28] Update libdave --- settings.gradle.kts | 6 +++++- testbot/build.gradle.kts | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 097588e..bedb159 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,16 +28,20 @@ dependencyResolutionManagement { library("jda", "net.dv8tion", "JDA").version("5.0.2") - version("libdave", "32a0ce524") + version("libdave", "6445322dc") library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") library("libdave-natives-linux", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") library("libdave-natives-win", "moe.kyokobot.libdave", "natives-win-x86-64").versionRef("libdave") + library("libdave-natives-win-arm64", "moe.kyokobot.libdave", "natives-win-aarch64").versionRef("libdave") + library("libdave-natives-darwin", "moe.kyokobot.libdave", "natives-darwin").versionRef("libdave") version("udpqueue", "0.2.12") library("udpqueue-api", "club.minnced", "udpqueue-api").versionRef("udpqueue") library("udpqueue-native-linux", "club.minnced", "udpqueue-native-linux-x86-64").versionRef("udpqueue") library("udpqueue-native-win", "club.minnced", "udpqueue-native-win-x86-64").versionRef("udpqueue") + library("udpqueue-native-win-arm64", "club.minnced", "udpqueue-native-win-aarch64").versionRef("udpqueue") + library("udpqueue-native-darwin", "club.minnced", "udpqueue-native-darwin").versionRef("udpqueue") } } } diff --git a/testbot/build.gradle.kts b/testbot/build.gradle.kts index 7302450..dd55dc7 100644 --- a/testbot/build.gradle.kts +++ b/testbot/build.gradle.kts @@ -16,7 +16,11 @@ dependencies { implementation(libs.libdave.natives.linux) implementation(libs.libdave.natives.win) + implementation(libs.libdave.natives.win.arm64) + implementation(libs.libdave.natives.darwin) implementation(libs.udpqueue.native.linux) implementation(libs.udpqueue.native.win) + implementation(libs.udpqueue.native.win.arm64) + implementation(libs.udpqueue.native.darwin) } From 7d011c20cb3a5626c48242e83ba24debe5881e5d Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 03:38:41 +0100 Subject: [PATCH 23/28] Update YouTube source --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index bedb159..1f2b40f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -24,7 +24,7 @@ dependencyResolutionManagement { version("lavaplayer", "2.2.6") library("lava-common", "dev.arbjerg", "lava-common").versionRef("lavaplayer") library("lavaplayer", "dev.arbjerg", "lavaplayer").versionRef("lavaplayer") - library("lavaplayer-youtube", "com.github.lavalink-devs", "lavaplayer-youtube-source").version("1.16.0") + library("lavaplayer-youtube", "com.github.lavalink-devs", "lavaplayer-youtube-source").version("1.18.0") library("jda", "net.dv8tion", "JDA").version("5.0.2") From fc6a0cae5bafc87aefbb574b4ef1797df1fcf3f3 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 04:00:28 +0100 Subject: [PATCH 24/28] Add all natives to testbot + update Gradle --- gradle/wrapper/gradle-wrapper.properties | 2 +- settings.gradle.kts | 26 +++++++++++++++++----- testbot/build.gradle.kts | 28 +++++++++++++++++++----- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index db05440..dea9e33 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Sun Apr 28 02:29:32 CEST 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.4-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/settings.gradle.kts b/settings.gradle.kts index 1f2b40f..f071f78 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,16 +31,30 @@ dependencyResolutionManagement { version("libdave", "6445322dc") library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") - library("libdave-natives-linux", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") - library("libdave-natives-win", "moe.kyokobot.libdave", "natives-win-x86-64").versionRef("libdave") - library("libdave-natives-win-arm64", "moe.kyokobot.libdave", "natives-win-aarch64").versionRef("libdave") library("libdave-natives-darwin", "moe.kyokobot.libdave", "natives-darwin").versionRef("libdave") + library("libdave-natives-linux-glibc-aarch64", "moe.kyokobot.libdave", "natives-linux-aarch64").versionRef("libdave") + library("libdave-natives-linux-glibc-arm", "moe.kyokobot.libdave", "natives-linux-arm").versionRef("libdave") + library("libdave-natives-linux-glibc-amd64", "moe.kyokobot.libdave", "natives-linux-x86-64").versionRef("libdave") + library("libdave-natives-linux-glibc-x86", "moe.kyokobot.libdave", "natives-linux-x86").versionRef("libdave") + library("libdave-natives-linux-musl-aarch64", "moe.kyokobot.libdave", "natives-linux-musl-aarch64").versionRef("libdave") + library("libdave-natives-linux-musl-arm", "moe.kyokobot.libdave", "natives-linux-musl-arm").versionRef("libdave") + library("libdave-natives-linux-musl-amd64", "moe.kyokobot.libdave", "natives-linux-musl-x86-64").versionRef("libdave") + library("libdave-natives-linux-musl-x86", "moe.kyokobot.libdave", "natives-linux-musl-x86").versionRef("libdave") + library("libdave-natives-win-aarch64", "moe.kyokobot.libdave", "natives-win-aarch64").versionRef("libdave") + library("libdave-natives-win-amd64", "moe.kyokobot.libdave", "natives-win-x86-64").versionRef("libdave") + library("libdave-natives-win-x86", "moe.kyokobot.libdave", "natives-win-x86").versionRef("libdave") version("udpqueue", "0.2.12") library("udpqueue-api", "club.minnced", "udpqueue-api").versionRef("udpqueue") - library("udpqueue-native-linux", "club.minnced", "udpqueue-native-linux-x86-64").versionRef("udpqueue") - library("udpqueue-native-win", "club.minnced", "udpqueue-native-win-x86-64").versionRef("udpqueue") - library("udpqueue-native-win-arm64", "club.minnced", "udpqueue-native-win-aarch64").versionRef("udpqueue") + library("udpqueue-native-linux-glibc-aarch64", "club.minnced", "udpqueue-native-linux-aarch64").versionRef("udpqueue") + library("udpqueue-native-linux-glibc-arm", "club.minnced", "udpqueue-native-linux-arm").versionRef("udpqueue") + library("udpqueue-native-linux-glibc-amd64", "club.minnced", "udpqueue-native-linux-x86-64").versionRef("udpqueue") + library("udpqueue-native-linux-glibc-x86", "club.minnced", "udpqueue-native-linux-x86").versionRef("udpqueue") + library("udpqueue-native-linux-musl-aarch64", "club.minnced", "udpqueue-native-linux-musl-aarch64").versionRef("udpqueue") + library("udpqueue-native-linux-musl-amd64", "club.minnced", "udpqueue-native-linux-musl-x86-64").versionRef("udpqueue") + library("udpqueue-native-win-aarch64", "club.minnced", "udpqueue-native-win-aarch64").versionRef("udpqueue") + library("udpqueue-native-win-amd64", "club.minnced", "udpqueue-native-win-x86-64").versionRef("udpqueue") + library("udpqueue-native-win-x86", "club.minnced", "udpqueue-native-win-x86").versionRef("udpqueue") library("udpqueue-native-darwin", "club.minnced", "udpqueue-native-darwin").versionRef("udpqueue") } } diff --git a/testbot/build.gradle.kts b/testbot/build.gradle.kts index dd55dc7..a8927f3 100644 --- a/testbot/build.gradle.kts +++ b/testbot/build.gradle.kts @@ -14,13 +14,29 @@ dependencies { implementation(libs.lavaplayer.youtube) implementation(libs.logback.classic) - implementation(libs.libdave.natives.linux) - implementation(libs.libdave.natives.win) - implementation(libs.libdave.natives.win.arm64) + implementation(libs.libdave.api) + implementation(libs.libdave.impl.jni) implementation(libs.libdave.natives.darwin) + implementation(libs.libdave.natives.linux.glibc.aarch64) + implementation(libs.libdave.natives.linux.glibc.arm) + implementation(libs.libdave.natives.linux.glibc.amd64) + implementation(libs.libdave.natives.linux.glibc.x86) + implementation(libs.libdave.natives.linux.musl.aarch64) + implementation(libs.libdave.natives.linux.musl.arm) + implementation(libs.libdave.natives.linux.musl.amd64) + implementation(libs.libdave.natives.linux.musl.x86) + implementation(libs.libdave.natives.win.aarch64) + implementation(libs.libdave.natives.win.amd64) + implementation(libs.libdave.natives.win.x86) - implementation(libs.udpqueue.native.linux) - implementation(libs.udpqueue.native.win) - implementation(libs.udpqueue.native.win.arm64) + implementation(libs.udpqueue.native.linux.glibc.aarch64) + implementation(libs.udpqueue.native.linux.glibc.arm) + implementation(libs.udpqueue.native.linux.glibc.amd64) + implementation(libs.udpqueue.native.linux.glibc.x86) + implementation(libs.udpqueue.native.linux.musl.aarch64) + implementation(libs.udpqueue.native.linux.musl.amd64) + implementation(libs.udpqueue.native.win.aarch64) + implementation(libs.udpqueue.native.win.amd64) + implementation(libs.udpqueue.native.win.x86) implementation(libs.udpqueue.native.darwin) } From 15044b470a93b7775fcc77533b3dfa119315a83c Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 04:01:33 +0100 Subject: [PATCH 25/28] Update Netty --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index f071f78..e479cc6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,7 +9,7 @@ enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") dependencyResolutionManagement { versionCatalogs { create("libs") { - version("netty", "4.1.112.Final") + version("netty", "4.2.10.Final") library("netty-transport", "io.netty", "netty-transport").versionRef("netty") library("netty-codec-http", "io.netty", "netty-codec-http").versionRef("netty") library("netty-transport-native-epoll-linux", "io.netty", "netty-transport-native-epoll").versionRef("netty") From 0eb2b04c61c05f47f935288bbbca8b2274779a59 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 04:04:36 +0100 Subject: [PATCH 26/28] Ignore /build/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f66c79e..7da27ea 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ buildNumber.properties */build/ */out/ +/build/ # CMake cmake-build-*/ From 5774727ebf342b3106787578b94fbb02bd4d5a46 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 04:34:24 +0100 Subject: [PATCH 27/28] Fix Netty 4.2 SSL verification problem --- .../main/java/moe/kyokobot/koe/KoeOptions.java | 2 ++ .../moe/kyokobot/koe/KoeOptionsBuilder.java | 17 +++++++++++++++-- .../kyokobot/koe/internal/KoeOptionsImpl.java | 8 ++++++++ .../gateway/AbstractMediaGatewayConnection.java | 13 +++++++++++-- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java index f1231eb..010b7a9 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptions.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptions.java @@ -41,5 +41,7 @@ static KoeOptionsBuilder builder() { boolean isEnableWSSPortOverride(); + boolean isVerifyWSSHostname(); + boolean isEnableDAVE(); } diff --git a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java index 8ee8250..24b6b34 100644 --- a/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java +++ b/core/src/main/java/moe/kyokobot/koe/KoeOptionsBuilder.java @@ -34,6 +34,7 @@ public class KoeOptionsBuilder { protected boolean highPacketPriority; protected boolean deafened; protected boolean enableWSSPortOverride; + protected boolean verifyWSSHostname; protected boolean enableDAVE; protected KoeOptionsBuilder() { @@ -56,7 +57,8 @@ protected KoeOptionsBuilder() { this.experimental = false; this.highPacketPriority = true; this.deafened = false; - this.enableWSSPortOverride = true; + this.enableWSSPortOverride = false; + this.verifyWSSHostname = true; this.enableDAVE = true; } @@ -177,6 +179,17 @@ public KoeOptionsBuilder setEnableWSSPortOverride(boolean enableWSSPortOverride) return this; } + /** + * Sets whether to verify the server certificate hostname for WSS voice gateway connections. + * Defaults to true. Set to false only for custom or self-signed voice endpoints. + * + * @param verifyWSSHostname true to verify hostname (default), false to disable verification + */ + public KoeOptionsBuilder setVerifyWSSHostname(boolean verifyWSSHostname) { + this.verifyWSSHostname = verifyWSSHostname; + return this; + } + /** * Sets whether End-to-End encryption using Discord's DAVE protocol is enabled. * Defaults to true. @@ -189,6 +202,6 @@ public KoeOptionsBuilder setDAVEEnabled(boolean enabled) { public KoeOptions create() { return new KoeOptionsImpl(eventLoopGroup, socketChannelClass, datagramChannelClass, byteBufAllocator, gatewayVersion, framePollerFactory, codecRegistry, experimental, - highPacketPriority, deafened, enableWSSPortOverride, enableDAVE); + highPacketPriority, deafened, enableWSSPortOverride, verifyWSSHostname, enableDAVE); } } diff --git a/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java b/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java index 3b95840..053b4cb 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/KoeOptionsImpl.java @@ -30,6 +30,7 @@ public class KoeOptionsImpl implements KoeOptionsExperimental { private final boolean highPacketPriority; private final boolean deafened; private final boolean enableWSSPortOverride; + private final boolean verifyWSSHostname; private final boolean enableDAVE; public KoeOptionsImpl( @@ -44,6 +45,7 @@ public KoeOptionsImpl( boolean highPacketPriority, boolean deafened, boolean enableWSSPortOverride, + boolean verifyWSSHostname, boolean daveEnabled ) { this.eventLoopGroup = Objects.requireNonNull(eventLoopGroup); @@ -57,6 +59,7 @@ public KoeOptionsImpl( this.highPacketPriority = highPacketPriority; this.deafened = deafened; this.enableWSSPortOverride = enableWSSPortOverride; + this.verifyWSSHostname = verifyWSSHostname; this.enableDAVE = daveEnabled; } @@ -122,6 +125,11 @@ public boolean isEnableWSSPortOverride() { return enableWSSPortOverride; } + @Override + public boolean isVerifyWSSHostname() { + return verifyWSSHostname; + } + @Override public boolean isEnableDAVE() { return enableDAVE; diff --git a/core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java b/core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java index e1ece8e..b0a532c 100644 --- a/core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java +++ b/core/src/main/java/moe/kyokobot/koe/internal/gateway/AbstractMediaGatewayConnection.java @@ -69,10 +69,17 @@ public AbstractMediaGatewayConnection(@NotNull MediaConnectionImpl connection, this.connection = Objects.requireNonNull(connection); this.voiceServerInfo = Objects.requireNonNull(voiceServerInfo); + this.websocketURI = new URI(String.format("wss://%s/?v=%d", endpoint, version)); this.bootstrap = NettyBootstrapFactory.socket(connection.getOptions()) .handler(new WebSocketInitializer()); - this.sslContext = SslContextBuilder.forClient().build(); + + var sslBuilder = SslContextBuilder.forClient(); + if (!connection.getOptions().isVerifyWSSHostname()) { + sslBuilder.endpointIdentificationAlgorithm(null); + } + this.sslContext = sslBuilder.build(); + this.allocator = connection.getOptions().getByteBufAllocator(); this.connectFuture = new CompletableFuture<>(); } catch (SSLException | URISyntaxException e) { @@ -285,7 +292,9 @@ private class WebSocketInitializer extends ChannelInitializer { @Override protected void initChannel(SocketChannel ch) { var pipeline = ch.pipeline(); - var engine = sslContext.newEngine(ch.alloc()); + var engine = connection.getOptions().isVerifyWSSHostname() + ? sslContext.newEngine(ch.alloc(), websocketURI.getHost(), websocketURI.getPort() == -1 ? 443 : websocketURI.getPort()) + : sslContext.newEngine(ch.alloc()); pipeline.addLast("ssl", new SslHandler(engine)); pipeline.addLast("http-codec", new HttpClientCodec()); pipeline.addLast("aggregator", new HttpObjectAggregator(65536)); From 84c8b45ca025a03a85cbceb7f57fbe44e9d291c3 Mon Sep 17 00:00:00 2001 From: Alula Date: Tue, 24 Feb 2026 06:07:38 +0100 Subject: [PATCH 28/28] Update libdave-jvm to 5f254c1fd --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index e479cc6..2d112c4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,7 +28,7 @@ dependencyResolutionManagement { library("jda", "net.dv8tion", "JDA").version("5.0.2") - version("libdave", "6445322dc") + version("libdave", "5f254c1fd") library("libdave-api", "moe.kyokobot.libdave", "api").versionRef("libdave") library("libdave-impl-jni", "moe.kyokobot.libdave", "impl-jni").versionRef("libdave") library("libdave-natives-darwin", "moe.kyokobot.libdave", "natives-darwin").versionRef("libdave")