From c2740274f0d922b096210b577477edfc54b89988 Mon Sep 17 00:00:00 2001 From: James A Date: Mon, 8 Nov 2021 15:06:02 -0500 Subject: [PATCH 1/6] Limit the number of video senders in a conference. Fail a video source-add once the limit is reached. --- .../impl/protocol/xmpp/ChatMemberImpl.java | 21 +++++++++++++++ .../jitsi/impl/protocol/xmpp/ChatRoom.java | 5 ++++ .../impl/protocol/xmpp/ChatRoomImpl.java | 26 +++++++++++++++++++ .../impl/protocol/xmpp/ChatRoomMember.java | 5 ++++ .../conference/JitsiMeetConferenceImpl.java | 10 +++++++ .../org/jitsi/jicofo/ConferenceConfig.kt | 4 +++ src/main/resources/reference.conf | 3 +++ 7 files changed, 74 insertions(+) diff --git a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatMemberImpl.java b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatMemberImpl.java index 218350a06e..25fce9bed3 100644 --- a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatMemberImpl.java +++ b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatMemberImpl.java @@ -94,6 +94,11 @@ public class ChatMemberImpl */ private String statsId; + /** + * Indicates whether the member's video sources are currently muted. + */ + private boolean isVideoMuted = true; + public ChatMemberImpl(EntityFullJid fullJid, ChatRoomImpl chatRoom, int joinOrderNumber) { @@ -199,6 +204,9 @@ public boolean isJibri() return isJibri; } + @Override + public boolean isVideoMuted() { return isVideoMuted; } + /** * Does presence processing. * @@ -287,6 +295,19 @@ void processPresence(Presence presence) { statsId = statsIdPacketExt.getStatsId(); } + + boolean wasVideoMuted = isVideoMuted; + StandardExtensionElement videoMutedExt = presence.getExtension("videomuted", "jabber:client"); + isVideoMuted = videoMutedExt == null || Boolean.valueOf(videoMutedExt.getText()); /* defaults to true */ + + if (isVideoMuted != wasVideoMuted) + { + logger.debug(() -> toString() + ". isMuted = " + isVideoMuted + "."); + if (isVideoMuted) + chatRoom.removeVideoSender(); + else + chatRoom.addVideoSender(); + } } /** diff --git a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoom.java b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoom.java index f20c4e4b18..a8110cc924 100644 --- a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoom.java +++ b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoom.java @@ -89,6 +89,11 @@ void join() */ int getMembersCount(); + /** + * Returns the number of members that currently have their video sources unmuted. + */ + int getVideoSendersCount(); + /** * Grants ownership privileges to another user. Room owners may grant * ownership privileges. Some room implementations will not allow to grant diff --git a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java index 30d10ddfe5..265420074d 100644 --- a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java +++ b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java @@ -126,6 +126,11 @@ public class ChatRoomImpl */ private EventEmitter eventEmitter = new SyncEventEmitter<>(); + /** + * The number of members that currently have their video sources unmuted. + */ + private int numVideoSenders; + /** * Creates new instance of ChatRoomImpl. * @@ -577,6 +582,21 @@ private void sendLastPresence() UtilKt.tryToSendStanza(xmppProvider.getXmppConnection(), lastPresenceSent); } + @Override + public int getVideoSendersCount() { return numVideoSenders; } + + public void addVideoSender() + { + ++numVideoSenders; + logger.debug(() -> "The number of video senders has increased to " + numVideoSenders + "."); + } + + public void removeVideoSender() + { + --numVideoSenders; + logger.debug(() -> "The number of video senders has decreased to " + numVideoSenders + "."); + } + /** * Adds a new {@link ChatMemberImpl} with the given JID to {@link #members}. * If a member with the given JID already exists, it returns the existing @@ -597,6 +617,9 @@ private ChatMemberImpl addMember(EntityFullJid jid) members.put(jid, newMember); + if (!newMember.isVideoMuted()) + addVideoSender(); + return newMember; } } @@ -858,6 +881,9 @@ private ChatMemberImpl removeMember(EntityFullJid occupantJid) logger.error(occupantJid + " not in " + roomJid); } + if (!removed.isVideoMuted()) + removeVideoSender(); + return removed; } } diff --git a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomMember.java b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomMember.java index ada001d560..f9de70a0ca 100644 --- a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomMember.java +++ b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomMember.java @@ -96,6 +96,11 @@ public interface ChatRoomMember boolean isJibri(); + /** + * Returns whether the room member has its video sources muted. + */ + boolean isVideoMuted(); + /** * Gets the region (e.g. "us-east") of this {@link ChatRoomMember}. */ diff --git a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java index c6a7135035..90ed6d297d 100644 --- a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java +++ b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java @@ -1504,6 +1504,16 @@ public StanzaError onAddSource(@NotNull JingleSession jingleSession, List= ConferenceConfig.config.getMaxVideoSenders()) + { + String errorMsg = "Source add rejected. Maximum number of video senders reached."; + logger.warn(() -> participantId + ": " + errorMsg); + return StanzaError.from(StanzaError.Condition.resource_constraint, errorMsg).build(); + } + logger.debug(() -> "Received source-add from " + participantId + ": " + sourcesAdvertised); logger.debug(() -> "Accepted sources from " + participantId + ": " + sourcesAccepted); diff --git a/src/main/kotlin/org/jitsi/jicofo/ConferenceConfig.kt b/src/main/kotlin/org/jitsi/jicofo/ConferenceConfig.kt index 6c62b0e65d..d2cb93b813 100644 --- a/src/main/kotlin/org/jitsi/jicofo/ConferenceConfig.kt +++ b/src/main/kotlin/org/jitsi/jicofo/ConferenceConfig.kt @@ -62,6 +62,10 @@ class ConferenceConfig { "jicofo.conference.min-participants".from(newConfig) } + val maxVideoSenders: Int by config { + "jicofo.conference.max-video-senders".from(newConfig) + } + val enableLipSync: Boolean by config { "jicofo.conference.enable-lip-sync".from(newConfig) } diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index 83dfb4b3e7..39858269c9 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -177,6 +177,9 @@ jicofo { // The minimum number of participants required for the conference to be started. min-participants = 2 + // The maximum number of participants that can send their video at the same time. + max-video-senders = 50 + // Experimental. enable-lip-sync = false From e712507502f642f18104ef2dd7a11848586e803a Mon Sep 17 00:00:00 2001 From: James A Date: Mon, 8 Nov 2021 17:08:44 -0500 Subject: [PATCH 2/6] Add and use EndpointSourceSet::hasVideo --- .../jicofo/conference/JitsiMeetConferenceImpl.java | 6 ++---- .../jicofo/conference/source/EndpointSourceSet.kt | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java index 90ed6d297d..6e28a51c9a 100644 --- a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java +++ b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java @@ -1504,10 +1504,8 @@ public StanzaError onAddSource(@NotNull JingleSession jingleSession, List= ConferenceConfig.config.getMaxVideoSenders()) + if (sourcesAdvertised.hasVideo() && + chatRoom.getVideoSendersCount() >= ConferenceConfig.config.getMaxVideoSenders()) { String errorMsg = "Source add rejected. Maximum number of video senders reached."; logger.warn(() -> participantId + ": " + errorMsg); diff --git a/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt b/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt index 21942a5d24..c03744fb43 100644 --- a/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt +++ b/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt @@ -41,6 +41,16 @@ data class EndpointSourceSet( */ fun isEmpty() = sources.isEmpty() && ssrcGroups.isEmpty() + /** + * Whether there are any audio sources in this set. + */ + fun hasAudio() = sources.any { it.mediaType == AUDIO } + + /** + * Whether there are any video sources in this set. + */ + fun hasVideo() = sources.any { it.mediaType == VIDEO } + /** * Creates a list of Jingle [ContentPacketExtension]s that describe the sources in this [EndpointSourceSet]. */ From a26e593f0ee286cf40327352fb625a3990da7575 Mon Sep 17 00:00:00 2001 From: James A Date: Mon, 8 Nov 2021 18:54:30 -0500 Subject: [PATCH 3/6] set video-limit-reached property --- .../impl/protocol/xmpp/ChatRoomImpl.java | 10 ++++++ .../conference/JitsiMeetConferenceImpl.java | 32 ++++++++++++++++++- .../jitsi/jicofo/xmpp/muc/ChatRoomListener.kt | 1 + 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java index 265420074d..876ca517d5 100644 --- a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java +++ b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java @@ -589,12 +589,22 @@ public void addVideoSender() { ++numVideoSenders; logger.debug(() -> "The number of video senders has increased to " + numVideoSenders + "."); + + eventEmitter.fireEvent(handler -> { + handler.numVideoSendersChanged(numVideoSenders); + return Unit.INSTANCE; + }); } public void removeVideoSender() { --numVideoSenders; logger.debug(() -> "The number of video senders has decreased to " + numVideoSenders + "."); + + eventEmitter.fireEvent(handler -> { + handler.numVideoSendersChanged(numVideoSenders); + return Unit.INSTANCE; + }); } /** diff --git a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java index 6e28a51c9a..607fdeced5 100644 --- a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java +++ b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java @@ -115,7 +115,7 @@ public class JitsiMeetConferenceImpl @NotNull private final JitsiMeetConfig config; - private final ChatRoomListener chatRoomListener = new ChatRoomListenerImpl(); + private final ChatRoomListener chatRoomListener = new ChatRoomListenerImpl(this); /** * Conference room chat instance. @@ -228,6 +228,11 @@ public class JitsiMeetConferenceImpl */ private final ValidatingConferenceSourceMap conferenceSources = new ValidatingConferenceSourceMap(); + /** + * Whether the limit on the number of video senders is currently hit. + */ + private boolean videoLimitReached = false; + /** * Creates new instance of {@link JitsiMeetConferenceImpl}. * @@ -526,6 +531,19 @@ private void setConferenceProperty(String key, String value, boolean updatePrese } } + /** + * Process the new number of video senders reported by the chat room. + */ + private void numVideoSendersChanged(int numVideoSenders) + { + boolean newValue = numVideoSenders >= ConferenceConfig.config.getMaxVideoSenders(); + if (videoLimitReached != newValue) + { + videoLimitReached = newValue; + setConferenceProperty("video-limit-reached", String.valueOf(videoLimitReached)); + } + } + /** * Leaves the conference room. */ @@ -2425,6 +2443,12 @@ public void bridgeAdded(Bridge bridge) private class ChatRoomListenerImpl implements ChatRoomListener { + private final JitsiMeetConferenceImpl conference; + private ChatRoomListenerImpl(JitsiMeetConferenceImpl conference) + { + this.conference = conference; + } + @Override public void roomDestroyed(@NotNull String reason) { @@ -2471,5 +2495,11 @@ public void localRoleChanged(@NotNull MemberRole newRole, @Nullable MemberRole o public void memberPresenceChanged(@NotNull ChatRoomMember member) { } + + @Override + public void numVideoSendersChanged(int numVideoSenders) + { + conference.numVideoSendersChanged(numVideoSenders); + } } } diff --git a/src/main/kotlin/org/jitsi/jicofo/xmpp/muc/ChatRoomListener.kt b/src/main/kotlin/org/jitsi/jicofo/xmpp/muc/ChatRoomListener.kt index 0016fae7c8..f296ef2a84 100644 --- a/src/main/kotlin/org/jitsi/jicofo/xmpp/muc/ChatRoomListener.kt +++ b/src/main/kotlin/org/jitsi/jicofo/xmpp/muc/ChatRoomListener.kt @@ -29,6 +29,7 @@ interface ChatRoomListener { fun roomDestroyed(reason: String) {} fun startMutedChanged(startAudioMuted: Boolean, startVideoMuted: Boolean) {} fun localRoleChanged(newRole: MemberRole, oldRole: MemberRole? = null) {} + fun numVideoSendersChanged(numVideoSenders: Int) {} } /** A class with the default kotlin method implementations (to avoid using @JvmDefault) **/ From e1dde34e1baccd9752fd60f1b7c58aa268b7db0e Mon Sep 17 00:00:00 2001 From: James A Date: Mon, 8 Nov 2021 20:13:43 -0500 Subject: [PATCH 4/6] Disable feature by default. --- src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index 39858269c9..7551a1e8cb 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -178,7 +178,7 @@ jicofo { min-participants = 2 // The maximum number of participants that can send their video at the same time. - max-video-senders = 50 + max-video-senders = 999999 // Experimental. enable-lip-sync = false From 3b73cb29955a977764056027781eaae48098fd67 Mon Sep 17 00:00:00 2001 From: James A Date: Tue, 9 Nov 2021 10:30:08 -0500 Subject: [PATCH 5/6] Clean up. Fix build - add stub functions to other derived classes. Use implicit reference from inner class. Do lazy evaluation of hasVideo. --- .../jicofo/conference/JitsiMeetConferenceImpl.java | 14 ++++---------- .../jicofo/conference/source/EndpointSourceSet.kt | 4 ++-- src/test/java/mock/muc/MockChatRoom.java | 6 ++++++ src/test/java/mock/muc/MockRoomMember.java | 6 ++++++ 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java index 607fdeced5..808b4ac140 100644 --- a/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java +++ b/src/main/java/org/jitsi/jicofo/conference/JitsiMeetConferenceImpl.java @@ -115,7 +115,7 @@ public class JitsiMeetConferenceImpl @NotNull private final JitsiMeetConfig config; - private final ChatRoomListener chatRoomListener = new ChatRoomListenerImpl(this); + private final ChatRoomListener chatRoomListener = new ChatRoomListenerImpl(); /** * Conference room chat instance. @@ -534,7 +534,7 @@ private void setConferenceProperty(String key, String value, boolean updatePrese /** * Process the new number of video senders reported by the chat room. */ - private void numVideoSendersChanged(int numVideoSenders) + private void onNumVideoSendersChanged(int numVideoSenders) { boolean newValue = numVideoSenders >= ConferenceConfig.config.getMaxVideoSenders(); if (videoLimitReached != newValue) @@ -1522,7 +1522,7 @@ public StanzaError onAddSource(@NotNull JingleSession jingleSession, List= ConferenceConfig.config.getMaxVideoSenders()) { String errorMsg = "Source add rejected. Maximum number of video senders reached."; @@ -2443,12 +2443,6 @@ public void bridgeAdded(Bridge bridge) private class ChatRoomListenerImpl implements ChatRoomListener { - private final JitsiMeetConferenceImpl conference; - private ChatRoomListenerImpl(JitsiMeetConferenceImpl conference) - { - this.conference = conference; - } - @Override public void roomDestroyed(@NotNull String reason) { @@ -2499,7 +2493,7 @@ public void memberPresenceChanged(@NotNull ChatRoomMember member) @Override public void numVideoSendersChanged(int numVideoSenders) { - conference.numVideoSendersChanged(numVideoSenders); + onNumVideoSendersChanged(numVideoSenders); } } } diff --git a/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt b/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt index c03744fb43..ac6d822725 100644 --- a/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt +++ b/src/main/kotlin/org/jitsi/jicofo/conference/source/EndpointSourceSet.kt @@ -44,12 +44,12 @@ data class EndpointSourceSet( /** * Whether there are any audio sources in this set. */ - fun hasAudio() = sources.any { it.mediaType == AUDIO } + val hasAudio: Boolean by lazy { sources.any { it.mediaType == AUDIO } } /** * Whether there are any video sources in this set. */ - fun hasVideo() = sources.any { it.mediaType == VIDEO } + val hasVideo: Boolean by lazy { sources.any { it.mediaType == VIDEO } } /** * Creates a list of Jingle [ContentPacketExtension]s that describe the sources in this [EndpointSourceSet]. diff --git a/src/test/java/mock/muc/MockChatRoom.java b/src/test/java/mock/muc/MockChatRoom.java index edeb959c0c..cfcb36ead6 100644 --- a/src/test/java/mock/muc/MockChatRoom.java +++ b/src/test/java/mock/muc/MockChatRoom.java @@ -255,6 +255,12 @@ public int getMembersCount() return members.size(); } + @Override + public int getVideoSendersCount() + { + return 0; /* implement */ + } + private void grantRole(EntityFullJid address, MemberRole newRole) { MockRoomMember member = findMember(address.getResourceOrNull()); diff --git a/src/test/java/mock/muc/MockRoomMember.java b/src/test/java/mock/muc/MockRoomMember.java index 121b138297..96673a0a86 100644 --- a/src/test/java/mock/muc/MockRoomMember.java +++ b/src/test/java/mock/muc/MockRoomMember.java @@ -110,6 +110,12 @@ public boolean isJibri() return false; } + @Override + public boolean isVideoMuted() + { + return false; + } + @Override public Presence getPresence() { From fb9e5b8f22f130023cc3d40766842a73bb350ad2 Mon Sep 17 00:00:00 2001 From: James A Date: Tue, 9 Nov 2021 10:49:05 -0500 Subject: [PATCH 6/6] Fix null dereference. --- src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java index 876ca517d5..1896aa84f7 100644 --- a/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java +++ b/src/main/java/org/jitsi/impl/protocol/xmpp/ChatRoomImpl.java @@ -890,9 +890,10 @@ private ChatMemberImpl removeMember(EntityFullJid occupantJid) { logger.error(occupantJid + " not in " + roomJid); } - - if (!removed.isVideoMuted()) + else if (!removed.isVideoMuted()) + { removeVideoSender(); + } return removed; }