Adds a Chat Reliability Layer (tgstation#79479)

## About The Pull Request

Everyone knows that chat will just eat your messages now and then, isn't
that annoying?
What if SSchat was smart enough to keep track of your messages and
notice when you didn't get one?
Well, now it can!
## Why It's Good For The Game

Chat messages poofing into the aether is bad, really bad.
## Changelog
:cl:
add: Chat Reliability Layer
code: TGUI chat messages now track their sequence and will be resent if
the client notices a discrepenency
/:cl:

---------

Co-authored-by: Kyle Spier-Swenson <kyleshome@gmail.com>

code/__DEFINES/chat.dm

@@ -3,6 +3,11 @@
  * SPDX-License-Identifier: MIT
  */
 
+/// How many chat payloads to keep in history
+#define CHAT_RELIABILITY_HISTORY_SIZE 5
+/// How many resends to allow before giving up
+#define CHAT_RELIABILITY_MAX_RESENDS 3
+
 #define MESSAGE_TYPE_SYSTEM "system"
 #define MESSAGE_TYPE_LOCALCHAT "localchat"
 #define MESSAGE_TYPE_RADIO "radio"

code/controllers/subsystem/chat.dm

@@ -5,39 +5,93 @@
 
 SUBSYSTEM_DEF(chat)
 	name = "Chat"
-	flags = SS_TICKER
+	flags = SS_TICKER|SS_NO_INIT
 	wait = 1
 	priority = FIRE_PRIORITY_CHAT
 	init_order = INIT_ORDER_CHAT
 
-	var/list/payload_by_client = list()
+	/// Assosciates a ckey with a list of messages to send to them.
+	var/list/list/datum/chat_payload/client_to_payloads = list()
 
-/datum/controller/subsystem/chat/Initialize()
-	// Just used by chat system to know that initialization is nearly finished.
-	// The to_chat checks could probably check the runlevel instead, but would require testing.
-	return SS_INIT_SUCCESS
+	/// Associates a ckey with an assosciative list of their last CHAT_RELIABILITY_HISTORY_SIZE messages.
+	var/list/list/datum/chat_payload/client_to_reliability_history = list()
+
+	/// Assosciates a ckey with their next sequence number.
+	var/list/client_to_sequence_number = list()
+
+/datum/controller/subsystem/chat/proc/generate_payload(client/target, message_data)
+	var/sequence = client_to_sequence_number[target.ckey]
+	client_to_sequence_number[target.ckey] += 1
+
+	var/datum/chat_payload/payload = new
+	payload.sequence = sequence
+	payload.content = message_data
+
+	if(!(target.ckey in client_to_reliability_history))
+		client_to_reliability_history[target.ckey] = list()
+	var/list/client_history = client_to_reliability_history[target.ckey]
+	client_history["[sequence]"] = payload
+
+	if(length(client_history) > CHAT_RELIABILITY_HISTORY_SIZE)
+		var/oldest = text2num(client_history[1])
+		for(var/index in 2 to length(client_history))
+			var/test = text2num(client_history[index])
+			if(test < oldest)
+				oldest = test
+		client_history -= "[oldest]"
+	return payload
+
+/datum/controller/subsystem/chat/proc/send_payload_to_client(client/target, datum/chat_payload/payload)
+	target.tgui_panel.window.send_message("chat/message", payload.into_message())
 
 /datum/controller/subsystem/chat/fire()
-	for(var/key in payload_by_client)
-		var/client/client = key
-		var/payload = payload_by_client[key]
-		payload_by_client -= key
-		if(client)
-			// Send to tgchat
-			client.tgui_panel?.window.send_message("chat/message", payload)
-			// Send to old chat
-			for(var/message in payload)
-				SEND_TEXT(client, message_to_html(message))
+	for(var/ckey in client_to_payloads)
+		var/client/target = GLOB.directory[ckey]
+		if(isnull(target)) // verify client still exists
+			LAZYREMOVE(client_to_payloads, ckey)
+			continue
+
+		for(var/datum/chat_payload/payload as anything in client_to_payloads[ckey])
+			send_payload_to_client(target, payload)
+		LAZYREMOVE(client_to_payloads, ckey)
+
 		if(MC_TICK_CHECK)
 			return
 
-/datum/controller/subsystem/chat/proc/queue(target, message)
-	if(islist(target))
-		for(var/_target in target)
-			var/client/client = CLIENT_FROM_VAR(_target)
-			if(client)
-				LAZYADD(payload_by_client[client], list(message))
+/datum/controller/subsystem/chat/proc/queue(queue_target, list/message_data)
+	var/list/targets = islist(queue_target) ? queue_target : list(queue_target)
+	for(var/target in targets)
+		var/client/client = CLIENT_FROM_VAR(target)
+		if(isnull(client))
+			continue
+		LAZYADDASSOCLIST(client_to_payloads, client.ckey, generate_payload(client, message_data))
+
+/datum/controller/subsystem/chat/proc/send_immediate(send_target, list/message_data)
+	var/list/targets = islist(send_target) ? send_target : list(send_target)
+	for(var/target in targets)
+		var/client/client = CLIENT_FROM_VAR(target)
+		if(isnull(client))
+			continue
+		send_payload_to_client(client, generate_payload(client, message_data))
+
+/datum/controller/subsystem/chat/proc/handle_resend(client/client, sequence)
+	var/list/client_history = client_to_reliability_history[client.ckey]
+	sequence = "[sequence]"
+	if(isnull(client_history) || !(sequence in client_history))
 		return
-	var/client/client = CLIENT_FROM_VAR(target)
-	if(client)
-		LAZYADD(payload_by_client[client], list(message))
+
+	var/datum/chat_payload/payload = client_history[sequence]
+	if(payload.resends > CHAT_RELIABILITY_MAX_RESENDS)
+		return // we tried but byond said no
+
+	payload.resends += 1
+	send_payload_to_client(client, client_history[sequence])
+	SSblackbox.record_feedback(
+		"nested tally",
+		"chat_resend_byond_version",
+		1,
+		list(
+			"[client.byond_version]",
+			"[client.byond_build]",
+		),
+	)

code/datums/chat_payload.dm

@@ -0,0 +1,12 @@
+/// Stores information about a chat payload
+/datum/chat_payload
+	/// Sequence number of this payload
+	var/sequence = 0
+	/// Message we are sending
+	var/list/content
+	/// Resend count
+	var/resends = 0
+
+/// Converts the chat payload into a JSON string
+/datum/chat_payload/proc/into_message()
+	return "{\"sequence\":[sequence],\"content\":[json_encode(content)]}"

code/modules/tgchat/README.md

@@ -0,0 +1,30 @@
+## /TG/ Chat
+
+/TG/ Chat, which will be referred to as TgChat from this point onwards, is a system in which we can send messages to clients in a controlled and semi-reliable manner. The standard way of sending messages to BYOND clients simply dumps whatever you output to them directly into their chat window, however BYOND allows us to load our own code on the client to change this behaviour in a way that allows us to do some pretty neat things.
+
+### Message Format
+
+TgChat handles sending messages from the server to the client through the use of JSON payloads, of which the format will change depending on the type of message and the intended client endpoint. An example of the payload for chat messages is as follows:
+```json
+{
+	"sequence": 0,
+	"content": {
+		"type": ". . .", // ?optional
+		"text": ". . .", // ?optional !atleast-one
+		"html": ". . .", // ?optional !atleast-one
+		"avoidHighlighting": 0 // ?optional
+	},
+}
+```
+
+### Reliability
+
+In the past there have been issues where BYOND will silently and without reason lose a message we sent to the client, to detect this and recover from it seamlessly TgChat also has a baked in reliability layer. This reliability layer is very primitive, and simply keeps track of recieved sequence numbers. Should the client recieve an unexpected sequence number TgChat asks the server to resend any missing packets. 
+
+### Ping System
+
+TgChat supports a round trip time ping measurement, which is displayed to the client so they can know how long it takes for their commands and inputs to reach the server. This is done by sending the client a ping request, `ping/soft`, which tells the client to send a ping to the server. When the server recieves said ping it sends a reply, `ping/reply`, to the client with a payload containing the current DateTime which the client can reference against the initial ping request.
+
+### Chat Tabs, Local Storage, and Highlighting
+
+To make organizing and managing chat easier and more functional for both players and admins, TgChat has the ability to filter out messages based on their primary tag, such as individual departmental radios, to a dedicated chat tab for easier reading and comprehension. These tabs can also be configured to highlist messages based on a simple keyword search. You can set a multitude of different keywords to search for and they will be highlighting for instant alerting of the client. Said tabs, highlighting rules, and your chat history will persist thanks to use of local storage on the client. Using local storage TgChat can ensure that your preferences are saved and maintained between client restarts and switching between other /TG/ servers. Local Storage is also used to keep your chat history aswell, should you need to scroll through your chat logs.

code/modules/tgchat/to_chat.dm

@@ -35,23 +35,9 @@
 	if(text) message["text"] = text
 	if(html) message["html"] = html
 	if(avoid_highlighting) message["avoidHighlighting"] = avoid_highlighting
-	var/message_blob = TGUI_CREATE_MESSAGE("chat/message", message)
-	var/message_html = message_to_html(message)
-	if(islist(target))
-		for(var/_target in target)
-			var/client/client = CLIENT_FROM_VAR(_target)
-			if(client)
-				// Send to tgchat
-				client.tgui_panel?.window.send_raw_message(message_blob)
-				// Send to old chat
-				SEND_TEXT(client, message_html)
-		return
-	var/client/client = CLIENT_FROM_VAR(target)
-	if(client)
-		// Send to tgchat
-		client.tgui_panel?.window.send_raw_message(message_blob)
-		// Send to old chat
-		SEND_TEXT(client, message_html)
+
+	// send it immediately
+	SSchat.send_immediate(target, message)
 
 /**
  * Sends the message to the recipient (target).

code/modules/tgui/tgui_window.dm

@@ -384,6 +384,8 @@
 			client << link(href_list["url"])
 		if("cacheReloaded")
 			reinitialize()
+		if("chat/resend")
+			SSchat.handle_resend(client, payload)
 
 /datum/tgui_window/vv_edit_var(var_name, var_value)
 	return var_name != NAMEOF(src, id) && ..()

tgstation.dme

@@ -747,6 +747,7 @@
 #include "code\datums\beam.dm"
 #include "code\datums\browser.dm"
 #include "code\datums\callback.dm"
+#include "code\datums\chat_payload.dm"
 #include "code\datums\chatmessage.dm"
 #include "code\datums\dash_weapon.dm"
 #include "code\datums\datum.dm"

tgui/packages/tgui-panel/chat/middleware.js

@@ -64,6 +64,8 @@ const loadChatFromStorage = async (store) => {
 export const chatMiddleware = (store) => {
   let initialized = false;
   let loaded = false;
+  const sequences = [];
+  const sequences_requested = [];
   chatRenderer.events.on('batchProcessed', (countByType) => {
     // Use this flag to workaround unread messages caused by
     // loading them from storage. Side effect of that, is that
@@ -85,9 +87,35 @@ export const chatMiddleware = (store) => {
       loadChatFromStorage(store);
     }
     if (type === 'chat/message') {
-      // Normalize the payload
-      const batch = Array.isArray(payload) ? payload : [payload];
-      chatRenderer.processBatch(batch);
+      const payload_obj = JSON.parse(payload);
+      const sequence = payload_obj.sequence;
+      if (sequences.includes(sequence)) {
+        return;
+      }
+
+      const sequence_count = sequences.length;
+      seq_check: if (sequence_count > 0) {
+        if (sequences_requested.includes(sequence)) {
+          sequences_requested.splice(sequences_requested.indexOf(sequence), 1);
+          // if we are receiving a message we requested, we can stop reliability checks
+          break seq_check;
+        }
+
+        // cannot do reliability if we don't have any messages
+        const expected_sequence = sequences[sequence_count - 1] + 1;
+        if (sequence !== expected_sequence) {
+          for (
+            let requesting = expected_sequence;
+            requesting < sequence;
+            requesting++
+          ) {
+            requested_sequences.push(requesting);
+            Byond.sendMessage('chat/resend', requesting);
+          }
+        }
+      }
+
+      chatRenderer.processBatch([payload_obj.content]);
       return;
     }
     if (type === loadChat.type) {