Merge branch 'master' into esbuild-code-splitting

* master: (36 commits) proper scala inlining Remove chess24 (lichess-org#14962) preload the broadcast group name tweak hcaptcha skip condition better log hcaptcha result type rotate oauth token on creation - closes lichess-org#14946 Add comments for NewTree.toBranch function fixes for ios safari < 15 fix error preventing pure javascript engine when none other are available Update netty-transport-native-epoll to 4.1.108.Final Scala tweak Fix addNodeAt test for the NewTree Scalafmt allow unicode emojis in CMS markdown Remove unused function from Tree Add debug extension for Root & NewRoot Use a more general predicate function move sign in link outside of dasher toggle, fix/simplify css Code golf Helpers Fix NewTree takeMainlineWhile test ...
schlawg · Mar 25, 2024 · c9e1f27 · c9e1f27
2 parents ca871ed + 85eba48
commit c9e1f27
Show file tree

Hide file tree

Showing 34 changed files with 255 additions and 151 deletions.
diff --git a/app/controllers/Mod.scala b/app/controllers/Mod.scala
@@ -8,6 +8,7 @@ import play.api.mvc.*
 import views.*
 
 import scala.annotation.nowarn
+import scala.util.chaining.scalaUtilChainingOps
 
 import lila.app.{ *, given }
 import lila.common.{ EmailAddress, HTTPRequest, IpAddress }
@@ -290,6 +291,19 @@ final class Mod(
   def communicationPublic(username: UserStr)  = communications(username, priv = false)
   def communicationPrivate(username: UserStr) = communications(username, priv = true)
 
+  def fullCommsExport(username: UserStr) =
+    SecureBody(_.FullCommsExport) { ctx ?=> me ?=>
+      Found(env.user.repo.byId(username)): user =>
+        val source = env.msg.api
+          .modFullCommsExport(user.id)
+          .map: (tid, msgs) =>
+            s"=== 0 === thread: ${tid}\n${msgs.map(m => s"${m.date} ${m.user}: ${m.text}\n--- 0 ---\n").toList.mkString("\n")}"
+        Ok.chunked(source)
+          .pipe(asAttachmentStream(s"full-comms-export-of-${user.id}.txt"))
+          .andDo(env.mod.logApi.fullCommExport(Suspect(user)))
+          .andDo(env.irc.api.fullCommExport(user))
+    }
+
   protected[controllers] def redirect(username: UserStr, mod: Boolean = true) =
     Redirect(userUrl(username, mod))
 

diff --git a/app/views/base/layout.scala b/app/views/base/layout.scala
@@ -123,10 +123,12 @@ object layout:
 
   private def anonDasher(using ctx: PageContext) =
     val prefs = trans.preferences.preferences.txt()
-    div(cls := "dasher")(
-      a(href := s"${routes.Auth.login.url}?referrer=${ctx.req.path}", cls := "signin")(trans.signIn.txt()),
-      button(cls := "toggle anon link", title := prefs, aria.label := prefs, dataIcon := licon.Gear),
-      div(id     := "dasher_app", cls         := "dropdown")
+    frag(
+      a(href := s"${routes.Auth.login.url}?referrer=${ctx.req.path}", cls := "signin")(trans.signIn()),
+      div(cls := "dasher")(
+        button(cls := "toggle anon link", title := prefs, aria.label := prefs, dataIcon := licon.Gear),
+        div(id     := "dasher_app", cls         := "dropdown")
+      )
     )
 
   private def allNotifications(using ctx: PageContext) =

diff --git a/app/views/mod/communication.scala b/app/views/mod/communication.scala
@@ -46,16 +46,32 @@ object communication:
                 titleOrText("Mod zone (Hotkey: m)"),
                 dataIcon := licon.Agent
               ),
-              isGranted(_.ViewPrivateComms).option {
-                if priv then
-                  a(cls := "priv button active", href := routes.Mod.communicationPublic(u.username))("PMs")
-                else
-                  a(
-                    cls   := "priv button",
-                    href  := routes.Mod.communicationPrivate(u.username),
-                    title := "View private messages. This will be logged in #commlog"
-                  )("PMs")
-              }
+              isGranted(_.ViewPrivateComms)
+                .option {
+                  if priv then
+                    a(cls := "priv button active", href := routes.Mod.communicationPublic(u.username))("PMs")
+                  else
+                    a(
+                      cls   := "priv button",
+                      href  := routes.Mod.communicationPrivate(u.username),
+                      title := "View private messages. This will be logged in #commlog"
+                    )("PMs")
+                },
+              (priv && isGranted(_.FullCommsExport))
+                .option {
+                  postForm(
+                    action := routes.Mod.fullCommsExport(u.username)
+                  )(
+                    form3.action(
+                      form3.submit(
+                        "Full comms export",
+                        icon = none,
+                        confirm =
+                          s"Confirm you want to export all comms from **${u.username}** (including other party)".some
+                      )(cls := "button-red comms-export")
+                    )
+                  )
+                }
             )
           )
         ),

diff --git a/app/views/relay/show.scala b/app/views/relay/show.scala
@@ -63,7 +63,7 @@ object show:
         div(cls := "box relay-tour")(
           div(cls := "relay-tour__header")(
             div(cls := "relay-tour__header__content")(
-              h1(rt.tour.name),
+              h1(data.group.fold(rt.tour.name.value)(_.value)),
               div(cls := "relay-tour__header__selectors"):
                 div(cls := "mselect relay-tour__mselect"):
                   label(cls := "mselect__label"):

diff --git a/app/views/team/request.scala b/app/views/team/request.scala
@@ -59,7 +59,7 @@ object request:
   private[team] def list(requests: List[lila.team.RequestWithUser], t: Option[lila.team.Team])(using
       ctx: PageContext
   ) =
-    table(cls := "slist requests @if(t.isEmpty){all}else{for-team} datatable")(
+    table(cls := "slist requests datatable")(
       tbody(
         requests.map { request =>
           tr(

diff --git a/conf/routes b/conf/routes
@@ -542,6 +542,7 @@ POST  /mod/:username/title             controllers.Mod.setTitle(username)
 POST  /mod/:username/inquiry           controllers.Mod.spontaneousInquiry(username)
 GET   /mod/:username/communication     controllers.Mod.communicationPublic(username)
 GET   /mod/:username/communication/private controllers.Mod.communicationPrivate(username)
+POST   /mod/:username/communication/full-comms-export controllers.Mod.fullCommsExport(username)
 POST  /mod/:username/rankban/:v        controllers.Mod.rankban(username, v: Boolean)
 POST  /mod/:username/arenaban/:v       controllers.Mod.arenaBan(username, v: Boolean)
 POST  /mod/:username/prizeban/:v       controllers.Mod.prizeban(username, v: Boolean)

diff --git a/modules/cms/src/main/CmsForm.scala b/modules/cms/src/main/CmsForm.scala
@@ -2,8 +2,9 @@ package lila.cms
 
 import play.api.data.*
 import play.api.data.Forms.*
+import play.api.data.validation.Constraints
 
-import lila.common.Form.{ cleanNonEmptyText, into, slugConstraint }
+import lila.common.Form.{ cleanNonEmptyText, cleanTextWithSymbols, into, slugConstraint }
 import lila.i18n.{ LangForm, Language }
 import lila.user.User
 
@@ -12,8 +13,10 @@ object CmsForm:
   val create = Form:
     mapping(
       "key" -> cleanNonEmptyText(minLength = 3, maxLength = 120).verifying(slugConstraint).into[CmsPage.Key],
-      "title"    -> cleanNonEmptyText(minLength = 3, maxLength = 150),
-      "markdown" -> cleanNonEmptyText(minLength = 0, maxLength = 1000_000).into[Markdown],
+      "title" -> cleanNonEmptyText(minLength = 3, maxLength = 150),
+      "markdown" -> cleanTextWithSymbols
+        .verifying(Constraints.minLength(0), Constraints.maxLength(1000_000))
+        .into[Markdown],
       "language" -> LangForm.popularLanguages.mapping,
       "live"     -> boolean,
       "canonicalPath" -> optional:

diff --git a/modules/common/src/main/WMMatching.scala b/modules/common/src/main/WMMatching.scala
@@ -588,8 +588,7 @@ object WMMatching:
               if bestedge(w) == -1 || kslack < slack(bestedge(w)) then bestedge(w) = k
               false
             else false
-        if neighbend(v).exists(go) then true
-        else substage()
+        neighbend(v).exists(go) || substage()
     private val vertices = 0 until nvertex
     private def updateDual(): Boolean =
       // There is no augmenting path under these constraints;

diff --git a/modules/common/src/test/WMMatchingTest.scala b/modules/common/src/test/WMMatchingTest.scala
@@ -16,7 +16,7 @@ class WMMatchingTest extends munit.FunSuite:
       e += p._2
       w += p._3
     val l = toMate(n, WMMatching.maxWeightMatching(e.result(), w.result(), maxcardinality))
-    assert(if l eq null then false else expectedMate.sameElements(l))
+    assert(expectedMate.sameElements(l))
   //  check(Array((0,1,1)), false, List (1, 0))
   def check(n: Int, a: Array[Int], res: (Int, Int)) =
     val v           = Array.range(0, n)

diff --git a/modules/game/src/main/BinaryFormat.scala b/modules/game/src/main/BinaryFormat.scala
@@ -275,7 +275,7 @@ object BinaryFormat:
               if bitAt(int, j) == 1 then set = set + Square.at(7 - j, 7 * i).get
       UnmovedRooks(set)
 
-  @inline private def toInt(b: Byte): Int = b & 0xff
+  inline private def toInt(inline b: Byte): Int = b & 0xff
 
   def writeInt24(int: Int) =
     val i = if int < (1 << 24) then int else 0

diff --git a/modules/irc/src/main/IrcApi.scala b/modules/irc/src/main/IrcApi.scala
@@ -58,6 +58,11 @@ final class IrcApi(
             dox = false
           )
 
+  def fullCommExport(user: User)(using mod: Me): Funit =
+    val topic = "/" + user.username
+    zulip(_.mod.trustSafety, topic):
+      s"${markdown.modLink(mod.username)} exported all comms of ${markdown.userLink(user.username)}"
+
   def usertableCheck(user: User)(using mod: Me): Funit =
     zulip(_.mod.cafeteria, "reports"):
       s"**${markdown.userLinkNoNotes(user.username)}** usertable check (requested by ${markdown.modLink(mod.username)})"

diff --git a/modules/irc/src/main/ZulipClient.scala b/modules/irc/src/main/ZulipClient.scala
@@ -81,6 +81,7 @@ private object ZulipClient:
       val adminAppeal  = "mod-admin-appeal"
       val cafeteria    = "mod-cafeteria"
       val usernames    = "mod-usernames"
+      val trustSafety  = "org-trustsafety"
       def adminMonitor(tpe: IrcApi.ModDomain) = tpe match
         case IrcApi.ModDomain.Comm  => "mod-admin-monitor-comm"
         case IrcApi.ModDomain.Cheat => "mod-admin-monitor-cheat"

diff --git a/modules/mod/src/main/Modlog.scala b/modules/mod/src/main/Modlog.scala
@@ -52,6 +52,7 @@ case class Modlog(
     case Modlog.chatTimeout         => "chat timeout"
     case Modlog.troll               => "shadowban"
     case Modlog.untroll             => "un-shadowban"
+    case Modlog.fullCommsExport     => "exported all comms"
     case Modlog.permissions         => "set permissions"
     case Modlog.kickFromRankings    => "kick from rankings"
     case Modlog.reportban           => "reportban"
@@ -111,6 +112,7 @@ object Modlog:
   val unbooster           = "unbooster"
   val troll               = "troll"
   val untroll             = "untroll"
+  val fullCommsExport     = "fullCommsExport"
   val permissions         = "permissions"
   val disableTwoFactor    = "disableTwoFactor"
   val closeAccount        = "closeAccount"

diff --git a/modules/mod/src/main/ModlogApi.scala b/modules/mod/src/main/ModlogApi.scala
@@ -51,6 +51,9 @@ final class ModlogApi(repo: ModlogRepo, userRepo: UserRepo, ircApi: IrcApi, pres
   def troll(sus: Suspect)(using Me.Id) = add:
     Modlog.make(sus, if sus.user.marks.troll then Modlog.troll else Modlog.untroll)
 
+  def fullCommExport(sus: Suspect)(using Me.Id) = add:
+    Modlog.make(sus, Modlog.fullCommsExport)
+
   def setKidMode(mod: ModId, kid: UserId) = add:
     Modlog(mod, kid.some, Modlog.setKidMode)
 

diff --git a/modules/msg/src/main/MsgApi.scala b/modules/msg/src/main/MsgApi.scala
@@ -309,12 +309,7 @@ final class MsgApi(
                       $and(
                         $doc("$eq" -> $arr("$user", userId)),
                         $doc("$eq" -> $arr("$tid", "$$t")),
-                        $doc:
-                          "$not" -> $doc:
-                            "$regexMatch" -> $doc(
-                              "input" -> "$text",
-                              "regex" -> "You received this because you are (subscribed to messages|part) of the team"
-                            )
+                        excludeTeamMessages
                       )
             )
           ,
@@ -329,6 +324,48 @@ final class MsgApi(
           date <- msg.getAsOpt[Instant]("date")
         yield (text, date)).toList
 
+  private val excludeTeamMessages = $doc:
+    "$not" -> $doc:
+      "$regexMatch" -> $doc(
+        "input" -> "$text",
+        "regex" -> "You received this because you are (subscribed to messages|part) of the team"
+      )
+
+  // include responses from the other user
+  def modFullCommsExport(userId: UserId): Source[(MsgThread.Id, NonEmptyList[Msg]), ?] =
+    colls.thread
+      .aggregateWith[Bdoc](readPreference = ReadPref.priTemp): framework =>
+        import framework.*
+        List(
+          Match($doc("users" -> userId)),
+          Sort(Descending("lastMsg.date")),
+          Project($id(true)),
+          PipelineOperator:
+            $lookup.pipelineFull(
+              from = colls.msg.name,
+              as = "msgs",
+              let = $doc("t" -> "$_id"),
+              pipe = List(
+                $doc(
+                  "$match" ->
+                    $expr:
+                      $and(
+                        $doc("$eq" -> $arr("$tid", "$$t")),
+                        excludeTeamMessages
+                      )
+                ),
+                $doc("$sort" -> $sort.desc("date"))
+              )
+            )
+        )
+      .documentSource()
+      .mapConcat: doc =>
+        (for
+          tid <- doc.getAsOpt[MsgThread.Id]("_id")
+          // filter conversation where only team messages where sent
+          msgs <- doc.getAsOpt[NonEmptyList[Msg]]("msgs")
+        yield (tid, msgs)).toList
+
 object MsgApi:
   enum PostResult:
     case Success, Invalid, Limited, Bounced
diff --git a/modules/oauth/src/main/AccessTokenApi.scala b/modules/oauth/src/main/AccessTokenApi.scala
@@ -16,12 +16,24 @@ final class AccessTokenApi(
   import OAuthScope.given
   import AccessToken.{ BSONFields as F, given }
 
-  private def create(token: AccessToken): Fu[AccessToken] = coll.insert.one(token).inject(token)
+  private def createAndRotate(token: AccessToken): Fu[AccessToken] = for
+    oldIds <- coll
+      .find($doc(F.userId -> token.userId, F.clientOrigin -> token.clientOrigin), $doc(F.id -> true).some)
+      .sort($doc(F.usedAt -> -1, F.createdAt -> -1))
+      .skip(30)
+      .cursor[Bdoc]()
+      .listAll()
+      .dmap:
+        _.flatMap { _.getAsOpt[AccessToken.Id](F.id) }
+    _ <- oldIds.nonEmpty.so:
+      coll.delete.one($doc(F.id.$in(oldIds))).void
+    _ <- coll.insert.one(token)
+  yield token
 
   def create(setup: OAuthTokenForm.Data, me: User, isStudent: Boolean): Fu[AccessToken] =
-    (fuccess(isStudent) >>| userRepo.isManaged(me.id)) flatMap { noBot =>
+    (fuccess(isStudent) >>| userRepo.isManaged(me.id)).flatMap { noBot =>
       val plain = Bearer.randomPersonal()
-      create:
+      createAndRotate:
         AccessToken(
           id = AccessToken.Id.from(plain),
           plain = plain,
@@ -42,7 +54,7 @@ final class AccessTokenApi(
 
   def create(granted: AccessTokenRequest.Granted): Fu[AccessToken] =
     val plain = Bearer.random()
-    create:
+    createAndRotate:
       AccessToken(
         id = AccessToken.Id.from(plain),
         plain = plain,
@@ -73,7 +85,7 @@ final class AccessTokenApi(
                 )
               .getOrElse:
                 val plain = Bearer.randomPersonal()
-                create:
+                createAndRotate:
                   AccessToken(
                     id = AccessToken.Id.from(plain),
                     plain = plain,
@@ -158,7 +170,8 @@ final class AccessTokenApi(
           F.id     -> id,
           F.userId -> user.id
         )
-      .void.andDo(onRevoke(id))
+      .void
+      .andDo(onRevoke(id))
 
   def revokeByClientOrigin(clientOrigin: String, user: User): Funit =
     coll
@@ -190,11 +203,11 @@ final class AccessTokenApi(
   def test(bearers: List[Bearer]): Fu[Map[Bearer, Option[AccessToken]]] =
     coll
       .optionsByOrderedIds[AccessToken, AccessToken.Id](
-        bearers map AccessToken.Id.from,
+        bearers.map(AccessToken.Id.from),
         readPref = _.sec
       )(_.id)
       .flatMap: tokens =>
-        userRepo.filterDisabled(tokens.flatten.map(_.userId)) map { closedUserIds =>
+        userRepo.filterDisabled(tokens.flatten.map(_.userId)).map { closedUserIds =>
           val openTokens = tokens.map(_.filter(token => !closedUserIds(token.userId)))
           bearers.zip(openTokens).toMap
         }

diff --git a/modules/relay/src/main/JsonView.scala b/modules/relay/src/main/JsonView.scala
@@ -104,12 +104,13 @@ final class JsonView(
         .add("isSubscribed" -> isSubscribed)
         .add("videoUrls" -> videoUrls),
       study = studyData.study,
-      analysis = studyData.analysis
+      analysis = studyData.analysis,
+      group = group.map(_.group.name)
     )
 
 object JsonView:
 
-  case class JsData(relay: JsObject, study: JsObject, analysis: JsObject)
+  case class JsData(relay: JsObject, study: JsObject, analysis: JsObject, group: Option[RelayGroup.Name])
 
   given OWrites[SyncLog.Event] = Json.writes
 

diff --git a/modules/round/src/main/SelfReport.scala b/modules/round/src/main/SelfReport.scala
@@ -32,12 +32,11 @@ final class SelfReport(
       //   Env.report.api.autoBotReport(u.id, referer, name)
       // }
       def doLog(): Unit =
-        if name != "ceval" then
-          if logOnceEvery(ip.str) then
-            lila.log("cheat").branch("jslog").info {
-              s"$ip https://lichess.org/$fullId ${user.fold("anon")(_.id)} $name"
-            }
-            lila.mon.cheat.selfReport(name, userId.isDefined).increment()
+        if name != "ceval" && logOnceEvery(ip.str) then
+          lila.log("cheat").branch("jslog").info {
+            s"$ip https://lichess.org/$fullId ${user.fold("anon")(_.id)} $name"
+          }
+          lila.mon.cheat.selfReport(name, userId.isDefined).increment()
       if fullId.value == "____________" then doLog()
       else
         proxyRepo