From 7f16cb1dbf525561918dda78af13562bc8557a20 Mon Sep 17 00:00:00 2001 From: nick-funk Date: Tue, 11 Aug 2020 15:29:08 -0600 Subject: [PATCH] Mark and allow word list timed out comments Will mark the comment's metdata as a word list time out if it happens and also tag them with suspect or banned word. Then we show "possible suspect/banned word" markers on the moderate card so moderators can manually review them. CORL-1264 --- .../ModerateCard/MarkersContainer.spec.tsx | 10 +++++++ .../ModerateCard/MarkersContainer.tsx | 29 +++++++++++++++++-- .../MarkersContainer.spec.tsx.snap | 10 +++++++ src/core/server/graph/schema/schema.graphql | 14 +++++++++ src/core/server/models/comment/revision.ts | 12 ++++++++ .../comments/pipeline/phases/wordList.ts | 12 ++++++-- src/locales/en-US/admin.ftl | 2 ++ 7 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/core/client/admin/components/ModerateCard/MarkersContainer.spec.tsx b/src/core/client/admin/components/ModerateCard/MarkersContainer.spec.tsx index cc6f3a1464..1c2ab90bb5 100644 --- a/src/core/client/admin/components/ModerateCard/MarkersContainer.spec.tsx +++ b/src/core/client/admin/components/ModerateCard/MarkersContainer.spec.tsx @@ -32,6 +32,11 @@ it("renders all markers", () => { }, }, }, + metadata: { + wordList: { + timedOut: false, + }, + }, }, }, settings: { @@ -71,6 +76,11 @@ it("renders some markers", () => { }, }, }, + metadata: { + wordList: { + timedOut: false, + }, + }, }, }, settings: { diff --git a/src/core/client/admin/components/ModerateCard/MarkersContainer.tsx b/src/core/client/admin/components/ModerateCard/MarkersContainer.tsx index 7d1ab00808..c441da276a 100644 --- a/src/core/client/admin/components/ModerateCard/MarkersContainer.tsx +++ b/src/core/client/admin/components/ModerateCard/MarkersContainer.tsx @@ -38,7 +38,17 @@ const markers: Array<( null, (c) => (c.revision && - c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_BANNED_WORD && ( + c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_BANNED_WORD && + c.revision.metadata?.wordList?.timedOut && ( + + Possible Banned Word + + )) || + null, + (c) => + (c.revision && + c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_BANNED_WORD && + !c.revision.metadata?.wordList?.timedOut && ( Banned Word @@ -46,7 +56,17 @@ const markers: Array<( null, (c) => (c.revision && - c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_SUSPECT_WORD && ( + c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_SUSPECT_WORD && + c.revision.metadata?.wordList?.timedOut && ( + + Possible Suspect Word + + )) || + null, + (c) => + (c.revision && + c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_SUSPECT_WORD && + !c.revision.metadata?.wordList?.timedOut && ( Suspect Word @@ -193,6 +213,11 @@ const enhanced = withFragmentContainer({ } } } + metadata { + wordList { + timedOut + } + } } } `, diff --git a/src/core/client/admin/components/ModerateCard/__snapshots__/MarkersContainer.spec.tsx.snap b/src/core/client/admin/components/ModerateCard/__snapshots__/MarkersContainer.spec.tsx.snap index 0a40f90b38..1b1eab6191 100644 --- a/src/core/client/admin/components/ModerateCard/__snapshots__/MarkersContainer.spec.tsx.snap +++ b/src/core/client/admin/components/ModerateCard/__snapshots__/MarkersContainer.spec.tsx.snap @@ -25,6 +25,11 @@ exports[`renders all markers 1`] = ` }, }, }, + "metadata": Object { + "wordList": Object { + "timedOut": false, + }, + }, }, "status": "PREMOD", } @@ -172,6 +177,11 @@ exports[`renders some markers 1`] = ` }, }, }, + "metadata": Object { + "wordList": Object { + "timedOut": false, + }, + }, }, "status": "PREMOD", } diff --git a/src/core/server/graph/schema/schema.graphql b/src/core/server/graph/schema/schema.graphql index 94bd5f043e..1e6e39e8e2 100644 --- a/src/core/server/graph/schema/schema.graphql +++ b/src/core/server/graph/schema/schema.graphql @@ -2655,12 +2655,26 @@ type CommentRevisionPerspectiveMetadata { score: Float! } +type CommentRevisionWordListMetadata { + """ + timedOut is whether the wordlist analysis timed out when this revision + of the comment was sent through the moderation phases. + """ + timedOut: Boolean +} + type CommentRevisionMetadata { """ perspective stores metadata associated with the pipeline analysis of this revision's body. """ perspective: CommentRevisionPerspectiveMetadata + + """ + wordList metadata stores extra status details about what occurred during + word list analysis of this comment revision. + """ + wordList: CommentRevisionWordListMetadata } """ diff --git a/src/core/server/models/comment/revision.ts b/src/core/server/models/comment/revision.ts index 0738c6f4d7..887df19839 100644 --- a/src/core/server/models/comment/revision.ts +++ b/src/core/server/models/comment/revision.ts @@ -28,6 +28,18 @@ export interface RevisionMetadata { model: string; }; + /** + * wordList metadata stores extra status details about what occurred during + * word list analysis of this comment revision. + */ + wordList?: { + /** + * timedOut is whether the wordlist analysis timed out when this revision + * of the comment was sent through the moderation phases. + */ + timedOut?: boolean; + }; + /** * nudge when true indicates that the comment was written on the first try * without a warning. diff --git a/src/core/server/services/comments/pipeline/phases/wordList.ts b/src/core/server/services/comments/pipeline/phases/wordList.ts index 0e6c162b05..7687855228 100755 --- a/src/core/server/services/comments/pipeline/phases/wordList.ts +++ b/src/core/server/services/comments/pipeline/phases/wordList.ts @@ -43,13 +43,17 @@ export const wordList: IntermediateModerationPhase = ({ }; } else if (banned === null) { return { - status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD, actions: [ { actionType: ACTION_TYPE.FLAG, reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_BANNED_WORD, }, ], + metadata: { + wordList: { + timedOut: true, + }, + }, }; } @@ -66,13 +70,17 @@ export const wordList: IntermediateModerationPhase = ({ }; } else if (suspect === null) { return { - status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD, actions: [ { actionType: ACTION_TYPE.FLAG, reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_SUSPECT_WORD, }, ], + metadata: { + wordList: { + timedOut: true, + }, + }, }; } }; diff --git a/src/locales/en-US/admin.ftl b/src/locales/en-US/admin.ftl index 96988e27af..cf29d1abbf 100644 --- a/src/locales/en-US/admin.ftl +++ b/src/locales/en-US/admin.ftl @@ -849,7 +849,9 @@ moderate-navigation-comment-count = { SHORT_NUMBER($count) } moderate-marker-preMod = Pre-mod moderate-marker-link = Link moderate-marker-bannedWord = Banned word +moderate-marker-possibleBannedWord = Possible Banned Word moderate-marker-suspectWord = Suspect word +moderate-marker-possibleSuspectWord = Possible Suspect Word moderate-marker-spam = Spam moderate-marker-spamDetected = Spam detected moderate-marker-toxic = Toxic