Skip to content

Commit

Permalink
Mark and allow word list timed out comments
Browse files Browse the repository at this point in the history
Will mark the comment's metdata as a word list
time out if it happens and also tag them with
suspect or banned word.

Then we show "possible suspect/banned word"
markers on the moderate card so moderators can
manually review them.

CORL-1264
  • Loading branch information
nick-funk committed Aug 11, 2020
1 parent fc6de50 commit 7f16cb1
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 4 deletions.
Expand Up @@ -32,6 +32,11 @@ it("renders all markers", () => {
},
},
},
metadata: {
wordList: {
timedOut: false,
},
},
},
},
settings: {
Expand Down Expand Up @@ -71,6 +76,11 @@ it("renders some markers", () => {
},
},
},
metadata: {
wordList: {
timedOut: false,
},
},
},
},
settings: {
Expand Down
29 changes: 27 additions & 2 deletions src/core/client/admin/components/ModerateCard/MarkersContainer.tsx
Expand Up @@ -38,15 +38,35 @@ const markers: Array<(
null,
(c) =>
(c.revision &&
c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_BANNED_WORD && (
c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_BANNED_WORD &&
c.revision.metadata?.wordList?.timedOut && (
<Localized id="moderate-marker-possibleBannedWord" key={keyCounter++}>
<Marker color="reported">Possible Banned Word</Marker>
</Localized>
)) ||
null,
(c) =>
(c.revision &&
c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_BANNED_WORD &&
!c.revision.metadata?.wordList?.timedOut && (
<Localized id="moderate-marker-bannedWord" key={keyCounter++}>
<Marker color="reported">Banned Word</Marker>
</Localized>
)) ||
null,
(c) =>
(c.revision &&
c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_SUSPECT_WORD && (
c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_SUSPECT_WORD &&
c.revision.metadata?.wordList?.timedOut && (
<Localized id="moderate-marker-possibleSuspectWord" key={keyCounter++}>
<Marker color="reported">Possible Suspect Word</Marker>
</Localized>
)) ||
null,
(c) =>
(c.revision &&
c.revision.actionCounts.flag.reasons.COMMENT_DETECTED_SUSPECT_WORD &&
!c.revision.metadata?.wordList?.timedOut && (
<Localized id="moderate-marker-suspectWord" key={keyCounter++}>
<Marker color="reported">Suspect Word</Marker>
</Localized>
Expand Down Expand Up @@ -193,6 +213,11 @@ const enhanced = withFragmentContainer<MarkersContainerProps>({
}
}
}
metadata {
wordList {
timedOut
}
}
}
}
`,
Expand Down
Expand Up @@ -25,6 +25,11 @@ exports[`renders all markers 1`] = `
},
},
},
"metadata": Object {
"wordList": Object {
"timedOut": false,
},
},
},
"status": "PREMOD",
}
Expand Down Expand Up @@ -172,6 +177,11 @@ exports[`renders some markers 1`] = `
},
},
},
"metadata": Object {
"wordList": Object {
"timedOut": false,
},
},
},
"status": "PREMOD",
}
Expand Down
14 changes: 14 additions & 0 deletions src/core/server/graph/schema/schema.graphql
Expand Up @@ -2655,12 +2655,26 @@ type CommentRevisionPerspectiveMetadata {
score: Float!
}

type CommentRevisionWordListMetadata {
"""
timedOut is whether the wordlist analysis timed out when this revision
of the comment was sent through the moderation phases.
"""
timedOut: Boolean
}

type CommentRevisionMetadata {
"""
perspective stores metadata associated with the pipeline analysis of this
revision's body.
"""
perspective: CommentRevisionPerspectiveMetadata

"""
wordList metadata stores extra status details about what occurred during
word list analysis of this comment revision.
"""
wordList: CommentRevisionWordListMetadata
}

"""
Expand Down
12 changes: 12 additions & 0 deletions src/core/server/models/comment/revision.ts
Expand Up @@ -28,6 +28,18 @@ export interface RevisionMetadata {
model: string;
};

/**
* wordList metadata stores extra status details about what occurred during
* word list analysis of this comment revision.
*/
wordList?: {
/**
* timedOut is whether the wordlist analysis timed out when this revision
* of the comment was sent through the moderation phases.
*/
timedOut?: boolean;
};

/**
* nudge when true indicates that the comment was written on the first try
* without a warning.
Expand Down
12 changes: 10 additions & 2 deletions src/core/server/services/comments/pipeline/phases/wordList.ts
Expand Up @@ -43,13 +43,17 @@ export const wordList: IntermediateModerationPhase = ({
};
} else if (banned === null) {
return {
status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD,
actions: [
{
actionType: ACTION_TYPE.FLAG,
reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_BANNED_WORD,
},
],
metadata: {
wordList: {
timedOut: true,
},
},
};
}

Expand All @@ -66,13 +70,17 @@ export const wordList: IntermediateModerationPhase = ({
};
} else if (suspect === null) {
return {
status: GQLCOMMENT_STATUS.SYSTEM_WITHHELD,
actions: [
{
actionType: ACTION_TYPE.FLAG,
reason: GQLCOMMENT_FLAG_REASON.COMMENT_DETECTED_SUSPECT_WORD,
},
],
metadata: {
wordList: {
timedOut: true,
},
},
};
}
};
2 changes: 2 additions & 0 deletions src/locales/en-US/admin.ftl
Expand Up @@ -849,7 +849,9 @@ moderate-navigation-comment-count = { SHORT_NUMBER($count) }
moderate-marker-preMod = Pre-mod
moderate-marker-link = Link
moderate-marker-bannedWord = Banned word
moderate-marker-possibleBannedWord = Possible Banned Word
moderate-marker-suspectWord = Suspect word
moderate-marker-possibleSuspectWord = Possible Suspect Word
moderate-marker-spam = Spam
moderate-marker-spamDetected = Spam detected
moderate-marker-toxic = Toxic
Expand Down

0 comments on commit 7f16cb1

Please sign in to comment.