Skip to content

Commit

Permalink
Content reporting on record fields (#1351)
Browse files Browse the repository at this point in the history
* content reporting on record fields

* fix test

* tests

* tidy
  • Loading branch information
dholms authored Jul 17, 2023
1 parent 8c9becc commit c9acdd3
Show file tree
Hide file tree
Showing 15 changed files with 329 additions and 101 deletions.
5 changes: 1 addition & 4 deletions packages/pds/src/api/com/atproto/identity/updateHandle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {
UserAlreadyExistsError,
} from '../../../../services/account'
import { httpLogger } from '../../../../logger'
import { backgroundHandleCheckForFlag } from '../../../../handle/moderation'

export default function (server: Server, ctx: AppContext) {
server.com.atproto.identity.updateHandle({
Expand Down Expand Up @@ -55,9 +54,7 @@ export default function (server: Server, ctx: AppContext) {
)
}

if (ctx.cfg.unacceptableHandleWordsB64) {
backgroundHandleCheckForFlag({ ctx, handle, did: requester })
}
ctx.contentReporter?.checkHandle({ handle, did: requester })
},
})
}
5 changes: 1 addition & 4 deletions packages/pds/src/api/com/atproto/server/createAccount.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import { UserAlreadyExistsError } from '../../../../services/account'
import AppContext from '../../../../context'
import Database from '../../../../db'
import { AtprotoData } from '@atproto/identity'
import { backgroundHandleCheckForFlag } from '../../../../handle/moderation'

export default function (server: Server, ctx: AppContext) {
server.com.atproto.server.createAccount(async ({ input, req }) => {
Expand Down Expand Up @@ -106,9 +105,7 @@ export default function (server: Server, ctx: AppContext) {
}
})

if (ctx.cfg.unacceptableHandleWordsB64) {
backgroundHandleCheckForFlag({ ctx, handle, did: result.did })
}
ctx.contentReporter?.checkHandle({ handle, did: result.did })

return {
encoding: 'application/json',
Expand Down
24 changes: 12 additions & 12 deletions packages/pds/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ export interface ServerConfigValues {
hiveApiKey?: string
labelerDid: string
labelerKeywords: Record<string, string>
unacceptableHandleWordsB64?: string
falsePositiveHandleWordsB64?: string
unacceptableWordsB64?: string
falsePositiveWordsB64?: string

feedGenDid?: string

Expand Down Expand Up @@ -166,11 +166,11 @@ export class ServerConfig {
const labelerDid = process.env.LABELER_DID || 'did:example:labeler'
const labelerKeywords = {}

const unacceptableHandleWordsB64 = nonemptyString(
process.env.UNACCEPTABLE_HANDLE_WORDS_B64,
const unacceptableWordsB64 = nonemptyString(
process.env.UNACCEPTABLE_WORDS_B64,
)
const falsePositiveHandleWordsB64 = nonemptyString(
process.env.FALSE_POSITIVE_HANDLE_WORDS_B64,
const falsePositiveWordsB64 = nonemptyString(
process.env.FALSE_POSITIVE_WORDS_B64,
)

const feedGenDid = process.env.FEED_GEN_DID
Expand Down Expand Up @@ -250,8 +250,8 @@ export class ServerConfig {
hiveApiKey,
labelerDid,
labelerKeywords,
unacceptableHandleWordsB64,
falsePositiveHandleWordsB64,
unacceptableWordsB64,
falsePositiveWordsB64,
feedGenDid,
maxSubscriptionBuffer,
repoBackfillLimitMs,
Expand Down Expand Up @@ -444,12 +444,12 @@ export class ServerConfig {
return this.cfg.labelerKeywords
}

get unacceptableHandleWordsB64() {
return this.cfg.unacceptableHandleWordsB64
get unacceptableWordsB64() {
return this.cfg.unacceptableWordsB64
}

get falsePositiveHandleWordsB64() {
return this.cfg.falsePositiveHandleWordsB64
get falsePositiveWordsB64() {
return this.cfg.falsePositiveWordsB64
}

get feedGenDid() {
Expand Down
21 changes: 21 additions & 0 deletions packages/pds/src/content-reporter/explicit-slurs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// regexes taken from: https://github.com/Blank-Cheque/Slurs
/* eslint-disable no-misleading-character-class */
const explicitSlurRegexes = [
/\b[cĆćĈĉČčĊċÇçḈḉȻȼꞒꞓꟄꞔƇƈɕ][hĤĥȞȟḦḧḢḣḨḩḤḥḪḫH̱ẖĦħⱧⱨꞪɦꞕΗНн][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLl][nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпNn][kḰḱǨǩĶķḲḳḴḵƘƙⱩⱪᶄꝀꝁꝂꝃꝄꝅꞢꞣ][sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?\b/,
/\b[cĆćĈĉČčĊċÇçḈḉȻȼꞒꞓꟄꞔƇƈɕ][ÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺOo0]{2}[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпNn][sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?\b/,
/\b[fḞḟƑƒꞘꞙᵮᶂ][aÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚAa@4][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶGgqꝖꝗꝘꝙɋʠ]{1,2}([ÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺOo0e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEeiÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLl][tŤťṪṫŢţṬṭȚțṰṱṮṯŦŧȾⱦƬƭƮʈT̈ẗᵵƫȶ]{1,2}([rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][yÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ]|[rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLl][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe])?)?[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?\b/,
/\b[kḰḱǨǩĶķḲḳḴḵƘƙⱩⱪᶄꝀꝁꝂꝃꝄꝅꞢꞣ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLlyÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ][kḰḱǨǩĶķḲḳḴḵƘƙⱩⱪᶄꝀꝁꝂꝃꝄꝅꞢꞣ][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe]([rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][yÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ]|[rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLl][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe])?[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]*\b/,
/\b[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпNn][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLloÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺOoІіa4ÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚAa][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶGgqꝖꝗꝘꝙɋʠ]{2}(l[e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe]t|[e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEeaÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚAa][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ]?|n[ÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺOo0][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶGgqꝖꝗꝘꝙɋʠ]|[a4ÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚAa]?)?[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?\b/,
/[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпNn][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLloÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺOoІіa4ÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚAa][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶGgqꝖꝗꝘꝙɋʠ]{2}(l[e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe]t|[e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ])[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?/,
/\b[tŤťṪṫŢţṬṭȚțṰṱṮṯŦŧȾⱦƬƭƮʈT̈ẗᵵƫȶ][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][aÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚAa4]+[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпNn]{1,2}([iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıIi1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴLl][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe]|[yÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ]|[e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳEe][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ])[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?\b/,
]

export const hasExplicitSlur = (handle: string): boolean => {
return explicitSlurRegexes.some(
(reg) =>
reg.test(handle) ||
reg.test(
handle.replaceAll('.', '').replaceAll('-', '').replaceAll('_', ''),
),
)
}
88 changes: 88 additions & 0 deletions packages/pds/src/content-reporter/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { AtUri } from '@atproto/uri'
import { RepoRecord } from '@atproto/lexicon'
import { CID } from 'multiformats/cid'
import * as ui8 from 'uint8arrays'
import { UnacceptableWordValidator } from './validator'
import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs'
import { isRecord as isList } from '../lexicon/types/app/bsky/graph/list'
import { isRecord as isProfile } from '../lexicon/types/app/bsky/actor/profile'
import { isRecord as isFeedGenerator } from '../lexicon/types/app/bsky/feed/generator'
import { BackgroundQueue } from '../event-stream/background-queue'
import { ModerationService } from '../services/moderation'

export class ContentReporter {
backgroundQueue: BackgroundQueue
moderationService: ModerationService
reporterDid: string
validator: UnacceptableWordValidator

constructor(opts: {
backgroundQueue: BackgroundQueue
moderationService: ModerationService
reporterDid: string
unacceptableB64: string
falsePositivesB64?: string
}) {
this.backgroundQueue = opts.backgroundQueue
this.moderationService = opts.moderationService
this.reporterDid = opts.reporterDid
this.validator = new UnacceptableWordValidator(
decode(opts.unacceptableB64),
opts.falsePositivesB64 ? decode(opts.falsePositivesB64) : undefined,
)
}

checkHandle(opts: { handle: string; did: string }) {
const { handle, did } = opts
return this.checkContent({
content: handle,
subject: { did },
})
}

checkRecord(opts: { record: RepoRecord; uri: AtUri; cid: CID }) {
const { record, uri, cid } = opts
let content = uri.rkey
if (isProfile(record)) {
content += ' ' + record.displayName
} else if (isList(record)) {
content += ' ' + record.name
} else if (isFeedGenerator(record)) {
content += ' ' + record.displayName
}

return this.checkContent({
content,
subject: { uri, cid },
})
}

checkContent(opts: {
content: string
subject: { did: string } | { uri: AtUri; cid?: CID }
}) {
const { content, subject } = opts
const possibleSlurs = this.validator.getMatches(content)
if (possibleSlurs.length < 1) {
return
}
this.backgroundQueue.add(async () => {
await this.moderationService.report({
reasonType: REASONOTHER,
reason: `Automatically flagged for possible slurs: ${possibleSlurs.join(
', ',
)}`,
subject,
reportedBy: this.reporterDid,
})
})
}
}

export const decode = (encoded: string): string[] => {
return ui8.toString(ui8.fromString(encoded, 'base64'), 'utf8').split(',')
}

export const encode = (words: string[]): string => {
return ui8.toString(ui8.fromString(words.join(','), 'utf8'), 'base64')
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { dedupeStrs } from '@atproto/common'

export class UnacceptableHandleValidator {
export class UnacceptableWordValidator {
private bannedWords: Set<string>
private falsePositives: Set<string>

Expand Down
6 changes: 6 additions & 0 deletions packages/pds/src/context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import DidSqlCache from './did-cache'
import { MountedAlgos } from './feed-gen/types'
import { Crawlers } from './crawlers'
import { LabelCache } from './label-cache'
import { ContentReporter } from './content-reporter'

export class AppContext {
private _appviewAgent: AtpAgent | null
Expand All @@ -41,6 +42,7 @@ export class AppContext {
sequencerLeader: SequencerLeader | null
labeler: Labeler
labelCache: LabelCache
contentReporter?: ContentReporter
backgroundQueue: BackgroundQueue
crawlers: Crawlers
algos: MountedAlgos
Expand Down Expand Up @@ -133,6 +135,10 @@ export class AppContext {
return this.opts.labelCache
}

get contentReporter(): ContentReporter | undefined {
return this.opts.contentReporter
}

get backgroundQueue(): BackgroundQueue {
return this.opts.backgroundQueue
}
Expand Down
4 changes: 2 additions & 2 deletions packages/pds/src/handle/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as ident from '@atproto/identifier'
import { InvalidRequestError } from '@atproto/xrpc-server'
import { reservedSubdomains } from './reserved'
import { hasExplicitSlur } from './moderation'
import { hasExplicitSlur } from '../content-reporter/explicit-slurs'
import AppContext from '../context'

export const normalizeAndValidateHandle = async (opts: {
Expand All @@ -20,7 +20,7 @@ export const normalizeAndValidateHandle = async (opts: {
'InvalidHandle',
)
}
// slur check etc
// slur check
if (hasExplicitSlur(handle)) {
throw new InvalidRequestError(
'Inappropriate language in handle',
Expand Down
Loading

0 comments on commit c9acdd3

Please sign in to comment.