-
Notifications
You must be signed in to change notification settings - Fork 53
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add a NSFW protection #520
Changes from 5 commits
147f595
0ee021b
0b3734d
f2e5ba8
1893b3c
61a0fd5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
/* | ||
Copyright 2024 The Matrix.org Foundation C.I.C. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
import { Protection } from "./IProtection"; | ||
import { Mjolnir } from "../Mjolnir"; | ||
import * as nsfw from 'nsfwjs'; | ||
import {LogLevel} from "matrix-bot-sdk"; | ||
import { node } from '@tensorflow/tfjs-node'; | ||
|
||
|
||
export class NsfwProtection extends Protection { | ||
settings = {}; | ||
// @ts-ignore | ||
private model: any; | ||
|
||
constructor() { | ||
super(); | ||
} | ||
|
||
async initialize() { | ||
this.model = await nsfw.load(); | ||
} | ||
|
||
public get name(): string { | ||
return 'NsfwProtection'; | ||
} | ||
|
||
public get description(): string { | ||
return "Scans all images sent into a protected room to determine if the image is " + | ||
"NSFW. If it is, the image will automatically be redacted."; | ||
} | ||
|
||
public async handleEvent(mjolnir: Mjolnir, roomId: string, event: any): Promise<any> { | ||
if (event['type'] === 'm.room.message') { | ||
const content = event['content'] || {}; | ||
const msgtype = content['msgtype'] || 'm.text'; | ||
const formattedBody = content['formatted_body'] || ''; | ||
const isMedia = msgtype === 'm.image' || formattedBody.toLowerCase().includes('<img'); | ||
|
||
if (isMedia) { | ||
const mxc = content["url"] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldnt the thumbnail be checked as well? It might be a different image? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Possibly, though thumbnails are a bit weird already. This can be revisted in a future PR. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if it helps this is how i've laid out Draupnir's attempt to copy this protection, so that it's possible to scrape out more urls from the message https://github.com/the-draupnir-project/Draupnir/blob/gnuxie/nsfwjs/src/protections/NSFWImageProtection.tsx#L55-L71 (incomplete code though, would like to avoid depending on |
||
const image = await mjolnir.client.downloadContent(mxc) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Won't this break if there is an event with There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. likely, yea. I think we should just remove the formatted body condition for now - parsing the HTML can be a future PR. |
||
const decodedImage = await node.decodeImage(image.data, 3); | ||
const predictions = await this.model.classify(decodedImage) | ||
|
||
for (const prediction of predictions) { | ||
if (prediction["className"] === "Porn") { | ||
if (prediction["probability"] > mjolnir.config.nsfwSensitivity) { | ||
await mjolnir.managementRoomOutput.logMessage(LogLevel.INFO, "NSFWProtection", `Redacting ${event["event_id"]} for inappropriate content.`); | ||
try { | ||
mjolnir.client.redactEvent(roomId, event["event_id"]) | ||
} catch (err) { | ||
await mjolnir.managementRoomOutput.logMessage(LogLevel.ERROR, "NSFWProtection", `There was an error redacting ${event["event_id"]}: ${err}`); | ||
|
||
} | ||
} | ||
} else if (prediction["className"] === "Hentai") { | ||
if (prediction["probability"] > mjolnir.config.nsfwSensitivity) { | ||
await mjolnir.managementRoomOutput.logMessage(LogLevel.INFO, "NSFWProtection", `Redacting ${event["event_id"]} for inappropriate content.`); | ||
try { | ||
mjolnir.client.redactEvent(roomId, event["event_id"]) | ||
} catch (err) { | ||
await mjolnir.managementRoomOutput.logMessage(LogLevel.ERROR, "NSFWProtection", `There was an error redacting ${event["event_id"]}: ${err}`); | ||
} | ||
} | ||
} | ||
turt2live marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
decodedImage.dispose() | ||
} | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
import {newTestUser} from "./clientHelper"; | ||
|
||
import {MatrixClient} from "matrix-bot-sdk"; | ||
import {getFirstReaction} from "./commands/commandUtils"; | ||
import {strict as assert} from "assert"; | ||
import { readFileSync } from 'fs'; | ||
|
||
describe("Test: NSFW protection", function () { | ||
let client: MatrixClient; | ||
let room: string; | ||
this.beforeEach(async function () { | ||
client = await newTestUser(this.config.homeserverUrl, {name: {contains: "nsfw-protection"}}); | ||
await client.start(); | ||
const mjolnirId = await this.mjolnir.client.getUserId(); | ||
room = await client.createRoom({ invite: [mjolnirId] }); | ||
await client.joinRoom(room); | ||
await client.joinRoom(this.config.managementRoom); | ||
await client.setUserPowerLevel(mjolnirId, room, 100) | ||
}) | ||
this.afterEach(async function () { | ||
await client.stop(); | ||
}) | ||
|
||
function delay(ms: number) { | ||
return new Promise(resolve => setTimeout(resolve, ms)); | ||
} | ||
|
||
|
||
it("Nsfw protection doesn't redact sfw images", async function() { | ||
this.timeout(20000); | ||
|
||
await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir rooms add ${room}` }); | ||
await getFirstReaction(client, this.mjolnir.managementRoomId, '✅', async () => { | ||
return await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir enable NsfwProtection` }); | ||
}); | ||
|
||
const data = readFileSync('test_tree.jpg') | ||
const mxc = await client.uploadContent(data, 'image/png') | ||
let content = {"msgtype": "m.image", "body": "test.jpeg", "url": mxc} | ||
let imageMessage = await client.sendMessage(room, content) | ||
|
||
await delay(500) | ||
let processedImage = await client.getEvent(room, imageMessage); | ||
assert.equal(Object.keys(processedImage.content).length, 3, "This event should not have been redacted"); | ||
}); | ||
|
||
it("Nsfw protection redacts nsfw images", async function() { | ||
this.timeout(20000); | ||
// dial the sensitivity on the protection way up so that all images are flagged as NSFW | ||
this.mjolnir.config.nsfwSensitivity = 0.0 | ||
H-Shay marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir rooms add ${room}` }); | ||
await getFirstReaction(client, this.mjolnir.managementRoomId, '✅', async () => { | ||
return await client.sendMessage(this.mjolnir.managementRoomId, { msgtype: 'm.text', body: `!mjolnir enable NsfwProtection` }); | ||
}); | ||
|
||
const data = readFileSync('test_tree.jpg') | ||
const mxc = await client.uploadContent(data, 'image/png') | ||
let content = {"msgtype": "m.image", "body": "test.jpeg", "url": mxc} | ||
let imageMessage = await client.sendMessage(room, content) | ||
|
||
await delay(500) | ||
let processedImage = await client.getEvent(room, imageMessage); | ||
assert.equal(Object.keys(processedImage.content).length, 0, "This event should have been redacted"); | ||
}); | ||
}); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This will pull in over
285MB
of dependencies btw, which is more weight than the entire mjolnir docker image.https://pkg-size.dev/@tensorflow%2Ftfjs-node
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Image size isn't really a concern here, but if you're aware of alternatives, we can certainly consider them.