From c7c3a6d3426568d46aae0b033b38ad8c22fa240e Mon Sep 17 00:00:00 2001 From: MystyPy Date: Fri, 24 May 2024 11:46:16 +1000 Subject: [PATCH 01/12] Initial commit for security scanners. Discord, PyPi and GitHub included. --- core/scanners.py | 131 ++++++++++++++++++++++++++++++++++++++++++++++ types_/scanner.py | 30 +++++++++++ 2 files changed, 161 insertions(+) create mode 100644 core/scanners.py create mode 100644 types_/scanner.py diff --git a/core/scanners.py b/core/scanners.py new file mode 100644 index 0000000..f8bff78 --- /dev/null +++ b/core/scanners.py @@ -0,0 +1,131 @@ +"""MystBin. Share code easily. + +Copyright (C) 2020-Current PythonistaGuild + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import base64 +import binascii +import enum +import logging +import re +from typing import ClassVar + +from types_.scanner import ScannerSecret + + +logger: logging.Logger = logging.getLogger(__name__) + + +class Services(enum.Enum): + discord = "Discord" + pypi = "PyPi" + github = "GitHub" + + +class BaseScanner: + REGEX: ClassVar[re.Pattern[str]] + SERVICE: Services + + @classmethod + def match(cls, content: str) -> ScannerSecret: + payload: ScannerSecret = { + "service": cls.SERVICE, + "tokens": set(cls.REGEX.findall(content)), + } + + return payload + + +class DiscordScanner(BaseScanner): + REGEX = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") + SERVICE = Services.discord + + @staticmethod + def validate_discord_token(token: str) -> bool: + try: + # Just check if the first part validates as a user ID + (user_id, _, _) = token.split(".") + user_id = int(base64.b64decode(user_id + "==", validate=True)) + except (ValueError, binascii.Error): + return False + else: + return True + + @classmethod + def match(cls, content: str) -> ScannerSecret: + payload: ScannerSecret = { + "service": cls.SERVICE, + "tokens": {t for t in cls.REGEX.findall(content) if cls.validate_discord_token(t)}, + } + + return payload + + +class PyPiScanner(BaseScanner): + REGEX = re.compile(r"pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{70,}") + SERVICE = Services.pypi + + +class GitHubScanner(BaseScanner): + REGEX = re.compile(r"((ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36})") + SERVICE = Services.github + + @classmethod + def match(cls, content: str) -> ScannerSecret: + payload: ScannerSecret = { + "service": cls.SERVICE, + "tokens": {t[0] for t in cls.REGEX.findall(content)}, + } + + return payload + + +class SecurityInfo: + __SERVICE_MAPPING: ClassVar[dict[Services, type[BaseScanner]]] = { + Services.discord: DiscordScanner, + Services.pypi: PyPiScanner, + Services.github: GitHubScanner, + } + + @classmethod + def scan_file( + cls, + file: str, + /, + *, + allowed: list[Services] = [], + disallowed: list[Services] = [], + ) -> list[ScannerSecret]: + """Scan for tokens in a given files content. + + You may pass a list of allowed or disallowed Services. + If both lists are empty (Default) all available services will be scanned. + """ + allowed = allowed if allowed else list(Services) + services: list[Services] = [s for s in allowed if s not in disallowed] + secrets: list[ScannerSecret] = [] + + for service in services: + scanner: type[BaseScanner] | None = cls.__SERVICE_MAPPING.get(service, None) + if not scanner: + logging.warning("The provided service %r is not a supported or a valid service.", service) + continue + + found: ScannerSecret = scanner.match(file) + if found["tokens"]: + secrets.append(found) + + return secrets diff --git a/types_/scanner.py b/types_/scanner.py new file mode 100644 index 0000000..b63fe14 --- /dev/null +++ b/types_/scanner.py @@ -0,0 +1,30 @@ +"""MystBin. Share code easily. + +Copyright (C) 2020-Current PythonistaGuild + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypedDict + + +if TYPE_CHECKING: + from core.scanners import Services + + +class ScannerSecret(TypedDict): + service: Services + tokens: set[str] From 1320ca7b47a84d0e656135ee3327cee8beda0e22 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Fri, 24 May 2024 16:51:44 +1000 Subject: [PATCH 02/12] Update schema --- migration.sql | 1 + schema.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/migration.sql b/migration.sql index dfc4bd6..899c850 100644 --- a/migration.sql +++ b/migration.sql @@ -15,6 +15,7 @@ ALTER TABLE files ALTER COLUMN filename SET NOT NULL; -- always require filenam ALTER TABLE files DROP COLUMN IF EXISTS attachment; -- we don't have these anymore ALTER TABLE files ADD COLUMN IF NOT EXISTS annotation TEXT; ALTER TABLE files RENAME COLUMN index TO file_index; -- bad column name +ALTER TABLE files ADD COLUMN IF NOT EXISTS warning_positions INTEGER[]; -- New line warning positions SAVEPOINT drops; DROP TABLE IF EXISTS bans CASCADE; -- no longer needed diff --git a/schema.sql b/schema.sql index 98bb93d..3ca0776 100644 --- a/schema.sql +++ b/schema.sql @@ -20,5 +20,6 @@ CREATE TABLE IF NOT EXISTS files ( charcount INTEGER GENERATED ALWAYS AS (LENGTH(content)) STORED, file_index SERIAL NOT NULL, annotation TEXT, + warning_positions INTEGER[], PRIMARY KEY (parent_id, file_index) ); From 129c43601ba30fd60bd12ba8b885ec2e533f7f5f Mon Sep 17 00:00:00 2001 From: MystyPy Date: Fri, 24 May 2024 16:55:13 +1000 Subject: [PATCH 03/12] Update scanners and scan positions at paste creation. --- core/database.py | 57 ++++++++++++++++++++++++++--------------------- core/models.py | 1 + core/scanners.py | 19 +++++++--------- types_/scanner.py | 2 +- 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/core/database.py b/core/database.py index 75aba77..13ba91b 100644 --- a/core/database.py +++ b/core/database.py @@ -21,7 +21,6 @@ import asyncio import datetime import logging -import re from typing import TYPE_CHECKING, Any, Self import aiohttp @@ -31,16 +30,18 @@ from . import utils from .models import FileModel, PasteModel +from .scanners import SecurityInfo, Services if TYPE_CHECKING: _Pool = asyncpg.Pool[asyncpg.Record] from types_.config import Github from types_.github import PostGist + from types_.scanner import ScannerSecret else: _Pool = asyncpg.Pool -DISCORD_TOKEN_REGEX: re.Pattern[str] = re.compile(r"[a-zA-Z0-9_-]{23,28}\.[a-zA-Z0-9_-]{6,7}\.[a-zA-Z0-9_-]{27,}") + LOGGER: logging.Logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def __init__(self, *, dsn: str, session: aiohttp.ClientSession | None = None, gi self._handling_tokens = bool(self.session and github_config) if self._handling_tokens: - LOGGER.info("Will handle compromised discord info.") + LOGGER.info("Setup to handle Discord Tokens.") assert github_config # guarded by if here self._gist_token = github_config["token"] @@ -83,11 +84,7 @@ async def _token_task(self) -> None: await asyncio.sleep(self._gist_timeout) - def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None: - formatted_bodies = "\n".join(b["content"] for b in bodies) - - tokens = list(DISCORD_TOKEN_REGEX.finditer(formatted_bodies)) - + def _handle_discord_tokens(self, tokens: list[str], paste_id: str) -> None: if not tokens: return @@ -95,8 +92,7 @@ def _handle_discord_tokens(self, *bodies: dict[str, str], paste_id: str) -> None "Discord bot token located and added to token bucket. Current bucket size is: %s", len(self.__tokens_bucket) ) - tokens = "\n".join([m[0] for m in tokens]) - self.__tokens_bucket[paste_id] = tokens + self.__tokens_bucket[paste_id] = "\n".join(tokens) async def _post_gist_of_tokens(self) -> None: assert self.session # guarded in caller @@ -211,8 +207,8 @@ async def create_paste(self, *, data: dict[str, Any]) -> PasteModel: """ file_query: str = """ - INSERT INTO files (parent_id, content, filename, loc, annotation) - VALUES ($1, $2, $3, $4, $5) + INSERT INTO files (parent_id, content, filename, loc, annotation, warning_positions) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING * """ @@ -246,28 +242,39 @@ async def create_paste(self, *, data: dict[str, Any]) -> PasteModel: name: str = (file.get("filename") or f"file_{index}")[-CONFIG["PASTES"]["name_limit"] :] name = "_".join(name.splitlines()) - content: str = file["content"] + # Normalise newlines... + content: str = file["content"].replace("\r\n", "\n").replace("\r", "\n") loc: int = file["content"].count("\n") + 1 - annotation: str = "" - tokens = [t for t in utils.TOKEN_REGEX.findall(content) if utils.validate_discord_token(t)] - if tokens: - annotation = "Contains possibly sensitive information: Discord Token(s)" - if not password: - annotation += ", which have now been invalidated." + positions: list[int] = [] + extra: str = "" + + secrets: list[ScannerSecret] = SecurityInfo.scan_file(content) + for payload in secrets: + service: Services = payload["service"] + + extra += f"{service.value}, " + positions += [t[0] for t in payload["tokens"]] + + if not password and self._handling_tokens and service is Services.discord: + self._handle_discord_tokens(tokens=[t[1] for t in payload["tokens"]], paste_id=paste.id) + + extra = extra.removesuffix(", ") + annotation = f"Contains possibly sensitive information or tokens from: {extra}" if extra else "" row: asyncpg.Record | None = await connection.fetchrow( - file_query, paste.id, content, name, loc, annotation + file_query, + paste.id, + content, + name, + loc, + annotation, + sorted(positions), ) if row: paste.files.append(FileModel(row)) - if not password: - # if the user didn't provide a password (a public paste) - # we check for discord tokens - self._handle_discord_tokens(*data["files"], paste_id=paste.id) - return paste async def fetch_paste_security(self, *, token: str) -> PasteModel | None: diff --git a/core/models.py b/core/models.py index ec21ae3..e8dab76 100644 --- a/core/models.py +++ b/core/models.py @@ -67,6 +67,7 @@ def __init__(self, record: asyncpg.Record | dict[str, Any]) -> None: self.charcount: int = record["charcount"] self.index: int = record["file_index"] self.annotation: str = record["annotation"] + self.warning_positions: list[int] = record["warning_positions"] class PasteModel(BaseModel): diff --git a/core/scanners.py b/core/scanners.py index f8bff78..9c2729d 100644 --- a/core/scanners.py +++ b/core/scanners.py @@ -41,9 +41,11 @@ class BaseScanner: @classmethod def match(cls, content: str) -> ScannerSecret: + matches: list[tuple[int, str]] = [(m.start(0), m.group(0)) for m in cls.REGEX.finditer(content)] + payload: ScannerSecret = { "service": cls.SERVICE, - "tokens": set(cls.REGEX.findall(content)), + "tokens": matches, } return payload @@ -66,9 +68,13 @@ def validate_discord_token(token: str) -> bool: @classmethod def match(cls, content: str) -> ScannerSecret: + matches: list[tuple[int, str]] = [ + (m.start(0), m.group(0)) for m in cls.REGEX.finditer(content) if cls.validate_discord_token(m.group(0)) + ] + payload: ScannerSecret = { "service": cls.SERVICE, - "tokens": {t for t in cls.REGEX.findall(content) if cls.validate_discord_token(t)}, + "tokens": matches, } return payload @@ -83,15 +89,6 @@ class GitHubScanner(BaseScanner): REGEX = re.compile(r"((ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36})") SERVICE = Services.github - @classmethod - def match(cls, content: str) -> ScannerSecret: - payload: ScannerSecret = { - "service": cls.SERVICE, - "tokens": {t[0] for t in cls.REGEX.findall(content)}, - } - - return payload - class SecurityInfo: __SERVICE_MAPPING: ClassVar[dict[Services, type[BaseScanner]]] = { diff --git a/types_/scanner.py b/types_/scanner.py index b63fe14..3b63909 100644 --- a/types_/scanner.py +++ b/types_/scanner.py @@ -27,4 +27,4 @@ class ScannerSecret(TypedDict): service: Services - tokens: set[str] + tokens: list[tuple[int, str]] From 81e071b828feac65155537429e4b4e620e53c875 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Fri, 24 May 2024 16:56:24 +1000 Subject: [PATCH 04/12] Change Line Numbers to Python side. --- views/htmx.py | 24 ++++++++++++++++++- web/password.html | 1 - web/paste.html | 1 - web/static/packages/highlight-ln.min.js | 1 - web/static/scripts/highlights.js | 5 ---- web/static/scripts/highlightsHTMX.js | 7 ------ web/static/styles/global.css | 31 +++++++++++++++++++++++++ 7 files changed, 54 insertions(+), 16 deletions(-) delete mode 100644 web/static/packages/highlight-ln.min.js diff --git a/views/htmx.py b/views/htmx.py index e752ce9..7e66b74 100644 --- a/views/htmx.py +++ b/views/htmx.py @@ -55,12 +55,34 @@ def highlight_code(self, *, files: list[dict[str, Any]]) -> str: raw_url: str = f'/raw/{file["parent_id"]}' annotation: str = file["annotation"] + positions: list[int] = file.get("warning_positions", []) content = bleach.clean( file["content"].replace("❌ {annotation}' if annotation else "" + position: int = 0 + next_pos: int | None = positions.pop(0) if positions else None + + numbers: list[str] = [] + for n, line in enumerate(content.splitlines(), 1): + length: int = len(line) + + if next_pos is not None and next_pos in range(position, position + length): + numbers.append(f"""{n}""") + + try: + next_pos = positions.pop(0) + except IndexError: + next_pos = None + + else: + numbers.append(f"""{n}""") + + position += length + 1 + + lines: str = f"""\n{"".join(numbers)}\n
""" html += f"""
@@ -72,7 +94,7 @@ def highlight_code(self, *, files: list[dict[str, Any]]) -> str:
{annotations} - 
{content}
+ 
{lines}{content}
""" return html diff --git a/web/password.html b/web/password.html index 5e217b3..0762e59 100644 --- a/web/password.html +++ b/web/password.html @@ -14,7 +14,6 @@ - diff --git a/web/paste.html b/web/paste.html index 9f43ed0..3231db5 100644 --- a/web/paste.html +++ b/web/paste.html @@ -13,7 +13,6 @@ - diff --git a/web/static/packages/highlight-ln.min.js b/web/static/packages/highlight-ln.min.js deleted file mode 100644 index a5f9f20..0000000 --- a/web/static/packages/highlight-ln.min.js +++ /dev/null @@ -1 +0,0 @@ -!function(r,o){"use strict";var e,i="hljs-ln",l="hljs-ln-line",h="hljs-ln-code",s="hljs-ln-numbers",c="hljs-ln-n",m="data-line-number",a=/\r\n|\r|\n/g;function u(e){for(var n=e.toString(),t=e.anchorNode;"TD"!==t.nodeName;)t=t.parentNode;for(var r=e.focusNode;"TD"!==r.nodeName;)r=r.parentNode;var o=parseInt(t.dataset.lineNumber),a=parseInt(r.dataset.lineNumber);if(o==a)return n;var i,l=t.textContent,s=r.textContent;for(a
{6}',[l,s,c,m,h,o+n.startFrom,0{1}',[i,r])}return e}(e.innerHTML,o)}function v(e){var n=e.className;if(/hljs-/.test(n)){for(var t=g(e.innerHTML),r=0,o="";r{1}\n',[n,0 code"); let highlighted = hljs.highlight(pasteStores[index], { language: chosen }); code.innerHTML = highlighted.value; - - // Add Line Numbers... - hljs.lineNumbersBlock(code, { singleLine: true }); - inp.placeholder = chosen; } \ No newline at end of file diff --git a/web/static/styles/global.css b/web/static/styles/global.css index 8ac7ea4..7616689 100644 --- a/web/static/styles/global.css +++ b/web/static/styles/global.css @@ -395,6 +395,7 @@ textarea { .fileContent { padding: 0.5rem; overflow-x: auto; + position: relative; } .identifierHeader { @@ -531,6 +532,36 @@ textarea { cursor: pointer; } +.lineNums { + display: block; + border-collapse: collapse; + border-spacing: 0; + border: none; + font-family: "JetBrains Mono", monospace; + font-size: 0.8em; + user-select: none; +} + +.lineNumRow { + padding: 0; + padding-right: 16px!important; + opacity: 0.7; +} + +.lineWarn { + background: var(--color-error); + position: absolute; + width: 100%; + z-index: 999; + height: 1rem; + opacity: 0.4; +} + +code { + font-family: "JetBrains Mono", monospace; + font-size: 0.8em; +} + /* Theme Switch */ .themeSwitch { --size: 1.5rem; From 7846e54314dace3d64daa7521100da08b3654093 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Fri, 24 May 2024 16:59:50 +1000 Subject: [PATCH 05/12] Invalidate caches --- web/index.html | 2 +- web/maint.html | 2 +- web/password.html | 4 ++-- web/paste.html | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/index.html b/web/index.html index 4fa4673..25f2cde 100644 --- a/web/index.html +++ b/web/index.html @@ -23,7 +23,7 @@ - + diff --git a/web/maint.html b/web/maint.html index 61261ea..945c2a6 100644 --- a/web/maint.html +++ b/web/maint.html @@ -15,7 +15,7 @@ - + diff --git a/web/password.html b/web/password.html index 0762e59..f9953fd 100644 --- a/web/password.html +++ b/web/password.html @@ -19,12 +19,12 @@ - + - + diff --git a/web/paste.html b/web/paste.html index 3231db5..29a210a 100644 --- a/web/paste.html +++ b/web/paste.html @@ -18,12 +18,12 @@ - + - + From f44da7ca2af5e4d85396a819bdef20a79d5f68e3 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Fri, 24 May 2024 17:49:24 +1000 Subject: [PATCH 06/12] Fix line warnings due to bleach. --- views/htmx.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/views/htmx.py b/views/htmx.py index 7e66b74..e42daab 100644 --- a/views/htmx.py +++ b/views/htmx.py @@ -56,20 +56,18 @@ def highlight_code(self, *, files: list[dict[str, Any]]) -> str: raw_url: str = f'/raw/{file["parent_id"]}' annotation: str = file["annotation"] positions: list[int] = file.get("warning_positions", []) + original: str = file["content"] - content = bleach.clean( - file["content"].replace("❌ {annotation}' if annotation else "" position: int = 0 next_pos: int | None = positions.pop(0) if positions else None numbers: list[str] = [] - for n, line in enumerate(content.splitlines(), 1): + for n, line in enumerate(original.splitlines(), 1): length: int = len(line) - if next_pos is not None and next_pos in range(position, position + length): + if next_pos is not None and position <= next_pos <= position + length: numbers.append(f"""{n}""") try: @@ -82,6 +80,8 @@ def highlight_code(self, *, files: list[dict[str, Any]]) -> str: position += length + 1 + content = bleach.clean(original.replace("\n{"".join(numbers)}\n""" html += f"""
From 22ea3897fe50de6fd4084a4699d7508d7dcfd85d Mon Sep 17 00:00:00 2001 From: MystyPy Date: Sat, 25 May 2024 07:21:17 +1000 Subject: [PATCH 07/12] Fix paste highlighting and enter behaviour on Password pastes. --- web/password.html | 4 ++-- web/static/scripts/highlightsHTMX.js | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/web/password.html b/web/password.html index f9953fd..2bcde32 100644 --- a/web/password.html +++ b/web/password.html @@ -19,7 +19,7 @@ - + @@ -55,7 +55,7 @@
diff --git a/web/static/scripts/highlightsHTMX.js b/web/static/scripts/highlightsHTMX.js index f1fafc2..a8e12fc 100644 --- a/web/static/scripts/highlightsHTMX.js +++ b/web/static/scripts/highlightsHTMX.js @@ -60,4 +60,6 @@ function changeLang(inp, area, index) { let code = area.querySelector("pre > code"); let highlighted = hljs.highlight(pasteStores[index], { language: chosen }); code.innerHTML = highlighted.value; + + inp.placeholder = chosen; } \ No newline at end of file From a5453917075545dc3e33a3695c34df6fe3938918 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Sat, 25 May 2024 09:00:24 +1000 Subject: [PATCH 08/12] Add annotation tooltip. --- web/static/styles/global.css | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/web/static/styles/global.css b/web/static/styles/global.css index 7616689..8c25cfd 100644 --- a/web/static/styles/global.css +++ b/web/static/styles/global.css @@ -3,6 +3,7 @@ --color-accent: #9069a7; --color-error: #dd374d; --color-security: #004ac0; + --color-warning: #004ac0; --color-background: #ffe5ee; --color-background--header: #fefefe; --color-background--pastes: #fff; @@ -51,6 +52,7 @@ --color-accent: #c89ee0; --color-error: #dd374d; --color-security: #c8e09e; + --color-warning: #feff99; --color-background: #15151c; --color-background--header: #1d1d26; --color-background--pastes: rgb(29, 29, 38, 0.9); @@ -562,6 +564,17 @@ code { font-size: 0.8em; } +.annotationSecond { + color: var(--color-warning); + opacity: 0.9; + padding-left: 0.125rem; + font-weight: 400; +} + +.annotationSecond:hover { + cursor: help; +} + /* Theme Switch */ .themeSwitch { --size: 1.5rem; @@ -604,6 +617,30 @@ code { align-self: center; } +.annotationSecond { + position:relative; +} + +.annotationSecond:after { + position:absolute; + content: attr(data-text); + color: var(--color-foreground); + background-color: var(--color-background); + top:50%; + transform:translateY(-50%); + left:100%; + width: max-content; + border-radius: 0.25rem; + margin-left: 0.5rem; + padding: 0.5rem; + display:none; /* hide by default */ + opacity: 1; +} + +.annotationSecond:hover:after { + display:block; +} + @media screen and (max-width: 600px) { .annotations { font-size: 0.8em; From 506b2d88516a147d38aea1dc9e14bf3f705e2b8a Mon Sep 17 00:00:00 2001 From: MystyPy Date: Sat, 25 May 2024 09:01:02 +1000 Subject: [PATCH 09/12] Slightly change annotations. --- core/database.py | 2 +- views/htmx.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/database.py b/core/database.py index 13ba91b..7acad6f 100644 --- a/core/database.py +++ b/core/database.py @@ -260,7 +260,7 @@ async def create_paste(self, *, data: dict[str, Any]) -> PasteModel: self._handle_discord_tokens(tokens=[t[1] for t in payload["tokens"]], paste_id=paste.id) extra = extra.removesuffix(", ") - annotation = f"Contains possibly sensitive information or tokens from: {extra}" if extra else "" + annotation = f"Contains possibly sensitive data from: {extra}" if extra else "" row: asyncpg.Record | None = await connection.fetchrow( file_query, diff --git a/views/htmx.py b/views/htmx.py index e42daab..773c761 100644 --- a/views/htmx.py +++ b/views/htmx.py @@ -58,7 +58,19 @@ def highlight_code(self, *, files: list[dict[str, Any]]) -> str: positions: list[int] = file.get("warning_positions", []) original: str = file["content"] - annotations: str = f'❌ {annotation}' if annotation else "" + parts: list[str] = annotation.split(":") + annotation = parts.pop(0) + + extra: str = ( + f"""{parts[0]}""" + if parts + else "" + ) + annotations: str = ( + f'❌ {annotation}{": " + extra if extra else ""}' + if annotation + else "" + ) position: int = 0 next_pos: int | None = positions.pop(0) if positions else None From 964b8bf751d3fa246939e514203636d12c323f18 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Sat, 25 May 2024 09:20:36 +1000 Subject: [PATCH 10/12] Fix line warnings so they don't deny text selection. --- web/static/styles/global.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/static/styles/global.css b/web/static/styles/global.css index 8c25cfd..1dc87d0 100644 --- a/web/static/styles/global.css +++ b/web/static/styles/global.css @@ -554,7 +554,7 @@ textarea { background: var(--color-error); position: absolute; width: 100%; - z-index: 999; + z-index: 1; height: 1rem; opacity: 0.4; } @@ -562,6 +562,7 @@ textarea { code { font-family: "JetBrains Mono", monospace; font-size: 0.8em; + z-index: 2; } .annotationSecond { From b826fed797f91dbd17ae1039cb990b8359dfaf72 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Tue, 28 May 2024 12:09:56 +1000 Subject: [PATCH 11/12] Only handle tokens when configured to --- core/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/database.py b/core/database.py index 7acad6f..5c31c05 100644 --- a/core/database.py +++ b/core/database.py @@ -85,7 +85,7 @@ async def _token_task(self) -> None: await asyncio.sleep(self._gist_timeout) def _handle_discord_tokens(self, tokens: list[str], paste_id: str) -> None: - if not tokens: + if not self._handling_tokens or not tokens: return LOGGER.info( From f9beddaeffcea0e24f5132b5d8fe2758d48b88a0 Mon Sep 17 00:00:00 2001 From: MystyPy Date: Tue, 28 May 2024 12:10:51 +1000 Subject: [PATCH 12/12] Fix requested changes. --- core/scanners.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/scanners.py b/core/scanners.py index 9c2729d..5d31dc8 100644 --- a/core/scanners.py +++ b/core/scanners.py @@ -16,14 +16,18 @@ along with this program. If not, see . """ +from __future__ import annotations + import base64 import binascii import enum import logging import re -from typing import ClassVar +from typing import TYPE_CHECKING, ClassVar + -from types_.scanner import ScannerSecret +if TYPE_CHECKING: + from types_.scanner import ScannerSecret logger: logging.Logger = logging.getLogger(__name__) @@ -37,7 +41,7 @@ class Services(enum.Enum): class BaseScanner: REGEX: ClassVar[re.Pattern[str]] - SERVICE: Services + SERVICE: ClassVar[Services] @classmethod def match(cls, content: str) -> ScannerSecret: @@ -103,15 +107,17 @@ def scan_file( file: str, /, *, - allowed: list[Services] = [], - disallowed: list[Services] = [], + allowed: list[Services] | None = None, + disallowed: list[Services] | None = None, ) -> list[ScannerSecret]: """Scan for tokens in a given files content. You may pass a list of allowed or disallowed Services. If both lists are empty (Default) all available services will be scanned. """ - allowed = allowed if allowed else list(Services) + disallowed = disallowed or [] + allowed = allowed or list(Services) + services: list[Services] = [s for s in allowed if s not in disallowed] secrets: list[ScannerSecret] = []