From 6bc39df5d296c1bc6ad0fd3f9e0e321bd37f5560 Mon Sep 17 00:00:00 2001 From: Yokarion Date: Wed, 5 Aug 2026 00:59:30 +0200 Subject: [PATCH] Update e621_tagger.py --- plugins/e621_tagger/e621_tagger.py | 819 ++++++++++++++++------------- 1 file changed, 459 insertions(+), 360 deletions(-) diff --git a/plugins/e621_tagger/e621_tagger.py b/plugins/e621_tagger/e621_tagger.py index 11185385..c56160eb 100644 --- a/plugins/e621_tagger/e621_tagger.py +++ b/plugins/e621_tagger/e621_tagger.py @@ -1,16 +1,34 @@ import hashlib +import json import re import sys -import json import time +from typing import List, Optional, Set + import requests -import itertools import stashapi.log as log from stashapi.stashapp import StashInterface -from typing import List, Optional, Tuple + +PLUGIN_ID = "e621_tagger" MD5_RE = re.compile(r"^[a-f0-9]{32}$") +TAG_DONE = "e621_tagged" +TAG_FAILED = "e621_tag_failed" +TAG_NOT_FOUND = "e621_not_found" + +# e621 rejects requests whose User-Agent lacks a descriptive project + contact +# component. Anything generic (curl/*, python-requests/*, bare project names, +# or an impersonated browser string) gets a 403 and an HTML "API misuse" page. +DEFAULT_USER_AGENT = "Stash-e621-Tagger/1.1 (by anonymous on e621)" +DEFAULT_DELAY_MS = 1000 +MIN_DELAY_MS = 500 # e621 hard-caps at 2 req/s + +# Custom GraphQL fragments to avoid stashapi's default fragments which reference +# fields that have been removed/renamed in newer Stash versions: +# - `Folder.basename` (removed) +# - `Image.checksum` (removed; now on file fingerprints) +# - `Scene.checksum` (deprecated/removed; now on file fingerprints) IMAGE_FRAGMENT = """ id organized @@ -59,7 +77,99 @@ """ +# --------------------------------------------------------------------------- +# e621 transport +# --------------------------------------------------------------------------- + +class E621Blocked(Exception): + """Transport-level rejection (403/429/503/HTML block page). + + This is NOT a per-item failure -- it means every subsequent request will + fail too, so the run aborts instead of poisoning the library with + e621_tag_failed tags. + """ + + +E621_SESSION = requests.Session() +_request_delay = DEFAULT_DELAY_MS / 1000.0 +_last_request_ts = 0.0 + + +def configure_e621(user_agent, delay_ms) -> None: + global _request_delay + + ua = (user_agent or "").strip() or DEFAULT_USER_AGENT + E621_SESSION.headers.update({"User-Agent": ua, "Accept": "application/json"}) + + try: + delay_ms = int(delay_ms) + except (TypeError, ValueError): + delay_ms = DEFAULT_DELAY_MS + if delay_ms < MIN_DELAY_MS: + log.warning( + f"RequestDelayMs={delay_ms} is below e621's rate cap; clamping to {MIN_DELAY_MS}ms" + ) + delay_ms = MIN_DELAY_MS + _request_delay = delay_ms / 1000.0 + + log.info(f"e621 configured: UA={ua!r}, delay={_request_delay:.2f}s") + if "(" not in ua: + log.warning( + "User-Agent has no '(by on e621)' component -- e621 will " + "likely reject every request with a 403." + ) + + +def _throttle() -> None: + global _last_request_ts + wait = _request_delay - (time.monotonic() - _last_request_ts) + if wait > 0: + time.sleep(wait) + _last_request_ts = time.monotonic() + + +def e621_lookup(md5: str) -> Optional[dict]: + """Return the post dict for an md5, or None if e621 has no such post. + + Raises E621Blocked when the request is refused at the transport level. + """ + _throttle() + try: + r = E621_SESSION.get( + "https://e621.net/posts.json", params={"md5": md5}, timeout=15 + ) + except requests.RequestException as e: + raise E621Blocked(f"network error: {e}") from e + + if r.status_code in (403, 429, 503): + raise E621Blocked(f"HTTP {r.status_code}") + if r.status_code == 404: + return None + if r.status_code >= 400: + raise E621Blocked(f"HTTP {r.status_code}") + + if "application/json" not in (r.headers.get("Content-Type") or ""): + raise E621Blocked("non-JSON response (block page)") + + try: + body = r.json() + except ValueError as e: + raise E621Blocked(f"malformed JSON: {e}") from e + + post = body.get("post") + if post: + return post + posts = body.get("posts") or [] + return posts[0] if posts else None + + +# --------------------------------------------------------------------------- +# Stash helpers +# --------------------------------------------------------------------------- + def _update_image_minimal(stash: StashInterface, payload: dict) -> dict: + """Direct GraphQL update that only returns `id` so we don't depend on + stashapi's default mutation fragments (which reference removed fields).""" query = """ mutation ImageUpdate($input: ImageUpdateInput!) { imageUpdate(input: $input) { id } @@ -77,7 +187,27 @@ def _update_scene_minimal(stash: StashInterface, payload: dict) -> dict: return stash.call_GQL(query, {"input": payload}) +def _get_item(stash: StashInterface, item_type: str, item_id: str) -> Optional[dict]: + if item_type == "image": + return stash.find_image(item_id, fragment=IMAGE_FRAGMENT) + return stash.find_scene(item_id, fragment=SCENE_FRAGMENT) + + +def _update_item(stash: StashInterface, item_type: str, payload: dict) -> dict: + if item_type == "image": + return _update_image_minimal(stash, payload) + return _update_scene_minimal(stash, payload) + + +def _item_files(item_type: str, obj: dict) -> list: + if item_type == "image": + return obj.get("visual_files") or [] + return obj.get("files") or obj.get("scene_files") or [] + + def _extract_file_md5(file_data: dict) -> Optional[str]: + """Pull a valid MD5 string from a file dict via the modern `fingerprints` + array. Falls back to a legacy `checksum` field if it happens to exist.""" if not file_data: return None for fp in file_data.get("fingerprints") or []: @@ -91,199 +221,71 @@ def _extract_file_md5(file_data: dict) -> Optional[str]: return None -def _build_filter(skip_tag_ids, exclude_organized): - f = {} - if skip_tag_ids: - f["tags"] = { - "value": [], - "excludes": skip_tag_ids, - "modifier": "INCLUDES_ALL", - "depth": -1, - } - if exclude_organized: - f["organized"] = False - return f - - -def count_images( - client: StashInterface, skip_tag_ids: list, exclude_organized: bool -) -> int: - image_filter = _build_filter(skip_tag_ids, exclude_organized) - pagination = {"page": 1, "per_page": 0, "sort": "created_at", "direction": "ASC"} - total, _ = client.find_images( - f=image_filter, - filter=pagination, - get_count=True, - fragment=IMAGE_FRAGMENT, - ) - return total - - -def count_scenes( - client: StashInterface, skip_tag_ids: list, exclude_organized: bool -) -> int: - scene_filter = _build_filter(skip_tag_ids, exclude_organized) - pagination = {"page": 1, "per_page": 0, "sort": "created_at", "direction": "ASC"} - total, _ = client.find_scenes( - f=scene_filter, - filter=pagination, - get_count=True, - fragment=SCENE_FRAGMENT, - ) - return total - - -def stream_images( - client: StashInterface, - skip_tag_ids: List[int], - exclude_organized: bool, - per_page: int = 100, -): - page = 1 - base_filter = _build_filter(skip_tag_ids, exclude_organized) - while True: - pagination = { - "page": page, - "per_page": per_page, - "sort": "created_at", - "direction": "ASC", - } - images = client.find_images( - f=base_filter, filter=pagination, fragment=IMAGE_FRAGMENT - ) - if not images: - break - log.info(f"Fetched image page {page} with {len(images)} images") - for img in images: - yield ("image", img) - page += 1 +def _md5_candidates(item_type: str, obj: dict, item_id: str) -> List[str]: + """Ordered, de-duplicated MD5s to try against e621. + Fingerprint first (it is the actual content hash Stash computed), then the + filename stem -- files downloaded from e621 are named by md5, and that name + survives re-encoding when the content hash would not. + """ + files = _item_files(item_type, obj) + if not files: + log.error(f"No files found for {item_type} {item_id}; cannot compute md5") + return [] -def stream_scenes( - client: StashInterface, - skip_tag_ids: List[int], - exclude_organized: bool, - per_page: int = 100, -): - page = 1 - base_filter = _build_filter(skip_tag_ids, exclude_organized) - while True: - pagination = { - "page": page, - "per_page": per_page, - "sort": "created_at", - "direction": "ASC", - } - scenes = client.find_scenes( - f=base_filter, filter=pagination, fragment=SCENE_FRAGMENT - ) - if not scenes: - break - log.info(f"Fetched scene page {page} with {len(scenes)} scenes") - for sc in scenes: - yield ("scene", sc) - page += 1 + file_data = files[0] + candidates: List[str] = [] + fp_md5 = _extract_file_md5(file_data) + if fp_md5: + candidates.append(fp_md5) -def process_e621_post_for_item( - stash: StashInterface, item_type: str, item_id: str, item_md5: str -) -> bool: - if item_type == "image": - obj = stash.find_image(item_id, fragment=IMAGE_FRAGMENT) - already_tagged = any(t["name"] == "e621_tagged" for t in obj.get("tags", [])) - already_failed = any( - t["name"] == "e621_tag_failed" for t in obj.get("tags", []) - ) - else: - obj = stash.find_scene(item_id, fragment=SCENE_FRAGMENT) - already_tagged = any(t["name"] == "e621_tagged" for t in obj.get("tags", [])) - already_failed = any( - t["name"] == "e621_tag_failed" for t in obj.get("tags", []) - ) + basename = file_data.get("basename") or "" + name_md5 = basename.split(".")[0].lower() if basename else "" + if MD5_RE.match(name_md5) and name_md5 not in candidates: + candidates.append(name_md5) - if already_tagged or already_failed: - return False + if candidates: + return candidates + path = file_data.get("path") + if not path: + log.error(f"No path for {item_type} {item_id}; cannot compute md5") + return [] try: - time.sleep(0.5) - response = requests.get( - f"https://e621.net/posts.json?md5={item_md5}", - headers={"User-Agent": "Stash-e621-Tagger/1.0"}, - timeout=10, - ) - response.raise_for_status() - post_data = response.json().get("post", {}) + md5_hash = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + md5_hash.update(chunk) + computed = md5_hash.hexdigest() + log.info(f"Generated content MD5 for {item_type} {item_id}: {computed}") + return [computed] except Exception as e: - log.error(f"Marking as failed. e621 API error: {str(e)}") - e621_tag_failed = get_or_create_tag(stash, "e621_tag_failed") - fail_ids = [e621_tag_failed["id"]] + [t["id"] for t in obj.get("tags", [])] - try: - payload = {"id": item_id, "tag_ids": list(set(fail_ids))} - if item_type == "image": - _update_image_minimal(stash, payload) - else: - _update_scene_minimal(stash, payload) - return True - except Exception as e2: - log.error(f"Failed to mark as failed: {str(e2)}") - return False - - if not post_data: - return False + log.error(f"Failed to generate MD5 for {item_type} {item_id}: {e}") + return [] - e621_tag = get_or_create_tag(stash, "e621_tagged") - post_url = f"https://e621.net/posts/{post_data['id']}" - tag_ids = [e621_tag["id"]] - for cat in ["general", "species", "artist", "copyright", "meta"]: - for tag in post_data.get("tags", {}).get(cat, []): - clean_tag = tag.strip() - if not clean_tag: - continue - stash_tag = get_or_create_tag(stash, clean_tag) - if stash_tag: - tag_ids.append(stash_tag["id"]) - - studio_id = None - if artists := post_data.get("tags", {}).get("artist"): - studio = get_or_create_studio(stash, artists[0]) - studio_id = studio["id"] - - performer_ids = [] - for char in post_data.get("tags", {}).get("character", []): - name = char - perf = get_or_create_performer(stash, name) - performer_ids.append(perf["id"]) +# --------------------------------------------------------------------------- +# Tag / studio / performer resolution (cached) +# --------------------------------------------------------------------------- - try: - update_payload = { - "id": item_id, - "organized": True, - "urls": [post_url], - "tag_ids": list(set(tag_ids)), - "studio_id": studio_id, - "performer_ids": performer_ids, - } - if item_type == "image": - _update_image_minimal(stash, update_payload) - log.debug(f"Image updated: {item_id}") - else: - _update_scene_minimal(stash, update_payload) - log.info(f"Scene updated: {item_id}") - return True - except Exception as e: - log.error(f"Update failed: {str(e)}") - return False +_TAG_CACHE = {} +_STUDIO_CACHE = {} +_PERFORMER_CACHE = {} -def get_or_create_tag(stash: StashInterface, tag_name: str) -> dict: - tag_name = tag_name.strip() +def get_or_create_tag(stash: StashInterface, tag_name: str) -> Optional[dict]: + tag_name = (tag_name or "").strip() if not tag_name: log.error("Attempted to create tag with empty name") return None + if tag_name in _TAG_CACHE: + return _TAG_CACHE[tag_name] + existing = stash.find_tags(f={"name": {"value": tag_name, "modifier": "EQUALS"}}) if existing: + _TAG_CACHE[tag_name] = existing[0] return existing[0] parts = tag_name.split(":") @@ -293,6 +295,10 @@ def get_or_create_tag(stash: StashInterface, tag_name: str) -> dict: if not current_name: continue + if current_name in _TAG_CACHE: + parent_id = _TAG_CACHE[current_name]["id"] + continue + existing = stash.find_tags( f={"name": {"value": current_name, "modifier": "EQUALS"}} ) @@ -307,236 +313,329 @@ def get_or_create_tag(stash: StashInterface, tag_name: str) -> dict: return None parent_id = new_tag["id"] except Exception as e: - log.error(f"Error creating tag {current_name}: {str(e)}") + log.error(f"Error creating tag {current_name}: {e}") return None else: parent_id = existing[0]["id"] - return {"id": parent_id} + _TAG_CACHE[current_name] = {"id": parent_id} + result = {"id": parent_id} + _TAG_CACHE[tag_name] = result + return result -def get_or_create_studio(stash: StashInterface, name: str) -> dict: + +def get_or_create_studio(stash: StashInterface, name: str) -> Optional[dict]: + name = (name or "").strip() + if not name: + return None + if name in _STUDIO_CACHE: + return _STUDIO_CACHE[name] studios = stash.find_studios(f={"name": {"value": name, "modifier": "EQUALS"}}) - return studios[0] if studios else stash.create_studio({"name": name}) + studio = studios[0] if studios else stash.create_studio({"name": name}) + _STUDIO_CACHE[name] = studio + return studio -def get_or_create_performer(stash: StashInterface, name: str) -> dict: +def get_or_create_performer(stash: StashInterface, name: str) -> Optional[dict]: + name = (name or "").strip() + if not name: + return None + if name in _PERFORMER_CACHE: + return _PERFORMER_CACHE[name] performers = stash.find_performers( f={"name": {"value": name, "modifier": "EQUALS"}} ) - return performers[0] if performers else stash.create_performer({"name": name}) + performer = performers[0] if performers else stash.create_performer({"name": name}) + _PERFORMER_CACHE[name] = performer + return performer -def scrape_image(client: StashInterface, image_id: str) -> bool: - image = client.find_image(image_id, fragment=IMAGE_FRAGMENT) - if not image or not image.get("visual_files"): +# --------------------------------------------------------------------------- +# Item processing +# --------------------------------------------------------------------------- + +def _mark_with_tag( + stash: StashInterface, item_type: str, item_id: str, obj: dict, tag_name: str +) -> bool: + """Add a bookkeeping tag while preserving the item's existing tags.""" + marker = get_or_create_tag(stash, tag_name) + if not marker: + return False + tag_ids = {marker["id"]} | {t["id"] for t in obj.get("tags") or []} + try: + _update_item(stash, item_type, {"id": item_id, "tag_ids": list(tag_ids)}) + log.info(f"Marked {item_type} {item_id} as {tag_name}") + return True + except Exception as e: + log.error(f"Failed to mark {item_type} {item_id} as {tag_name}: {e}") return False - file_data = image["visual_files"][0] - filename = file_data.get("basename", "") - filename_md5 = filename.split(".")[0] if filename else "" - if MD5_RE.match(filename_md5): - final_md5 = filename_md5 - log.debug(f"Using filename MD5 for image: {final_md5}") - else: - file_md5 = _extract_file_md5(file_data) - if file_md5: - final_md5 = file_md5 - log.debug(f"Using file fingerprint MD5 for image: {final_md5}") - else: - try: - md5_hash = hashlib.md5() - with open(file_data["path"], "rb") as f: - for chunk in iter(lambda: f.read(65536), b""): - md5_hash.update(chunk) - final_md5 = md5_hash.hexdigest() - log.info(f"Generated content MD5 for image: {final_md5}") - except Exception as e: - log.error(f"Failed to generate MD5 for image: {str(e)}") - return False +def _apply_post( + stash: StashInterface, item_type: str, item_id: str, obj: dict, post: dict +) -> bool: + post_tags = post.get("tags") or {} + post_url = f"https://e621.net/posts/{post['id']}" + + done_tag = get_or_create_tag(stash, TAG_DONE) + tag_ids = {done_tag["id"]} if done_tag else set() - return process_e621_post_for_item(client, "image", image_id, final_md5) + for cat in ("general", "species", "artist", "copyright", "meta"): + for tag in post_tags.get(cat) or []: + clean_tag = (tag or "").strip() + if not clean_tag: + continue + stash_tag = get_or_create_tag(stash, clean_tag) + if stash_tag and stash_tag.get("id"): + tag_ids.add(stash_tag["id"]) + + studio_id = None + artists = post_tags.get("artist") or [] + if artists: + studio = get_or_create_studio(stash, artists[0]) + if studio: + studio_id = studio["id"] + performer_ids = [] + for char in post_tags.get("character") or []: + perf = get_or_create_performer(stash, char) + if perf: + performer_ids.append(perf["id"]) + + payload = { + "id": item_id, + "organized": True, + "urls": [post_url], + "tag_ids": list(tag_ids), + "performer_ids": performer_ids, + } + if studio_id: + payload["studio_id"] = studio_id -def scrape_scene(client: StashInterface, scene_id: str) -> bool: - scene = client.find_scene(scene_id, fragment=SCENE_FRAGMENT) - if not scene: + try: + _update_item(stash, item_type, payload) + log.info(f"Tagged {item_type} {item_id} from {post_url}") + return True + except Exception as e: + log.error(f"Update failed for {item_type} {item_id}: {e}") return False - final_md5 = None - files = scene.get("files") or scene.get("scene_files") or [] - if not files: - log.error(f"No files found for scene {scene_id}; cannot compute md5") +def process_item( + stash: StashInterface, item_type: str, item_id: str, skip_tag_ids: Set[str] +) -> bool: + """Return True if the item was updated or marked, False if left untouched. + + Propagates E621Blocked so the caller can abort the whole run. + """ + obj = _get_item(stash, item_type, item_id) + if not obj: return False - file_data = files[0] - file_md5 = _extract_file_md5(file_data) - if file_md5: - final_md5 = file_md5 - log.info(f"Using file fingerprint MD5 for scene: {final_md5}") - else: - basename = file_data.get("basename", "") - filename_md5 = basename.split(".")[0] if basename else "" - if MD5_RE.match(filename_md5): - final_md5 = filename_md5 - log.info(f"Using filename MD5 for scene: {final_md5}") + current_tag_ids = {t["id"] for t in obj.get("tags") or []} + if current_tag_ids & skip_tag_ids: + log.debug(f"Skipping {item_type} {item_id} - already has a skip tag") + return False + + candidates = _md5_candidates(item_type, obj, item_id) + if not candidates: + return _mark_with_tag(stash, item_type, item_id, obj, TAG_FAILED) + + post = None + for md5 in candidates: + post = e621_lookup(md5) + if post: + break + + if not post: + # Marked so it is not re-queried on every subsequent pass. Remove the + # e621_not_found tag in the UI to retry these later. + return _mark_with_tag(stash, item_type, item_id, obj, TAG_NOT_FOUND) + + return _apply_post(stash, item_type, item_id, obj, post) + + +# --------------------------------------------------------------------------- +# Scanning +# --------------------------------------------------------------------------- + +def _build_filter(skip_tag_ids, exclude_organized): + f = {} + if skip_tag_ids: + f["tags"] = { + "value": [], + "excludes": list(skip_tag_ids), + "modifier": "INCLUDES_ALL", + "depth": -1, + } + if exclude_organized: + f["organized"] = False + return f + + +def count_items( + client: StashInterface, item_type: str, skip_tag_ids, exclude_organized: bool +) -> int: + item_filter = _build_filter(skip_tag_ids, exclude_organized) + pagination = {"page": 1, "per_page": 0, "sort": "created_at", "direction": "ASC"} + try: + if item_type == "image": + total, _ = client.find_images( + f=item_filter, filter=pagination, get_count=True, fragment=IMAGE_FRAGMENT + ) else: - try: - md5_hash = hashlib.md5() - with open(file_data["path"], "rb") as f: - for chunk in iter(lambda: f.read(65536), b""): - md5_hash.update(chunk) - final_md5 = md5_hash.hexdigest() - log.info(f"Generated content MD5 for scene: {final_md5}") - except Exception as e: - log.error(f"Failed to generate MD5 for scene: {str(e)}") - return False + total, _ = client.find_scenes( + f=item_filter, filter=pagination, get_count=True, fragment=SCENE_FRAGMENT + ) + return total + except Exception as e: + log.error(f"Failed to count {item_type}s: {e}") + return 0 + - return process_e621_post_for_item(client, "scene", scene_id, final_md5) +def find_page( + client: StashInterface, + item_type: str, + skip_tag_ids, + exclude_organized: bool, + page: int, + per_page: int, +) -> list: + pagination = { + "page": page, + "per_page": per_page, + "sort": "created_at", + "direction": "ASC", + } + item_filter = _build_filter(skip_tag_ids, exclude_organized) + if item_type == "image": + return client.find_images( + f=item_filter, filter=pagination, fragment=IMAGE_FRAGMENT + ) + return client.find_scenes(f=item_filter, filter=pagination, fragment=SCENE_FRAGMENT) -if __name__ == "__main__": - log.info("Starting tagger with scanning passes until no work left...") +def _load_settings(stash: StashInterface) -> dict: + defaults = { + "SkipTags": f"{TAG_DONE}, {TAG_FAILED}, {TAG_NOT_FOUND}", + "ExcludeOrganized": False, + "UserAgent": DEFAULT_USER_AGENT, + "RequestDelayMs": DEFAULT_DELAY_MS, + } + try: + raw = (stash.get_configuration().get("plugins") or {}).get(PLUGIN_ID) or {} + except Exception as e: + log.error(f"Could not read plugin configuration, using defaults: {e}") + return defaults + + merged = dict(defaults) + for key, value in raw.items(): + if key not in defaults or value is None: + continue + if isinstance(value, str) and not value.strip(): + continue + merged[key] = value + return merged + + +def main() -> None: + log.info("Starting e621 tagger...") json_input = json.loads(sys.stdin.read()) stash = StashInterface(json_input["server_connection"]) - config = stash.get_configuration().get("plugins", {}) - settings = {"SkipTags": "e621_tagged, e621_tag_failed", "ExcludeOrganized": False} - settings.update(config.get("e621_tagger", {})) + settings = _load_settings(stash) + configure_e621(settings["UserAgent"], settings["RequestDelayMs"]) + exclude_organized = bool(settings["ExcludeOrganized"]) - e621_tagged = get_or_create_tag(stash, "e621_tagged") - e621_failed = get_or_create_tag(stash, "e621_tag_failed") + skip_tag_ids: Set[str] = set() + for name in {TAG_DONE, TAG_FAILED, TAG_NOT_FOUND}: + tag = get_or_create_tag(stash, name) + if tag and tag.get("id"): + skip_tag_ids.add(tag["id"]) - skip_tag_names = [n.strip() for n in settings["SkipTags"].split(",") if n.strip()] - skip_tag_ids: List[int] = [] - for name in skip_tag_names: + for name in [n.strip() for n in str(settings["SkipTags"]).split(",") if n.strip()]: found = stash.find_tags(f={"name": {"value": name, "modifier": "EQUALS"}}) if found: - skip_tag_ids.append(found[0]["id"]) - skip_tag_ids.extend([e621_tagged["id"], e621_failed["id"]]) + skip_tag_ids.add(found[0]["id"]) + else: + log.warning(f"SkipTags entry {name!r} does not exist in Stash; ignoring") per_page = 50 - log.info("Counting images...") - num_images = count_images(stash, skip_tag_ids, settings["ExcludeOrganized"]) - log.info("Counting scenes...") - num_scenes = count_scenes(stash, skip_tag_ids, settings["ExcludeOrganized"]) - + log.info("Counting items...") + num_images = count_items(stash, "image", skip_tag_ids, exclude_organized) + num_scenes = count_items(stash, "scene", skip_tag_ids, exclude_organized) total = (num_images + num_scenes) or 1 - - log.info(f"Total items (images + scenes): {total}") + log.info(f"Items to process: {num_images} images + {num_scenes} scenes = {total}") processed_count = 0 pass_num = 0 - - while True: + blocked = False + + # Each pass re-queries from page 1 because tagging an item removes it from + # the filtered result set, which shifts pagination underneath us. + while not blocked: pass_num += 1 + pass_updated = 0 log.info(f"Starting scanning pass #{pass_num}") - pass_processed = 0 - - - page = 1 - while True: - pagination = { - "page": page, - "per_page": per_page, - "sort": "created_at", - "direction": "ASC", - } - images = stash.find_images( - f=_build_filter(skip_tag_ids, settings["ExcludeOrganized"]), - filter=pagination, - fragment=IMAGE_FRAGMENT, - ) - log.info(f"[pass {pass_num}] fetched image page {page}, count={len(images)}") - if not images: - break - for img in images: - item_id = img.get("id") - if not item_id: - log.error(f"[pass {pass_num}] image without id on page {page}") - continue - - current = stash.find_image(item_id, fragment=IMAGE_FRAGMENT) - current_tag_ids = [t["id"] for t in current.get("tags", [])] - if any(tid in current_tag_ids for tid in skip_tag_ids): - log.info(f"[pass {pass_num}] skipping image {item_id} - now has skip tag") - processed_count += 1 - pass_processed += 1 - log.progress(float(processed_count) / float(total)) - continue + for item_type in ("image", "scene"): + page = 1 + while not blocked: try: - updated = scrape_image(stash, item_id) + items = find_page( + stash, item_type, skip_tag_ids, exclude_organized, page, per_page + ) except Exception as e: - log.error(f"[pass {pass_num}] scrape_image exception for {item_id}: {str(e)}") - updated = False - - if updated: - processed_count += 1 - pass_processed += 1 - log.debug(f"[pass {pass_num}] processed image {item_id} (processed_count={processed_count})") - log.progress(float(processed_count) / float(total)) - - if len(images) < per_page: - break - page += 1 - - - page = 1 - while True: - pagination = { - "page": page, - "per_page": per_page, - "sort": "created_at", - "direction": "ASC", - } - scenes = stash.find_scenes( - f=_build_filter(skip_tag_ids, settings["ExcludeOrganized"]), - filter=pagination, - fragment=SCENE_FRAGMENT, - ) - log.info(f"[pass {pass_num}] fetched scene page {page}, count={len(scenes)}") - if not scenes: - break - for sc in scenes: - item_id = sc.get("id") - if not item_id: - log.error(f"[pass {pass_num}] scene without id on page {page}") - continue - - current = stash.find_scene(item_id, fragment=SCENE_FRAGMENT) - current_tag_ids = [t["id"] for t in current.get("tags", [])] - if any(tid in current_tag_ids for tid in skip_tag_ids): - log.info(f"[pass {pass_num}] skipping scene {item_id} - now has skip tag") - processed_count += 1 - pass_processed += 1 - log.progress(float(processed_count) / float(total)) - continue - - try: - updated = scrape_scene(stash, item_id) - except Exception as e: - log.error(f"[pass {pass_num}] scrape_scene exception for {item_id}: {str(e)}") - updated = False - - if updated: - processed_count += 1 - pass_processed += 1 - log.info(f"[pass {pass_num}] processed scene {item_id} (processed_count={processed_count})") - log.progress(float(processed_count) / float(total)) - - if len(scenes) < per_page: - break - page += 1 - - log.info(f"Pass #{pass_num} finished. items processed this pass: {pass_processed}") - - if pass_processed == 0: - log.info("No items processed in last pass; finishing scan.") + log.error(f"[pass {pass_num}] failed to fetch {item_type} page {page}: {e}") + break + + if not items: + break + log.info( + f"[pass {pass_num}] fetched {item_type} page {page}, count={len(items)}" + ) + + for item in items: + item_id = item.get("id") + if not item_id: + log.error(f"[pass {pass_num}] {item_type} without id on page {page}") + continue + + try: + updated = process_item(stash, item_type, item_id, skip_tag_ids) + except E621Blocked as e: + log.error( + f"e621 refused the request ({e}). Aborting -- no items " + f"marked as failed. Check the UserAgent setting; it must " + f"look like 'Project/1.0 (by on e621)'." + ) + blocked = True + break + except Exception as e: + log.error(f"[pass {pass_num}] error on {item_type} {item_id}: {e}") + updated = False + + if updated: + processed_count += 1 + pass_updated += 1 + log.progress(min(float(processed_count) / float(total), 1.0)) + + if len(items) < per_page: + break + page += 1 + + log.info(f"Pass #{pass_num} finished. Items updated this pass: {pass_updated}") + if pass_updated == 0: + log.info("No items updated in last pass; finishing scan.") break - time.sleep(0.2) - + if blocked: + log.error(f"Run aborted after {processed_count} items.") + else: + log.info(f"Done. {processed_count} items processed.") log.progress(1.0) + + +if __name__ == "__main__": + main()