Skip to content

Commit

Permalink
Update neo-scraper to 0.9.0 + post.uploadMode changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@neobooru
neobooru committed Jun 25, 2024
1 parent 9843f0b commit eae1b39
Show file tree
Hide file tree
Showing 6 changed files with 107 additions and 69 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ jobs:
runs-on: ubuntu-20.04
# Doesn't really need the Chrome build, but we can reuse its cache.
needs: build_chrome
# Only run on master branch AND if tagging or manually dispatching.
# Only run (on master branch && if tagged) or (manually dispatching).
if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_amo_listed == true
steps:
- name: Checkout
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"deepmerge": "^4.3.1",
"lodash": "^4.17.21",
"markdown-it": "^14.0.0",
"neo-scraper": "^0.8.0",
"neo-scraper": "^0.9.0",
"normalize.css": "^8.0.1",
"pinia": "^2.1.7",
"primeflex": "^3.3.1",
Expand Down
10 changes: 5 additions & 5 deletions pnpm-lock.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 12 additions & 11 deletions src/models/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ export class TagDetails {
public names: string[],
public category?: string,
public usages?: number,
) {}
) { }

get name() {
return this.names[0];
Expand Down Expand Up @@ -38,7 +38,7 @@ export class PoolDetails {
public names: string[],
public category?: string,
public postCount?: number,
) {}
) { }

get name() {
return this.names[0];
Expand Down Expand Up @@ -73,11 +73,11 @@ export class ScrapedPostDetails {
contentType: ContentType;
contentSubType: string | undefined;
rating: BooruTypes.SafetyRating;
source = "";
source;
uploadMode: UploadMode;
referrer?: string;
resolution?: [number, number];
instanceSpecificData: MappedInstanceSpecificData = {};
uploadMode: UploadMode = "url";

constructor(post: ScrapedPost) {
this.contentUrl = post.contentUrl;
Expand All @@ -91,6 +91,7 @@ export class ScrapedPostDetails {
this.tags = post.tags.map((x) => TagDetails.fromScapedTag(x));
this.notes = post.notes;
this.resolution = post.resolution;
this.uploadMode = post.uploadMode;
}
}

Expand All @@ -108,7 +109,7 @@ export class SimilarPostInfo {
constructor(
public readonly id: number,
public readonly percentage: number,
) {}
) { }
}

export type PostUploadState = "uploading" | "uploaded" | "error";
Expand Down Expand Up @@ -147,38 +148,38 @@ export class PostUploadCommandData {
constructor(
public readonly post: ScrapedPostDetails,
public readonly selectedSite: SzuruSiteConfig,
) {}
) { }
}

export class SetPostUploadInfoData {
constructor(
public instanceId: string,
public postId: string,
public info: PostUploadInfo,
) {}
) { }
}

export class SetExactPostId {
constructor(
public readonly instanceId: string,
public readonly postId: string,
public readonly exactPostId: number,
) {}
) { }
}

export class PostUpdateCommandData {
constructor(
public readonly postId: number,
public readonly updateRequest: UpdatePostRequest,
public readonly selectedSite: SzuruSiteConfig,
) {}
) { }
}

export class FetchCommandData {
constructor(
public readonly url: string,
public readonly options: RequestInit | undefined = undefined,
) {}
) { }
}

export class SzuruSiteConfig {
Expand All @@ -192,7 +193,7 @@ export class TagCategoryColor {
constructor(
public name: string,
public color: string,
) {}
) { }
}

export const getDefaultTagCategories = () => [
Expand Down
37 changes: 28 additions & 9 deletions src/options/App.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import Column from "primevue/column";
import { SzuruSiteConfig, TagCategoryColor, getDefaultTagCategories } from "~/models";
import SzurubooruApi from "~/api";
type StatusType = "success" | "error" | "quiet";
const statusText = ref("");
const statusType = ref("status-quiet");
const versionInfo = "Version: " + (import.meta.env.VITE_SZ_VERSION ?? browser.runtime.getManifest().version);
Expand All @@ -26,9 +28,7 @@ const selectedSite = computed(() => {
}
});
type StatusType = "success" | "error" | "quiet";
let mode = useColorMode({ emitAuto: true });
const mode = useColorMode({ emitAuto: true });
async function testConnection() {
if (
Expand Down Expand Up @@ -221,8 +221,13 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<template v-if="selectedSite">
<div class="field col-12">
<label>URL</label>
<input v-if="selectedSite" text="Szurubooru URL" type="text" name="domain"
v-model="selectedSite.domain" />
<input
v-if="selectedSite"
text="Szurubooru URL"
type="text"
name="domain"
v-model="selectedSite.domain"
/>
</div>

<div class="field col-12 md:col-6">
Expand Down Expand Up @@ -251,10 +256,14 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<template v-if="field == 'color'">
<div class="color-preview">
<input type="text" class="color" v-model="data[field]" />
<div class="preview background-preview"
:style="{ 'border-color': data[field], 'background-color': data[field] }"></div>
<div class="preview text-preview" :style="{ 'border-color': data[field], color: data[field] }">
</div>
<div
class="preview background-preview"
:style="{ 'border-color': data[field], 'background-color': data[field] }"
></div>
<div
class="preview text-preview"
:style="{ 'border-color': data[field], color: data[field] }"
></div>
</div>
</template>

Expand Down Expand Up @@ -285,6 +294,16 @@ wnd.szc_set_config_version = (v = 0) => (cfg.value.version = v);
<!-- TODO: Category ignore list -->
</TabPanel>

<!-- <TabPanel header="About">
<div class="grid">
<div v-for="engine in scraper.engines" :key="engine.name" class="col-12 md:col-4 flex flex-column">
<span class="font-bold">{{ engine.name }}</span>
<span>Features: {{ engine.features.join(", ") }}</span>
<span>Hosts: {{ engine.supportedHosts.join(", ") }}</span>
</div>
</div>
</TabPanel> -->

<!--
<TabPanel header="Developer">
<div class="flex flex-column gap-2">
Expand Down
102 changes: 60 additions & 42 deletions src/popup/pages/PopupMain.vue
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<script setup lang="ts">
import { useDark } from "@vueuse/core";
import { cloneDeep } from "lodash";
import { NeoScraper, ScrapeResults } from "neo-scraper";
import { ScrapeResults } from "neo-scraper";
import { getUrl, encodeTagName, getErrorMessage, getPostInfoSummary } from "~/utils";
import {
BrowserCommand,
Expand All @@ -14,7 +14,6 @@ import {
SzuruSiteConfig,
PoolDetails,
} from "~/models";
import { ImageSearchResult } from "~/api/models";
import { isMobile } from "~/env";
import { DeepReadonly } from "vue";
import { cfg, usePopupStore } from "~/stores";
Expand Down Expand Up @@ -99,14 +98,11 @@ async function grabPost() {
// Clear current posts array
pop.posts.splice(0);
const scraper = new NeoScraper();
for (const result of res.results) {
const engine = scraper.engines.find((x) => x.name == result.engine);
for (const i in result.posts) {
const vm = new ScrapedPostDetails(result.posts[i]);
vm.name = `[${result.engine}] Post ${parseInt(i) + 1}`; // parseInt() is required!
const name = result.posts[i].name ?? `Post ${parseInt(i) + 1}`; // parseInt() is required!
vm.name = `[${result.engine}] ${name}`;
if (!cfg.value.addAllParsedTags) {
vm.tags.splice(0);
Expand All @@ -125,10 +121,6 @@ async function grabPost() {
vm.instanceSpecificData[site.id] = {};
}
if (engine) {
vm.uploadMode = engine.uploadMode;
}
pop.posts.push(vm);
}
}
Expand All @@ -145,7 +137,7 @@ async function grabPost() {
}
if (cfg.value.fetchPostInfo) {
await fetchPostInfo();
await fetchPostsInfo();
}
// Delayed call to findSimilar, as fetchPostInfo might change the post's contentUrl.
Expand All @@ -168,7 +160,7 @@ async function upload() {
}
try {
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost);
const post: ScrapedPostDetails = cloneDeep(pop.selectedPost)!;
// uploadMode "content" requires a content token to work. So ensure it is set.
if (post.uploadMode == "content") {
Expand Down Expand Up @@ -291,10 +283,10 @@ async function findSimilar(post: ScrapedPostDetails | undefined) {
isSearchingForSimilarPosts.value++;
await ensurePostHasContentToken(post);
try {
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken);
await ensurePostHasContentToken(post);
const res = await selectedInstance.reverseSearchToken(instanceSpecificData.contentToken!);
instanceSpecificData.reverseSearchResult = {
exactPostId: res.exactPost?.id,
Expand Down Expand Up @@ -331,33 +323,58 @@ async function loadTagCounts() {
}
}
async function fetchPostInfo() {
async function updatePostWithRemoteInfo(post: ScrapedPostDetails, contentUrl: string) {
// We are missing cookies/etc which means that the request might fail.
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
try {
// This request follows redirects.
const res = await fetch(contentUrl, { method: "HEAD" });
const size = res.headers.get("Content-Length");
const type = res.headers.get("Content-Type");
if (type) {
if (type.indexOf("text/html") != -1) {
// If we get a HTML page back it usually means that the request failed.
// Not all sites (e.g. e-hentai) reply with a 403.
throw new Error(
"Received a text/html content type. This probably means that we don't have permission to access the resource.",
);
}
const [_main, sub] = type.split("/");
if (sub) post.contentSubType = sub.toUpperCase();
}
// This should be after the type check, because the type check also checks whether the response we get is valid/unsable.
if (size) post.contentSize = parseInt(size);
// Update url if it changes. The url can change when:
// a. the extraContentUrl was successfully loaded
// b. because of redirects
if (res.url != post.contentUrl) {
// The developer needs to make sure that the remote server can load this URL if `post.uploadMode == 'url'`!
console.log(`Updating post.contentUrl to '${res.url}'`);
post.contentUrl = res.url;
}
return true;
} catch (ex) {
console.error(ex);
return false;
}
}
async function fetchPostsInfo() {
for (const post of pop.posts) {
if (!post.contentSize || post.extraContentUrl) {
// We are missing cookies/etc which means that the request might fail.
// If you want access to the cookies/session/etc then you need to execute this code inside the content script.
try {
const contentUrl = post.extraContentUrl ?? post.contentUrl;
// TODO: Check whether we need to call this in the background page, or if it is fine to call it from the popup.
const res = await fetch(contentUrl, { method: "HEAD" });
const size = res.headers.get("Content-Length");
const type = res.headers.get("Content-Type");
if (size) post.contentSize = parseInt(size);
if (type) {
const [_main, sub] = type.split("/");
if (sub) post.contentSubType = sub.toUpperCase();
}
// Resolve extraContentUrl redirects.
if (post.extraContentUrl && res.url != post.contentUrl) {
post.contentUrl = res.url;
}
} catch (ex) {
// TODO: Maybe display an error in the UI.
console.error(ex);
let ok = false;
if (post.extraContentUrl) {
// Prioritize info from extraContentUrl.
ok = await updatePostWithRemoteInfo(post, post.extraContentUrl);
}
if (!ok) {
// Fall back to normal contentUrl if we can't get the data from the extraContentUrl.
await updatePostWithRemoteInfo(post, post.contentUrl);
}
}
}
Expand All @@ -369,7 +386,8 @@ function getUpdatedTagsText(count: number) {
}
function onResolutionLoaded(res: any) {
if (pop.selectedPost && !pop.selectedPost.resolution) {
// This no longer checks for `!pop.selectedPost.resolution` because the resolution can change if the contentUrl is updated.
if (pop.selectedPost) {
pop.selectedPost.resolution = res;
}
}
Expand Down

0 comments on commit eae1b39

Please sign in to comment.