Rewrite Savefrom

package.json

@@ -34,6 +34,7 @@
         "packages/scraper-others",
         "packages/scraper-primbon",
         "packages/scraper-religions",
+        "packages/scraper-savefrom",
         "packages/scraper-sosmed",
         "packages/scraper-texts",
         "packages/scraper-youtube"

packages/scraper-instagram/README.md

@@ -1,22 +1,18 @@
 # Youtube Scraper
-Scraper Youtube data like download video Youtube
+Module to download content from Instagram
 
 ## Installation
 ```sh
-npm i @bochilteam/scraper-youtube
+npm i @bochilteam/scraper-instagram
 ```
 
 ## Usage 
-Example download youtube video
+Example download Instagram video
 ```ts
 // import module
-import { youtubedl } from '@bochilteam/scraper-youtube'
+import { instagramdl } from '@bochilteam/scraper-instagram'
 
-const data = await youtubedl('https://youtu.be/iik25wqIuFo')
+const data = await instagramdl('https://www.instagram.com/reel/CxSEjxfyJtN')
 console.log(data) // JSON
-const resolutions = Object.keys(data.video) // List of resolution/quality
-console.log(resolutions) 
-const url = await data.video[resolutions[0]].download() // Download '720p' video
-console.log(url) // string
 ```
-[Documentation](https://bochilteam.github.io/scraper/modules/_bochilteam_scraper_youtube.html)
+[Documentation](https://bochilteam.github.io/scraper/modules/_bochilteam_scraper_instagram.html)

packages/scraper-instagram/package.json

@@ -41,7 +41,6 @@
   "dependencies": {
     "cheerio": "^1.0.0-rc.12",
     "got": "^11.8.6",
-    "human-readable": "^0.2.1",
     "zod": "^3.20.2"
   }
 }

packages/scraper-savefrom/README.md

@@ -0,0 +1,18 @@
+# Youtube Scraper
+Scraper Savefrom
+
+## Installation
+```sh
+npm i @bochilteam/scraper-savefrom
+```
+
+## Usage 
+Example download youtube video
+```ts
+// import module
+import { savefrom } from '@bochilteam/scraper-savefrom'
+
+const data = await savefrom('https://youtu.be/iik25wqIuFo')
+console.log(data) // JSON
+```
+[Documentation](https://bochilteam.github.io/scraper/modules/_bochilteam_scraper_savefrom.html)

packages/scraper-savefrom/index.ts

@@ -0,0 +1 @@
+export { default as savefrom } from './src/savefrom.js'

packages/scraper-savefrom/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@bochilteam/scraper-savefrom",
+  "version": "0.0.1",
+  "description": "SaveFrom scraper module",
+  "main": "lib/cjs/index.js",
+  "module": "lib/esm/index.js",
+  "types": "lib/@types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./lib/esm/index.js",
+      "require": "./lib/cjs/index.js",
+      "types": "./lib/@types/index.d.ts"
+    }
+  },
+  "scripts": {
+    "test": "npm run test:cjs & npm run test:esm",
+    "test:cjs": "node --test ./lib/cjs/test",
+    "test:esm": "node --test ./lib/esm/test",
+    "build": "npm run build:cjs && npm run build:esm && npm run build:types",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build:types": "tsc -p tsconfig.types.json",
+    "compile": "node ../../scripts/compile-typescript.js && npm run write:package",
+    "write:package": "node ../../scripts/write-package.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/BochilTeam/scraper.git"
+  },
+  "author": "BochilTeam",
+  "license": "GPL-3.0-or-later",
+  "bugs": {
+    "url": "https://github.com/BochilTeam/scraper/issues"
+  },
+  "homepage": "https://github.com/BochilTeam/scraper/tree/master/packages/scraper-savefrom#readme",
+  "files": [
+    "lib/**/*.js",
+    "lib/**/**.d.ts",
+    "lib/**/package.json"
+  ],
+  "dependencies": {
+    "got": "^11.8.6",
+    "zod": "^3.20.2"
+  }
+}

packages/scraper-savefrom/src/constant.ts

@@ -0,0 +1,11 @@
+export const DEFAULT_HEADERS = {
+    'accept': '*/*',
+    'accept-encoding': 'gzip, deflate, br',
+    'accept-language': 'en-US,en;q=0.9',
+    'sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-origin',
+    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'
+}

packages/scraper-savefrom/src/savefrom.ts

@@ -0,0 +1,74 @@
+import got from 'got'
+import vm from 'vm'
+import { DEFAULT_HEADERS } from './constant.js'
+import { generateHash } from './util.js'
+import {
+    SavefromArgsSchema,
+    SavefromSchema
+} from '../types/savefrom.js'
+
+export default async function savefrom (url: string) {
+    SavefromArgsSchema.parse(arguments)
+
+    const form = {
+        sf_url: url,
+        sf_submit: '',
+        new: '2',
+        lang: 'en',
+        app: '',
+        country: 'en',
+        os: 'Windows',
+        browser: 'Chrome',
+        channel: 'main',
+        'sf-nomad': '1',
+        url,
+        ts: Date.now(),
+        _ts: 1695977397919,
+        _tsc: 0,
+        _s: generateHash(url),
+        _x: 0
+    }
+    const data = await got.post('https://worker.savefrom.net/savefrom.php', {
+        headers: {
+            ...DEFAULT_HEADERS,
+            'content-type': 'application/x-www-form-urlencoded',
+            origin: 'https://en.savefrom.net',
+            referer: 'https://en.savefrom.net/'
+        },
+        form
+    }).text()
+    const executeCode = '[]["filter"]["constructor"](b).call(a);'
+    if (data.indexOf(executeCode) === -1) {
+        console.error(data)
+        throw new Error('Cannot find executable code!')
+    }
+    const script = data.replace(executeCode, `
+try {const script = ${executeCode.split('.call')[0]}.toString();if (script.includes('function showResult')) results = script;else (${executeCode.replace(/;/, '')});} catch {}
+`.trim())
+    const context: { results: string | null } = {
+        results: null,
+    }
+    vm.createContext(context)
+    new vm.Script(script).runInContext(context)
+    const executed = context.results!.split('window.parent.sf.videoResult.show(')?.[1]
+        || context.results!.split('window.parent.sf.videoResult.showRows(')?.[1]
+    if (!executed) {
+        console.error(executed, script)
+        throw new Error('Cannot find result data from evaluation!')
+    }
+
+    let json
+    try {
+        if (context.results!.includes('showRows')) {
+            const splits = executed.split('],"')
+            const lastIndex = splits.findIndex(v => v.includes('window.parent.sf.enableElement'))
+            json = JSON.parse(splits.slice(0, lastIndex).join('],"') + ']')
+        } else {
+            json = [JSON.parse(executed.split(');')[0])]
+        }
+    } catch (e) {
+        console.error(e, executed)
+        throw new Error('Cannot parse results data from evaluation!')
+    }
+    return SavefromSchema.parse(json)
+}

packages/scraper-savefrom/src/util.ts

@@ -0,0 +1,10 @@
+import crypto from 'crypto'
+
+const hashed = "fe72c5ca495558be535f81cb175ecab8536889752abd6a2512b9b84ce26f2c81"
+
+export function generateHash (url: string) {
+    const data = url + Date.now() + hashed
+    const hash = crypto.createHash('sha256')
+    hash.update(data)
+    return hash.digest('hex')
+}

packages/scraper-savefrom/test/savefrom.ts

@@ -0,0 +1,38 @@
+import { savefrom } from '../index.js'
+import { test } from 'node:test'
+import assert from 'node:assert'
+
+const YT_URL = 'https://youtu.be/iik25wqIuFo'
+const IG_URL = 'https://www.instagram.com/p/CaHpoweBjmx'
+const TIKTOK_URL = 'https://www.tiktok.com/@omagadsus/video/7025456384175017243'
+const FB_URL = 'https://fb.watch/9WktuN9j-z/'
+// X_URL ?
+const TWITTER_URL = 'https://twitter.com/jen_degen/status/1458167531869458440'
+
+test('Savefrom', async (t) => {
+
+    await t.test('Download Youtube', async () => {
+        const data = await savefrom(YT_URL)
+        assert.strictEqual(data.length > 0, true)
+    })
+
+    await t.test('Download Instagram', async () => {
+        const data = await savefrom(IG_URL)
+        assert.strictEqual(data.length > 0, true)
+    })
+
+    await t.test('Download Tiktok', async () => {
+        const data = await savefrom(TIKTOK_URL)
+        assert.strictEqual(data.length > 0, true)
+    })
+
+    await t.test('Download Facebook', async () => {
+        const data = await savefrom(FB_URL)
+        assert.strictEqual(data.length > 0, true)
+    })
+
+    await t.test('Download Twitter (X)', async () => {
+        const data = await savefrom(TWITTER_URL)
+        assert.strictEqual(data.length > 0, true)
+    })
+})

packages/scraper-savefrom/tsconfig.cjs.json

@@ -0,0 +1,14 @@
+{
+    "extends": "../../tsconfig.cjs.json",
+    "compilerOptions": {
+        "outDir": "./lib/cjs"
+    },
+    "include": [
+        "./index.ts",
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-savefrom/tsconfig.esm.json

@@ -0,0 +1,14 @@
+{
+    "extends": "../../tsconfig.esm.json",
+    "compilerOptions": {
+        "outDir": "./lib/esm"
+    },
+    "include": [
+        "./index.ts",
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-savefrom/tsconfig.types.json

@@ -0,0 +1,14 @@
+{
+    "extends": "../../tsconfig.types.json",
+    "compilerOptions": {
+        "outDir": "./lib/@types"
+    },
+    "include": [
+        "./index.ts",
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-savefrom/typedoc.json

@@ -0,0 +1,10 @@
+{
+  "extends": [
+    "../../typedoc.base.json"
+  ],
+  "entryPoints": [
+    "index.ts",
+    "types/*"
+  ],
+  "tsconfig": "../../tsconfig.base.json"
+}

packages/scraper-savefrom/types/savefrom.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod'
+
+export const SavefromArgsSchema = z.object({
+    0: z.string().url()
+})
+export const SavefromUrlItemSchema = z.object({
+    url: z.string().url(),
+    name: z.string(),
+    type: z.string(),
+    ext: z.string(),
+})
+export const SavefromVideoSchema = z.object({
+    url: z.string().url(),
+    format: z.string().optional()
+})
+export const SavefromItemSchema = z.object({
+    url: z.array(SavefromUrlItemSchema),
+    thumb: z.string().url(),
+    sd: SavefromVideoSchema.nullish(),
+    meta: z.object({
+        title: z.string(),
+        source: z.string().url(),
+        duration: z.string().optional(),
+        tags: z.string().optional()
+    }),
+    video_quality: z.array(z.string()).optional(),
+    hosting: z.string().optional(),
+    hd: SavefromVideoSchema.nullish(),
+})
+export const SavefromSchema = z.array(SavefromItemSchema)
+export type Savefrom = z.infer<typeof SavefromSchema>

packages/scraper-sosmed/index.ts

@@ -1,5 +1,4 @@
 export { default as aiovideodl } from './src/aiovideodl.js'
-export { default as savefrom } from './src/savefrom.js'
 export { default as snapsave } from './src/snapsave.js'
 
 export { groupWA } from './src/groupWA.js'