Monorepo

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [20.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

.gitignore

@@ -106,5 +106,8 @@ dist
 # lib
 lib/
 
+# TurboRepo
+.turbo
+
 package-lock.json
 pnpm-lock.yaml

package.json

@@ -1,79 +1,25 @@
 {
-  "name": "@bochilteam/scraper",
-  "version": "4.0.22-alpha.0",
-  "description": "scraper module",
-  "exports": {
-    ".": {
-      "types": "./lib/@types/index.d.ts",
-      "import": "./lib/esm/index.js",
-      "require": "./lib/cjs/index.js"
-    }
-  },
-  "main": "./lib/cjs/index.js",
-  "module": "./lib/esm/index.js",
-  "types": "./lib/@types/index.d.ts",
-  "engines": {
-    "node": ">= 12.20"
-  },
-  "scripts": {
-    "test": "npm run build && npm run test:cjs && npm run test:esm",
-    "test:cjs": "mocha --timeout 20000 ./lib/cjs/**/test.js",
-    "test:esm": "mocha --timeout 20000 ./lib/esm/**/test.js",
-    "build": "npm run clean && npm run build:cjs && npm run build:esm && npm run build:types && node scripts/write-package.js",
-    "build:cjs": "tsc --p src/tsconfig.cjs.json",
-    "build:esm": "tsc --p src/tsconfig.esm.json",
-    "build:types": "tsc --p tsconfig.types.json",
-    "lint:fix": "npx eslint src/** --fix --ignore-pattern src/*.json",
-    "clean": "rimraf ./lib",
-    "prepare": "node scripts/compile-typescript.js"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/BochilTeam/scraper.git"
-  },
-  "author": "BochilTeam",
-  "license": "GPL-3.0-or-later",
-  "bugs": {
-    "url": "https://github.com/BochilTeam/scraper/issues"
-  },
-  "homepage": "https://github.com/BochilTeam/scraper#readme",
-  "devDependencies": {
-    "@types/chai": "^4.3.0",
-    "@types/mocha": "^10.0.0",
-    "@types/node": "^18.0.3",
-    "@types/ws": "^8.2.2",
-    "@typescript-eslint/eslint-plugin": "^5.13.0",
-    "@typescript-eslint/parser": "^5.13.0",
-    "chai": "^4.3.6",
-    "eslint": "^7.32.0",
-    "eslint-config-standard": "^16.0.3",
-    "eslint-plugin-import": "^2.25.4",
-    "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^6.1.0",
-    "mocha": "^10.0.0",
-    "typescript": "^4.9.4"
-  },
-  "dependencies": {
-    "cheerio": "1.0.0-rc.12",
-    "form-data": "^4.0.0",
-    "got": "^11.8.3",
-    "human-readable": "^0.2.1",
-    "rimraf": "^3.0.2",
-    "similarity": "^1.2.1",
-    "zod": "^3.20.2"
-  },
-  "directories": {
-    "lib": "lib",
-    "src": "src",
-    "scripts": "scripts"
-  },
-  "files": [
-    "lib/**/*.js",
-    "lib/**/**.d.ts",
-    "lib/**/package.json",
-    "scripts/*"
-  ],
-  "optionalDependencies": {
-    "ws": "^8.4.2"
-  }
-}
+    "name": "@bochilteam/scraper-root",
+    "private": true,
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/BochilTeam/scraper.git"
+    },
+    "scripts": {
+        "compile": "turbo run compile --parallel"
+    },
+    "devDependencies": {
+        "turbo": "^1.6.3"
+    },
+    "workspaces": [
+        "packages/scraper-downloader",
+        "packages/scraper-games",
+        "packages/scraper-images",
+        "packages/scraper-news",
+        "packages/scraper-others",
+        "packages/scraper-primbon",
+        "packages/scraper-religions",
+        "packages/scraper-sosmed",
+        "packages/scraper-texts"
+    ]
+}

packages/scraper-downloader/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@bochilteam/scraper-downloader",
+  "version": "1.0.0",
+  "description": "Downloader scraper module",
+  "main": "lib/cjs/index.js",
+  "module": "lib/esm/index.js",
+  "types": "lib/@types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./lib/esm/index.js",
+      "require": "./lib/cjs/index.js",
+      "types": "./lib/@types/index.d.ts"
+    }
+  },
+  "scripts": {
+    "test": "npm run test:cjs & npm run test:esm",
+    "test:cjs": "mocha ./lib/cjs/test/*.js",
+    "test:esm": "mocha ./lib/esm/test/*.js",
+    "build": "npm run build:cjs && npm run build:esm && npm run build:types",
+    "build:cjs": "tsc -p tsconfig.cjs.json",
+    "build:esm": "tsc -p tsconfig.esm.json",
+    "build:types": "tsc -p tsconfig.types.json",
+    "compile": "node ../../scripts/compile-typescript.js && node ../../scripts/write-package.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/BochilTeam/scraper.git"
+  },
+  "author": "BochilTeam",
+  "license": "GPL-3.0-or-later",
+  "bugs": {
+    "url": "https://github.com/BochilTeam/scraper/issues"
+  },
+  "homepage": "https://github.com/BochilTeam/scraper#readme",
+  "files": [
+    "./lib/**/*.js"
+  ],
+  "dependencies": {
+    "got": "^11.8.6",
+    "zod": "^3.20.2"
+  },
+  "devDependencies": {
+    "@types/mocha": "^10.0.1",
+    "@types/node": "^18.11.17",
+    "mocha": "^10.2.0"
+  }
+}

packages/scraper-downloader/src/index.ts

@@ -0,0 +1,2 @@
+export * from './mediafire.js'
+export * from './sfilemobi.js'

src/others/mediafire.ts → packages/scraper-downloader/src/mediafire.ts

@@ -1,7 +1,7 @@
 import got from 'got'
 import cheerio from 'cheerio'
-import { Mediafire, MediafireArgsSchema, MediafireSchema } from './types.js'
-import { parseFileSize } from '../utils.js'
+import { Mediafire, MediafireArgsSchema, MediafireSchema } from '../types/index.js'
+import { parseFileSize } from '../utils/index.js'
 
 export async function mediafiredl (url: string): Promise<Mediafire> {
   MediafireArgsSchema.parse(arguments)

src/others/sfilemobi.ts → packages/scraper-downloader/src/sfilemobi.ts

@@ -2,16 +2,15 @@ import got from 'got'
 import cherio from 'cheerio'
 import {
   parseFileSize,
-  ScraperError
-} from '../utils.js'
+} from '../utils/index.js'
 import {
   SfileMobi,
   SfileMobiArgsSchema,
   SfileMobiSchema,
   SfileMobiSearch,
   SfileMobiSearchArgsSchema,
   SfileMobiSearchSchema
-} from './types.js'
+} from '../types/index.js'
 
 export async function sfilemobiSearch (query: string, page: number = 1): Promise<SfileMobiSearch[]> {
   SfileMobiSearchArgsSchema.parse(arguments)
@@ -41,7 +40,7 @@ export async function sfilemobiSearch (query: string, page: number = 1): Promise
   })
 
   if (!results.length) {
-    throw new ScraperError(`No results for ${query}`)
+    throw new Error(`No results for ${query}`)
   }
 
   return results.map(res => SfileMobiSearchSchema.parse(res))
@@ -50,19 +49,16 @@ export async function sfilemobiSearch (query: string, page: number = 1): Promise
 export async function sfilemobi (url: string): Promise<SfileMobi> {
   SfileMobiArgsSchema.parse(arguments)
 
-  if (!/sfile\.mobi/i.test(url)) throw new ScraperError(`Invalid URL: ${url}`)
+  if (!/sfile\.mobi/i.test(url)) throw new Error(`Invalid URL: ${url}`)
 
   const html = await got(url).text()
   const $ = cherio.load(html)
 
   const $k = /var z = (.*?);/i.exec($.html())?.[1]
-  const urlPage = (
+  const dlUrl = (
     (/var db = "(.*?)"/i.exec($.html())?.[1] || /var sf = "(.*?)"/i.exec($.html())?.[1])?.replace(/\\(\\)?/gi, '') ||
     $('#download').attr('href')
   ) + `&k=${$k}`
-  const urlHtml = await got(urlPage).text()
-  const $$ = cherio.load(urlHtml)
-  const _url = $$('div.menu > div > p > a').attr('href')
   const filename = $('div.intro-container > img').attr('alt') || $('div.intro-container > h1').text()
   const icon = $('div.intro-container > img').attr('src')
   const type = /\/smallicon\/(.*?)\.svg/.exec(icon!)?.[1]
@@ -75,11 +71,11 @@ export async function sfilemobi (url: string): Promise<SfileMobi> {
   const aploudon = $aploud.eq(1).text()
   const aploudonUrl = $aploud.eq(1).attr('href')
   const downloads = parseInt($list.eq(3).text().split('Downloads:')[1]?.trim())
-  const filesizeH = /\((.*?)\)/i.exec($$('div.menu > div > h1 > b').text())?.[1]
+  const filesizeH = /\((.*?)\)/i.exec($('#download').text())?.[1]
   const filesize = filesizeH && parseFileSize(filesizeH)
 
   const results = {
-    url: _url!,
+    url: dlUrl,
     filename,
     icon,
     type,
@@ -91,7 +87,7 @@ export async function sfilemobi (url: string): Promise<SfileMobi> {
     aploudonUrl,
     downloads,
     filesizeH,
-    filesize: filesize! as number
+    filesize
   }
   return SfileMobiSchema.parse(results)
 }

packages/scraper-downloader/test/index.ts

@@ -0,0 +1,33 @@
+import { describe, it } from 'mocha'
+import {
+    mediafiredl,
+    sfilemobiSearch, sfilemobi,
+} from '../src/index.js'
+
+describe('Downloader', () => {
+    describe('Mediafire', () => {
+        it('Mediafire Download', done => {
+            mediafiredl('https://www.mediafire.com/file/gpeiucmm1xo6ln0/hello_world.mp4/file').then(() => {
+
+                return done()
+            }).catch(done)
+        })
+    })
+
+    describe('sfilemobi', function () {
+        this.timeout(5000)
+        it('sfilemobi search', done => {
+            sfilemobiSearch('minecraft').then(() => {
+
+                return done()
+            }).catch(done)
+        })
+
+        it('sfilemobi download', done => {
+            sfilemobi('https://sfile.mobi/oGm8kAIQCs7').then(() => {
+
+                return done()
+            }).catch(done)
+        })
+    })
+})

packages/scraper-downloader/tsconfig.cjs.json

@@ -0,0 +1,13 @@
+{
+    "extends": "../../tsconfig.cjs.json",
+    "compilerOptions": {
+        "outDir": "./lib/cjs"
+    },
+    "include": [
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-downloader/tsconfig.esm.json

@@ -0,0 +1,13 @@
+{
+    "extends": "../../tsconfig.esm.json",
+    "compilerOptions": {
+        "outDir": "./lib/esm"
+    },
+    "include": [
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}

packages/scraper-downloader/tsconfig.types.json

@@ -0,0 +1,13 @@
+{
+    "extends": "../../tsconfig.types.json",
+    "compilerOptions": {
+        "outDir": "./lib/@types"
+    },
+    "include": [
+        "./src/**/*.ts",
+        "./test/*.ts"
+    ],
+    "exclude": [
+        "./node_modules/**"
+    ]
+}