Skip to content

Commit

Permalink
refactor: separate side effect operations to service from controller (#…
Browse files Browse the repository at this point in the history 
…98)

* try: inject services to controller

* refactor(ExhentaiController): getLastestExHentaiSet & getThumbnaiInfo

* refactor(ExhentaiController): download

* refactor(DocumentController): combine Markdown and Mapping controller

* refactor(MainPageController): add service

* refactor(DocumentController): add, update, delete
  • Loading branch information
@orzyyyy
orzyyyy committed Jul 2, 2019
1 parent 5fc56d5 commit 99c11bf
Show file tree
Hide file tree
Showing 18 changed files with 595 additions and 563 deletions.
74 changes: 74 additions & 0 deletions server/controller/DocumentController.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { Controller, Request } from '../utils/decorator';
import md5 from 'blueimp-md5';
import { getWriteFilesPaths } from '../utils/document';
import DocumentService from '../service/DocumentService';

export interface MappingProps {
[x: string]: any;
createTime?: number;
modifyTime?: number;
id: string;
title?: string;
url?: string;
type?: string;
subType?: string;
category?: 'mapping' | 'markdown';
}

@Controller('/document')
export default class MarkdownController {
@Request({ url: '/update', method: 'post' })
async updateTargetDocument(ctx: any) {
const { layout, id, title, type, subType, category } = ctx.request.body;
if (!id) {
throw Error('id is undefined');
}
const service = new DocumentService();
// update mapping
service.updateMapping({
id,
title,
modifyTime: new Date().getTime(),
type,
subType,
category,
});
// update layout for mapping, content for markdown
const writeFilesPaths = getWriteFilesPaths(category, id);
const originContent = service.getOriginContent(
writeFilesPaths[0],
layout,
id,
);
service.updateContent(category, writeFilesPaths, originContent);
ctx.response.body = true;
}

@Request({ url: '/add', method: 'post' })
async initDocument(ctx: any) {
const { title, type, subType, category } = ctx.request.body;
const service = new DocumentService();
const timeStamp = new Date().getTime();
const id = md5(timeStamp.toString());
const writeFilesPaths = getWriteFilesPaths(category, id);
service.updateMapping({
id,
title,
type,
subType,
category,
});
service.updateContent(category, writeFilesPaths);
ctx.response.body = id;
}

@Request({ url: '/delete', method: 'delete' })
async deleteTargetDocument(ctx: any) {
const { id, category } = ctx.request.body;
const service = new DocumentService();
const writeFilesPaths = getWriteFilesPaths(category, id);
service.deleteTargetDocument(writeFilesPaths);
service.updateMapping({ id }, true);
ctx.response.body = true;
}
}
281 changes: 32 additions & 249 deletions server/controller/ExhentaiController.ts
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,8 @@
import { getTargetResource } from '../utils/resource';
import fs from 'fs-extra';
import path from 'path';
import puppeteer from 'puppeteer-core';
import { format } from 'date-fns';
import request from 'request-promise';
import { success, info, trace, error } from '../utils/log';
import { success, info } from '../utils/log';
import { Controller, Request } from '../utils/decorator';

const { exHentai: exHentaiCookie } = getTargetResource('cookie');
const { exHentai } = getTargetResource('server');
import { writeIntoJsonFile, getTimeStamp } from '../utils/common';
import ExhentaiService from '../service/ExhentaiService';
import { getLastestListInfo, getLastestListFileName } from '../utils/exhentai';

export interface ExHentaiInfoItem {
name: string;
Expand All @@ -17,261 +11,50 @@ export interface ExHentaiInfoItem {
thumbnailUrl: string;
}

const getLastestFileName = () => {
const exHentaiInfoPath = path.join(process.cwd(), './src/assets/exhentai/');
const exHentaiInfoFiles = fs
.readdirSync(exHentaiInfoPath)
.filter((item: string) => item !== '.gitkeep')
.map((item: string) => parseInt(item, 10));
return exHentaiInfoFiles.sort((a: any, b: any) => b - a);
};

const setExHentaiCookie = async (page: any) => {
for (const item of exHentaiCookie) {
await page.setCookie(item);
}
};

const getExHentaiInfo = async ({
pageIndex,
page,
}: {
pageIndex: number;
page: any;
}) => {
await page.goto('https://www.google.com/', {
waitUntil: 'domcontentloaded',
});
await page.goto(exHentai.href + pageIndex, {
waitUntil: 'domcontentloaded',
});
const exHentaiInfo = await page.$$eval(
'div.gl1t',
(wrappers: any[]) =>
new Promise(resolve => {
const results: ExHentaiInfoItem[] = [];
for (const item of wrappers) {
const tempPostTime = item.lastChild.innerText.replace(/[^0-9]/gi, '');
const year = tempPostTime.substring(0, 4);
const month = tempPostTime.substring(5, 6) - 1;
const day = tempPostTime.substring(7, 8);
const hour = tempPostTime.substring(9, 10);
const minute = tempPostTime.substring(11, 12);

const postTime = new Date(year, month, day, hour, minute).getTime();
results.push({
name: item.firstChild.innerText,
detailUrl: item.firstChild.href,
postTime,
thumbnailUrl: item.childNodes[1].firstChild.firstChild.src,
});
}
resolve(results);
}),
);
return exHentaiInfo;
};

const launchExHentaiPage = async () => {
const browser = await puppeteer.launch({
executablePath: exHentai.executablePath,
args: exHentai.launchArgs,
devtools: exHentai.devtools,
});
success('launch puppeteer');
const page = await browser.newPage();
setExHentaiCookie(page);
success('set cookie');
return { page, browser };
};

const getAllThumbnaiUrls = async (page: any) =>
await page.$$eval(
exHentai.thumbnailClass,
(wrappers: any[]) =>
new Promise(resolve => {
const result: any[] = [];
for (const item of wrappers) {
result.push(item.href);
}
resolve(result);
}),
);

const getUrlFromPaginationInfo = async (page: any) =>
await page.$$eval(
'table.ptt a',
(wrappers: any[]) =>
new Promise(resolve => {
if (wrappers.length !== 1) {
const result: string[] = [];
wrappers.pop();
wrappers.shift();
for (const item of wrappers) {
result.push(item.href);
}
resolve(result);
} else {
resolve([]);
}
}),
);

@Controller('/exhentai')
export default class ExhentaiController {
@Request({ url: '/', method: 'get' })
async getExhentai(ctx: any) {
const { page, browser } = await launchExHentaiPage();
let results: ExHentaiInfoItem[] = [];
const lastestFileName = getLastestFileName()[0];
const lastestFilePath = path.join(
process.cwd(),
`src/assets/exhentai/${lastestFileName}.json`,
);
const { postTime } = JSON.parse(
fs.readFileSync(lastestFilePath).toString(),
)[0];
for (let i = 0; i < exHentai.maxPageIndex; i++) {
info(`fetching pageIndex => ${i + 1}`);
const result = await getExHentaiInfo({ pageIndex: i, page });
results = [...results, ...result];
// compare lastest date of comic, break when current comic has been fetched
if (result.length > 0) {
if (result[result.length - 1].postTime < postTime) {
break;
}
}

await page.waitFor(exHentai.waitTime);
}
await browser.close();

trace('write into json');
const createTime = format(new Date(), 'yyyyMMddHHmmss');
fs.outputJSON(
path.join(process.cwd(), `src/assets/exhentai/${createTime}.json`),
results,
).catch((err: any) => {
error('write into json' + err);
});
success('write into json');

ctx.response.body = `./assets/${createTime}.json`;
async getThumbnaiInfo(ctx: { response: { body: string } }) {
const service = new ExhentaiService();
await service.initBrowser();
const lastestListInfo = await getLastestListInfo();
const results = await service.fetchListInfo(lastestListInfo);
const createTime = getTimeStamp();
writeIntoJsonFile(`src/assets/exhentai/${createTime}`, results);
success('fetch completed.');
ctx.response.body = `./assets/exhentai/${createTime}.json`;
}

@Request({ url: '/getLastestSet', method: 'get' })
async getLastestExHentaiSet(ctx: any) {
ctx.response.body = `./assets/exhentai/${getLastestFileName()[0]}.json`;
ctx.response.body = `./assets/exhentai/${getLastestListFileName()}.json`;
}

@Request({ url: '/download', method: 'post' })
async downloadImages(ctx: any) {
const { url, name } = ctx.request.body;
const subName = name.replace(
/[·！#￥（——）：；“”‘、，|《。》？、【】[\]]/gim,
'',
);
info(`download from: ${url}`);
const { page, browser } = await launchExHentaiPage();
await page.goto('https://www.google.com/', {
waitUntil: 'domcontentloaded',
});
await page.goto(url, { waitUntil: 'domcontentloaded' });
const { url } = ctx.request.body;
const service = new ExhentaiService();
await service.initBrowser();
await service.gotoTargetPage(url);
const prefixPath = await service.ensureFolderForSave();

info(`start fetching thumbnai urls from: ${url}`);

const thumbnailUrls = await service.getThumbnaiUrlFromDetailPage();
writeIntoJsonFile(`${prefixPath}/restDetailUrls`, thumbnailUrls);

info(`start fetching target images`);

const images = await service.fetchTargetImageUrls(thumbnailUrls);

// prepare for download
const datePath = format(new Date(), 'yyyyMMdd');
fs.ensureDirSync(
path.join(
process.cwd(),
`${exHentai.downloadPath}/${datePath}/${subName}`,
),
);
success('fetch all image urls');

const restDetailUrls = await getUrlFromPaginationInfo(page);
const firstPageThumbnailUrls = await getAllThumbnaiUrls(page);
await page.waitFor(exHentai.waitTime);
writeIntoJsonFile(`${prefixPath}/detailImageUrls`, images);

for (const item of restDetailUrls) {
await page.goto('https://www.google.com/', {
waitUntil: 'domcontentloaded',
});
await page.goto(item, { waitUntil: 'domcontentloaded' });
const thumbnailUrlsFromNextPage = await getAllThumbnaiUrls(page);
firstPageThumbnailUrls.push(...thumbnailUrlsFromNextPage);
info('image length: ' + firstPageThumbnailUrls.length);
await page.waitFor(exHentai.waitTime);
}
await service.downImages(images, prefixPath);

const images = [];
const targetImgUrls = firstPageThumbnailUrls;
// get thumbnail url in detail page
for (let i = 0; i < targetImgUrls.length; i++) {
await page.goto('https://www.google.com/', {
waitUntil: 'domcontentloaded',
});
await page.goto(targetImgUrls[i], { waitUntil: 'domcontentloaded' });
info(`fetching image url => ${targetImgUrls[i]}`);
const imgUrl = await page.$eval(
'[id=i3] img',
(target: any) =>
new Promise(resolve => {
resolve(target.src);
}),
);
images.push(imgUrl);
await page.waitFor(exHentai.waitTime);
}
success('fetch all images');
// save image url into file, for unexpect error
fs.outputJSON(
path.join(
process.cwd(),
`${exHentai.downloadPath}/${datePath}/${subName}/restDetailUrls.json`,
),
targetImgUrls,
).catch((err: any) => {
error('write into json' + err);
});
fs.outputJSON(
path.join(
process.cwd(),
`${exHentai.downloadPath}/${datePath}/${subName}/detailImageUrls.json`,
),
images,
).catch((err: any) => {
error('write into json' + err);
});
success('download completed');

// fetch and save images
for (let i = 0; i < images.length; i++) {
const item = images[i];
trace('download begin: ' + item);
await request
.get({ url: item, proxy: exHentai.proxy } as {
url: string;
proxy: string;
})
.on('error', (err: any) => {
error(err + ' => ' + item);
})
.pipe(
fs
.createWriteStream(
path.join(
process.cwd(),
`${exHentai.downloadPath}/${datePath}/${subName}/${i + 1}.jpg`,
),
)
.on('finish', () => success(`${i + 1}.jpg`))
.on('error', (err: any) =>
error(`${subName}-${i + 1}.jpg failed, ${err}`),
),
);
if (i % 4 === 0) {
await page.waitFor(exHentai.waitTime);
}
}
await browser.close();
ctx.response.body = true;
}
}

0 comments on commit 99c11bf

Please sign in to comment.