-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.js
121 lines (104 loc) · 3.15 KB
/
app.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
"use strict";
const puppeteer = require("puppeteer");
const devices = require("puppeteer/DeviceDescriptors");
const express = require("express");
const bodyParser = require("body-parser");
const app = express();
app.use(bodyParser.json());
app.post("/", async (req, res) => {
const body = req.body;
let args = ["--no-sandbox"];
if (body.lang) {
args.push("--lang=" + body.lang.join(","));
}
const browser = await puppeteer.launch({ args: args });
const page = await browser.newPage();
if (body.emulate) {
await page.emulate(devices[body.emulate]);
}
const viewport = buildViewport(body.viewport);
page.setViewport({
width: viewport.width,
height: viewport.height,
deviceScaleFactor: viewport.deviceScaleFactor
});
await preScrape(page, body.login);
await page.goto(body.url, { waitUntil: "networkidle2" });
if (typeof body.imageNumber == "number") {
const image = await getImage(page, body.selector, body.imageNumber);
res.type("image/png").send(image);
browser.close();
} else {
const result = await getText(page, body.selector);
res.send(result);
browser.close();
}
});
const server = app.listen(process.env.PORT || 8080, err => {
if (err) return console.error(err);
const port = server.address().port;
console.info(`App listening on port ${port}`);
});
const buildViewport = viewport => {
const _viewport = {
width: 1200,
height: 600,
deviceScaleFactor: 1
};
if (viewport) {
if (viewport.width) _viewport.width = viewport.width;
if (viewport.height) _viewport.height = viewport.height;
if (viewport.deviceScaleFactor)
_viewport.deviceScaleFactor = viewport.deviceScaleFactor;
}
return _viewport;
};
const preScrape = async (page, login) => {
if (login) {
if (login.url) await page.goto(login.url, { waitUntil: "networkidle2" });
if (login.id) {
await page.focus(login.id.selector);
await page.type(login.id.selector, login.id.value);
}
if (login.password) {
await page.focus(login.password.selector);
await page.type(login.password.selector, login.password.value);
}
if (login.submit) {
const submit = await page.$(login.submit.selector);
await submit.click();
}
}
};
const getText = async (page, selector) => {
return await page.evaluate(_selector => {
const result = Array.from(document.querySelectorAll(_selector));
return result.map(data => data.textContent);
}, selector);
};
const getImage = async (page, selector, imageNumber) => {
const rects = await page.evaluate(async _selector => {
const elements = Array.from(document.querySelectorAll(_selector));
return elements.map(element => {
const { x, y, width, height } = element.getBoundingClientRect();
return {
left: x,
top: y,
width: width,
height: height,
id: element.id
};
});
}, selector);
const rect = rects[imageNumber];
const padding = 16;
const buffer = await page.screenshot({
clip: {
x: rect.left - padding,
y: rect.top - padding,
width: rect.width + padding * 2,
height: rect.height + padding * 2
}
});
return buffer;
};