/
provider.ts
44 lines (42 loc) · 1.1 KB
/
provider.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
import { IncomingHttpHeaders } from "http";
import { AxiosRequestConfig } from "axios";
import { Crawler } from "./crawler";
import { Url, Response } from "./http";
export interface ProviderFactory {
new (ctx: Crawler): Provider;
}
export interface Provider {
/**
* The name of provider
*/
name: string;
/**
* The urls that was originally crawled
*/
urls: Url[];
/**
* The default headers for send request.
*/
defaultHeaders?: IncomingHttpHeaders;
/**
* How to parse the response and extract data
* @param response The request response
*/
parse(response: Response): Promise<any[]>;
/**
* Before start crawl the starts up. we can return a new start urls.
* You can change `urls` in this function.
* @param startUrls
*/
beforeStartUrl?(startUrls: Url[]): Promise<Url[]>;
/**
* Before start provider. something we want to do.
* eg. connect to database.
*/
beforeStart?(): Promise<void>;
/**
* When before request. You can change the config before send request.
* @param config
*/
beforeRequest?(config: AxiosRequestConfig): Promise<void>;
}