-
Notifications
You must be signed in to change notification settings - Fork 0
/
http.ts
69 lines (63 loc) 路 1.61 KB
/
http.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
/**
* A utility module for HTTP requests
*
* Uses `got` to enable RFC 7234 compliant HTTP caching
*
* @module util/http
*/
import { getLogger } from '@stencila/logga'
import fs from 'fs'
import got, { Options, Response } from 'got'
import stream from 'stream'
import util from 'util'
import { cache } from './cache'
const pipeline = util.promisify(stream.pipeline)
const log = getLogger('jesta:get')
/**
* A `got` instance with default options for HTTP requests.
*
* User agent is set, and includes a `mailto`, for politeness:
* https://github.com/CrossRef/rest-api-doc#good-manners--more-reliable-service
*/
export const client = got.extend({
cache,
retry: {
limit: 5,
},
headers: {
'user-agent':
'Stencila (https://github.com/stencila/jesta; mailto:hello@stenci.la)',
'accept-encoding': 'gzip, deflate',
},
})
/**
* Get content from a URL
*
* @param url The URL to get
* @param options Options to pass to `got`
*/
export async function get(
url: string,
options: Options = {}
): Promise<Response<string>> {
try {
return (await client.get(url, options)) as Response<string>
} catch (error) {
const { message, response } = error as {
message: string
response: Response<string>
}
const body = response?.body ?? ''
log.warn(`Unable to get ${url}: ${message}: ${body.slice(0, 500)}`)
return response
}
}
/**
* Download a file
*
* @param from The URL to download from
* @param to The file path to download to
*/
export async function download(from: string, to: string): Promise<void> {
return pipeline(client.stream(from), fs.createWriteStream(to))
}