-
Notifications
You must be signed in to change notification settings - Fork 36
/
query.js
69 lines (62 loc) 路 2.09 KB
/
query.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
const fetch = require('isomorphic-fetch')
const queryString = require('query-string')
const debug = require('debug')('scrape-twitter:query')
const https = require('https')
const checkStatus = (response) => {
const requiresLogin = /login\?redirect_after_login/.test(response.url || '')
if (requiresLogin) {
const error = new Error('An active login is required for this API call')
error.response = response
error.statusCode = 403
throw error
} else if (response.ok) {
debug('response was ok')
return response
} else {
debug('response was error:', response)
const error = new Error(response.statusText)
error.response = response
error.statusCode = response.status
throw error
}
}
const toJson = response => response.json()
const toText = response => response.text()
const toHtml = response => {
let minPosition = null
let html = null
if ('items_html' in response) {
minPosition = response['min_position']
html = response['items_html'].trim()
} else if ('conversation_html' in response) {
minPosition = response['min_position']
html = response['conversation_html'].trim()
} else if ('descendants' in response && 'items-html' in response['descendants']) {
minPosition = response['descendants']['min_position']
html = response['descendants']['items_html'].trim()
}
debug('received html of length:', html.length)
if (minPosition) {
debug('the min_position within the response is:', minPosition)
}
return { html, _minPosition: minPosition }
}
const query = (url, options, fetcher = fetch) => {
const qs = queryString.stringify(options)
const resource = url + (qs.length ? `?${qs}` : '')
debug('query on resource:', resource)
return fetcher(resource, { agent: https.globalAgent })
.then(checkStatus)
.then(toJson)
.then(toHtml)
}
const get = (resource, fetcher = fetch) => {
debug('get on resource:', resource)
return fetcher(resource, { agent: https.globalAgent })
.then(checkStatus)
.then(toText)
.then(html => ({ html }))
}
module.exports = query
module.exports.query = query
module.exports.get = get