-
Notifications
You must be signed in to change notification settings - Fork 11
/
airtableToElasticSearch.js
86 lines (70 loc) · 2.06 KB
/
airtableToElasticSearch.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import '../util/catchUnhandledRejection';
import config from 'config';
import Airtable from 'airtable';
import elasticsearch from 'elasticsearch'
const client = new elasticsearch.Client({
host: config.get('ELASTICSEARCH_URL'),
log: 'trace',
})
const base = new Airtable({ apiKey: config.get('AIRTABLE_APIKEY') }).base(config.get('AIRTABLE_BASE'));
function fetchRumors() {
const data = {
rumors: [],
answers: [],
};
return new Promise((resolve, reject) => {
base('Rumors').select({
view: config.get('AIRTABLE_VIEW'),
}).eachPage(function page(records, fetchNextPage) {
// This function (`page`) will get called for each page of records.
records.forEach(function(record) {
console.log('Retrieved ', record.get('Message ID'));
const answer = record.get('Answer');
const answerIds = []
if(answer) {
answerIds.push(`${record.get('Message ID')}-answer`);
}
data.rumors.push({
id: `${record.get('Message ID')}-rumor`,
text: record.get('Rumor Text'),
answerIds,
});
if(record.get('Answer')) {
data.answers.push({
id: answerIds[0],
versions: [{
text: record.get('Answer'),
reference: record.get('Reference'),
}],
});
}
});
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
fetchNextPage();
}, function done(error) {
if (error) {
reject(error);
} else {
resolve(data);
}
});
});
}
async function main() {
const data = await fetchRumors();
const body = [];
Object.keys(data).forEach(indexName => {
data[indexName].forEach(({id, ...doc}) => {
// action description
body.push({index: {_index: indexName, _type: 'basic', _id: id}});
// document
body.push(doc);
});
});
console.log(await client.bulk({
body
}));
}
main();