Skip to content

Commit

Permalink
feat(openLibrary/producer): Add parser functionality in the OL produc…
Browse files Browse the repository at this point in the history
…ers code
  • Loading branch information
shivam-tripathi committed Jul 15, 2018
1 parent 89c889a commit 5c92a06
Showing 1 changed file with 17 additions and 3 deletions.
20 changes: 17 additions & 3 deletions src/openLibrary/producer/producer.js
Expand Up @@ -23,6 +23,7 @@ import {Queue} from '../../queue';
import fs from 'fs';
import {isNotDefined} from '../../helpers/utils';
import log from '../../helpers/logger';
import parser from './parser';
import readline from 'readline';


Expand Down Expand Up @@ -67,10 +68,23 @@ function readLine({base, id, init}, callback) {
// ➜ revision - revision number of the record
// ➜ last_modified - last modified timestamp
// ➜ JSON - the complete record in JSON format
const json = JSON.parse(line.split('\t')[4]);
const record = line.split('\t');

const source = 'OPENLIBRARY';
const json = JSON.parse(record[4]);
const OLType = record[0].split('/')[2];
const data = parser(OLType, json);
const originId = record[1].split('/')[2];
const lastEdited = record[3];

log.log(`WORKER${id}:: Pushing record ${count}`);
// TODO : parse(json) Implement parser function to process records
queue.push(json);
queue.push({
data,
entityType: data.entityType,
lastEdited: lastEdited || data.lastEdited,
originId: originId || data.originId,
source
});
}
catch (err) {
log.warning(
Expand Down

0 comments on commit 5c92a06

Please sign in to comment.