refactor, sourceId

paolo-chiabrera · Aug 24, 2016 · 2a519c4 · 2a519c4
1 parent 0faead7
commit 2a519c4
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 13 deletions.
diff --git a/lib/index.js b/lib/index.js
@@ -21,7 +21,6 @@ export default class PMPScraper {
       scraperApiUrl: 'http://api.scraper.d3lirium.eu',
       folderPath: './images',
       concurrency: 4,
-      threshold: 0.75,
       statsInterval: 5,
       request: {
         timeout: 20000,
@@ -91,7 +90,7 @@ export default class PMPScraper {
         saveImages: ['getImagesThreshold', (results, next) => {
           main.saveImages({
             links: results.filterDuplicates.links,
-            source: val.source,
+            sourceId: val.source.id,
             options: val.options,
             logger: this.logger
           }, next);
@@ -157,7 +156,7 @@ export default class PMPScraper {
             return;
           }
 
-          if (res.threshold < this.options.threshold) {
+          if (res.threshold < val.source.threshold) {
             thresholdMessage = 'threshold not reached: ' + res.threshold;
             next(new Error(thresholdMessage));
             return;

diff --git a/lib/modules/saveImages.js b/lib/modules/saveImages.js
@@ -14,7 +14,7 @@ import validators from './validators';
 export default function saveImages(args, done) {
   const schema = Joi.object().required().keys({
     links: validators.links,
-    source: validators.source,
+    sourceId: validators.sourceId,
     options: validators.options,
     logger: validators.logger
   });
@@ -25,7 +25,7 @@ export default function saveImages(args, done) {
       return;
     }
 
-    const pmpImage = new PmpImage(val.source, {
+    const pmpImage = new PmpImage(val.sourceId, {
       folderPath: val.options.folderPath,
       request: _.omit(val.options.request, 'timeout')
     });

diff --git a/lib/modules/validators.js b/lib/modules/validators.js
@@ -1,22 +1,24 @@
 import Joi from 'joi';
 
 export default {
+  sourceId: Joi.string().required(),
   source: Joi.object().required().keys({
+    _id: Joi.string().optional(),
     id: Joi.string().required(),
     url: Joi.string().required(),
     offset: Joi.number().required(),
     startingOffset: Joi.number().required(),
     mainPageSelector: Joi.string().required(),
     mainPageAttribute: Joi.string().required(),
     imagePageSelector: Joi.string().required(),
-    imagePageAttribute: Joi.string().required()
+    imagePageAttribute: Joi.string().required(),
+    threshold: Joi.number().min(0).max(1).default(0.75).required()
   }),
   options: Joi.object().required().keys({
     pmpApiUrl: Joi.string().required(),
     scraperApiUrl: Joi.string().required(),
     folderPath: Joi.string().required(),
     concurrency: Joi.number().min(1).required(),
-    threshold: Joi.number().min(0).max(1).default(0.75).required(),
     statsInterval: Joi.number().min(1).max(100).default(0.75).optional(),
     request: Joi.object().required().keys({
       json: Joi.boolean().required(),

diff --git a/package.json b/package.json
@@ -54,7 +54,7 @@
     "joi": "^9.0.4",
     "lodash": "^4.15.0",
     "needle": "^1.1.0",
-    "pmp-image": "0.0.6",
+    "pmp-image": "0.0.10",
     "pretty-bytes": "^3.0.1",
     "winston": "^2.2.0"
   }

diff --git a/test/mocks.js b/test/mocks.js
@@ -13,14 +13,14 @@ export default {
     mainPageSelector: 'a.link',
     mainPageAttribute: 'href',
     imagePageSelector: 'img.image',
-    imagePageAttribute: 'src'
+    imagePageAttribute: 'src',
+    threshold: 0.75
   },
   options: {
     pmpApiUrl: 'http://api.picmeplease.eu',
     scraperApiUrl: 'http://api.scraper.d3lirium.eu',
     folderPath: './test-images',
     concurrency: 1,
-    threshold: 0.75,
     statsInterval: 10,
     request: {
       json: true,

diff --git a/test/modules/saveImages.spec.js b/test/modules/saveImages.spec.js
@@ -45,7 +45,7 @@ describe('saveImages', function () {
 
     saveImages({
       links: mocks.filteredDuplicates,
-      source: mocks.source,
+      sourceId: mocks.source.id,
       options: mocks.options,
       logger: {
         log: log
@@ -73,7 +73,7 @@ describe('saveImages', function () {
 
     saveImages({
       links: mocks.filteredDuplicates,
-      source: mocks.source,
+      sourceId: mocks.source.id,
       options: mocks.options
     }, cb);
   }));
@@ -97,7 +97,7 @@ describe('saveImages', function () {
 
     saveImages({
       links: mocks.filteredDuplicates,
-      source: mocks.source,
+      sourceId: mocks.source.id,
       options: mocks.options
     }, cb);
   }));