From 2a519c4719374ff7ef263519b2e7dee21493e586 Mon Sep 17 00:00:00 2001
From: "=paolo.chiabrera" <=paolo.chiabrera@gmail.com>
Date: Wed, 24 Aug 2016 21:57:17 +0200
Subject: [PATCH] refactor, sourceId

---
 lib/index.js                    | 5 ++---
 lib/modules/saveImages.js       | 4 ++--
 lib/modules/validators.js       | 6 ++++--
 package.json                    | 2 +-
 test/mocks.js                   | 4 ++--
 test/modules/saveImages.spec.js | 6 +++---
 6 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/index.js b/lib/index.js
index 6d2c7c0..68a92b8 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -21,7 +21,6 @@ export default class PMPScraper {
       scraperApiUrl: 'http://api.scraper.d3lirium.eu',
       folderPath: './images',
       concurrency: 4,
-      threshold: 0.75,
       statsInterval: 5,
       request: {
         timeout: 20000,
@@ -91,7 +90,7 @@ export default class PMPScraper {
         saveImages: ['getImagesThreshold', (results, next) => {
           main.saveImages({
             links: results.filterDuplicates.links,
-            source: val.source,
+            sourceId: val.source.id,
             options: val.options,
             logger: this.logger
           }, next);
@@ -157,7 +156,7 @@ export default class PMPScraper {
             return;
           }
 
-          if (res.threshold < this.options.threshold) {
+          if (res.threshold < val.source.threshold) {
             thresholdMessage = 'threshold not reached: ' + res.threshold;
             next(new Error(thresholdMessage));
             return;
diff --git a/lib/modules/saveImages.js b/lib/modules/saveImages.js
index 58fd608..a8e2bbf 100644
--- a/lib/modules/saveImages.js
+++ b/lib/modules/saveImages.js
@@ -14,7 +14,7 @@ import validators from './validators';
 export default function saveImages(args, done) {
   const schema = Joi.object().required().keys({
     links: validators.links,
-    source: validators.source,
+    sourceId: validators.sourceId,
     options: validators.options,
     logger: validators.logger
   });
@@ -25,7 +25,7 @@ export default function saveImages(args, done) {
       return;
     }
 
-    const pmpImage = new PmpImage(val.source, {
+    const pmpImage = new PmpImage(val.sourceId, {
       folderPath: val.options.folderPath,
       request: _.omit(val.options.request, 'timeout')
     });
diff --git a/lib/modules/validators.js b/lib/modules/validators.js
index 842643e..5072b86 100644
--- a/lib/modules/validators.js
+++ b/lib/modules/validators.js
@@ -1,7 +1,9 @@
 import Joi from 'joi';
 
 export default {
+  sourceId: Joi.string().required(),
   source: Joi.object().required().keys({
+    _id: Joi.string().optional(),
     id: Joi.string().required(),
     url: Joi.string().required(),
     offset: Joi.number().required(),
@@ -9,14 +11,14 @@ export default {
     mainPageSelector: Joi.string().required(),
     mainPageAttribute: Joi.string().required(),
     imagePageSelector: Joi.string().required(),
-    imagePageAttribute: Joi.string().required()
+    imagePageAttribute: Joi.string().required(),
+    threshold: Joi.number().min(0).max(1).default(0.75).required()
   }),
   options: Joi.object().required().keys({
     pmpApiUrl: Joi.string().required(),
     scraperApiUrl: Joi.string().required(),
     folderPath: Joi.string().required(),
     concurrency: Joi.number().min(1).required(),
-    threshold: Joi.number().min(0).max(1).default(0.75).required(),
     statsInterval: Joi.number().min(1).max(100).default(0.75).optional(),
     request: Joi.object().required().keys({
       json: Joi.boolean().required(),
diff --git a/package.json b/package.json
index 88f736d..fae0f23 100644
--- a/package.json
+++ b/package.json
@@ -54,7 +54,7 @@
     "joi": "^9.0.4",
     "lodash": "^4.15.0",
     "needle": "^1.1.0",
-    "pmp-image": "0.0.6",
+    "pmp-image": "0.0.10",
     "pretty-bytes": "^3.0.1",
     "winston": "^2.2.0"
   }
diff --git a/test/mocks.js b/test/mocks.js
index 3a331a1..8e4b3af 100644
--- a/test/mocks.js
+++ b/test/mocks.js
@@ -13,14 +13,14 @@ export default {
     mainPageSelector: 'a.link',
     mainPageAttribute: 'href',
     imagePageSelector: 'img.image',
-    imagePageAttribute: 'src'
+    imagePageAttribute: 'src',
+    threshold: 0.75
   },
   options: {
     pmpApiUrl: 'http://api.picmeplease.eu',
     scraperApiUrl: 'http://api.scraper.d3lirium.eu',
     folderPath: './test-images',
     concurrency: 1,
-    threshold: 0.75,
     statsInterval: 10,
     request: {
       json: true,
diff --git a/test/modules/saveImages.spec.js b/test/modules/saveImages.spec.js
index 66d64ef..5adf141 100644
--- a/test/modules/saveImages.spec.js
+++ b/test/modules/saveImages.spec.js
@@ -45,7 +45,7 @@ describe('saveImages', function () {
 
     saveImages({
       links: mocks.filteredDuplicates,
-      source: mocks.source,
+      sourceId: mocks.source.id,
       options: mocks.options,
       logger: {
         log: log
@@ -73,7 +73,7 @@ describe('saveImages', function () {
 
     saveImages({
       links: mocks.filteredDuplicates,
-      source: mocks.source,
+      sourceId: mocks.source.id,
       options: mocks.options
     }, cb);
   }));
@@ -97,7 +97,7 @@ describe('saveImages', function () {
 
     saveImages({
       links: mocks.filteredDuplicates,
-      source: mocks.source,
+      sourceId: mocks.source.id,
       options: mocks.options
     }, cb);
   }));