Merge 9161c1a into dc4ab93

website-scraper · Apr 22, 2016 · 27e22f0 · 27e22f0
2 parents dc4ab93 + 9161c1a
commit 27e22f0
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 32 deletions.
diff --git a/lib/file-handlers/html.js b/lib/file-handlers/html.js
@@ -82,6 +82,7 @@ function loadImgSrcsetResource (context, parentResource, childResourceHtmlData)
 
 		return context.loadResource(childResource).then(function updateSrcsetPart (loadedResource) {
 			if(loadedResource){
+				parentResource.updateChild(childResource, loadedResource);
 				imgScrsetPart.url = loadedResource.getFilename();
 			}
 		});
@@ -108,7 +109,8 @@ function loadGeneralResource (context, parentResource, childResourceHtmlData) {
 		if(!loadedResource){
 			return attr;
 		}
-
+		parentResource.updateChild(htmlResource, loadedResource);
+
 		var relativePath = utils.getRelativePath(parentResource.getFilename(), loadedResource.getFilename());
 		if(context.options.prettifyUrls){
 			relativePath = relativePath.replace(context.options.defaultFilename, '');

diff --git a/test/unit/file-handlers/css-test.js b/test/unit/file-handlers/css-test.js
@@ -98,17 +98,20 @@ describe('Css handler', function () {
 				.c {background: url("img/c.jpg")}\
 			';
 
-			var po = new Resource('http://example.com', '1.css');
-			po.setText(css);
+			var parentResource = new Resource('http://example.com', '1.css');
+			parentResource.setText(css);
+			var updateChildSpy = sinon.spy(parentResource, 'updateChild');
 
-			return loadCss(scraper, po).then(function(){
-				var text = po.getText();
+			return loadCss(scraper, parentResource).then(function(){
+				var text = parentResource.getText();
 				text.should.not.containEql('http://first.com/img/a.jpg');
 				text.should.not.containEql('http://first.com/b.jpg');
 				text.should.not.containEql('img/c.jpg');
 				text.should.containEql('local/a.jpg');
 				text.should.containEql('local/b.jpg');
 				text.should.containEql('local/c.jpg');
+
+				updateChildSpy.calledThrice.should.be.eql(true);
 				done();
 			}).catch(done);
 		});
@@ -127,18 +130,21 @@ describe('Css handler', function () {
 				.c {background: url("img/c.jpg")}\
 			';
 
-			var po = new Resource('http://example.com', '1.css');
-			po.setText(css);
+			var parentResource = new Resource('http://example.com', '1.css');
+			parentResource.setText(css);
+			var updateChildSpy = sinon.spy(parentResource, 'updateChild');
 
-			return loadCss(scraper, po).then(function(){
-				var text = po.getText();
+			return loadCss(scraper, parentResource).then(function(){
+				var text = parentResource.getText();
 				text.should.containEql('http://first.com/img/a.jpg');
 				text.should.containEql('http://first.com/b.jpg');
 				text.should.not.containEql('local/a.jpg');
 				text.should.not.containEql('local/b.jpg');
 
 				text.should.not.containEql('img/c.jpg');
 				text.should.containEql('local/c.jpg');
+
+				updateChildSpy.calledOnce.should.be.eql(true);
 
 				done();
 			}).catch(done);

diff --git a/test/unit/file-handlers/html-test.js b/test/unit/file-handlers/html-test.js
@@ -1,9 +1,10 @@
 require('should');
 var _ = require('lodash');
+var Promise = require('bluebird');
 var sinon = require('sinon');
+require('sinon-as-promised')(Promise);
 var nock = require('nock');
 var fs = require('fs-extra');
-var Promise = require('bluebird');
 var Scraper = require('../../../lib/scraper');
 var Resource = require('../../../lib/resource');
 var loadHtml = require('../../../lib/file-handlers/html');
@@ -167,15 +168,18 @@ describe('Html handler', function () {
 				</html>\
 			';
 
-			var po = new Resource('http://example.com', 'index.html');
-			po.setText(html);
+			var parentResource = new Resource('http://example.com', 'index.html');
+			parentResource.setText(html);
+			var updateChildSpy = sinon.spy(parentResource, 'updateChild');
 
 			// order of loading is determined by order of sources in scraper options
-			loadHtml(scraper, po).then(function() {
+			loadHtml(scraper, parentResource).then(function() {
 				loadResourceSpy.calledThrice.should.be.eql(true);
 				loadResourceSpy.args[0][0].url.should.be.eql('http://example.com/a.jpg');
 				loadResourceSpy.args[1][0].url.should.be.eql('http://example.com/b.css');
 				loadResourceSpy.args[2][0].url.should.be.eql('http://example.com/c.js');
+
+				updateChildSpy.calledThrice.should.be.eql(true);
 				done();
 			}).catch(done);
 		});
@@ -200,11 +204,12 @@ describe('Html handler', function () {
 				</html>\
 			';
 
-			var po = new Resource('http://example.com', 'index.html');
-			po.setText(html);
+			var parentResource = new Resource('http://example.com', 'index.html');
+			parentResource.setText(html);
+			var updateChildSpy = sinon.spy(parentResource, 'updateChild');
 
-			return loadHtml(scraper, po).then(function(){
-				var text = po.getText();
+			return loadHtml(scraper, parentResource).then(function(){
+				var text = parentResource.getText();
 				text.should.containEql('http://example.com/public/img/a.jpg');
 				text.should.containEql('http://example.com/b.css');
 				text.should.not.containEql('local/a.jpg');
@@ -213,6 +218,8 @@ describe('Html handler', function () {
 				text.should.not.containEql('scripts/c.js');
 				text.should.containEql('local/c.js');
 
+				updateChildSpy.calledOnce.should.be.eql(true);
+
 				done();
 			}).catch(done);
 		});
@@ -325,14 +332,14 @@ describe('Html handler', function () {
 		});
 
 		it('should handle img tag with srcset attribute correctly', function (done) {
-
 			var image45Stub = new Resource('http://example.com/image45.jpg', 'local/image45.jpg');
 			var image150Stub = new Resource('http://example.com/image150.jpg', 'local/image150.jpg');
 
 			sinon.stub(scraper, 'loadResource')
-				.onFirstCall().returns(Promise.resolve(image45Stub))
-				.onSecondCall().returns(Promise.resolve(image150Stub))
-				.onThirdCall().returns(Promise.resolve(image45Stub));
+				.onFirstCall().resolves(image45Stub)
+				.onSecondCall().resolves(image150Stub)
+				.onThirdCall().resolves(image45Stub);
+
 
 			var html = '\
 				<html> \
@@ -344,11 +351,14 @@ describe('Html handler', function () {
 				</html>\
 			';
 
-			var po = new Resource('http://example.com', 'index.html');
-			po.setText(html);
+			var parentResource = new Resource('http://example.com', 'index.html');
+			parentResource.setText(html);
+			var updateChildSpy = sinon.spy(parentResource, 'updateChild');
 
-			return loadHtml(scraper, po).then(function () {
-				var text = po.getText();
+			return loadHtml(scraper, parentResource).then(function () {
+				var text = parentResource.getText();
+
+				updateChildSpy.calledThrice.should.be.eql(true);
 
 				text.should.not.containEql('http://example.com/image45.jpg');
 				text.should.not.containEql('http://example.com/image150.jpg');
@@ -363,9 +373,9 @@ describe('Html handler', function () {
 			var image150Stub = new Resource('http://example.com/image150.jpg', 'local/image150.jpg');
 
 			sinon.stub(scraper, 'loadResource')
-				.onFirstCall().returns(Promise.resolve(null))
-				.onSecondCall().returns(Promise.resolve(image150Stub))
-				.onThirdCall().returns(Promise.resolve(null));
+				.onFirstCall().resolves(null)
+				.onSecondCall().resolves(image150Stub)
+				.onThirdCall().resolves(null);
 
 			var html = '\
 				<html> \
@@ -377,11 +387,14 @@ describe('Html handler', function () {
 				</html>\
 			';
 
-			var po = new Resource('http://example.com', 'index.html');
-			po.setText(html);
+			var parentResource = new Resource('http://example.com', 'index.html');
+			parentResource.setText(html);
+			var updateChildSpy = sinon.spy(parentResource, 'updateChild');
 
-			return loadHtml(scraper, po).then(function () {
-				var text = po.getText();
+			return loadHtml(scraper, parentResource).then(function () {
+				var text = parentResource.getText();
+
+				updateChildSpy.calledOnce.should.be.eql(true);
 
 				text.should.containEql('http://example.com/image45.jpg');
 				text.should.not.containEql('src="local/image45.jpg"');