Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ Default options you can find in [lib/config/defaults.js](https://github.com/s0ph
- `result`: if error - `null`, if success - array of [Resource](https://github.com/s0ph1e/node-website-scraper/blob/master/lib/resource.js) objects containing:
- `url`: url of loaded page
- `filename`: filename where page was saved (relative to `directory`)
- `assets`: array of children Resources
- `children`: array of children Resources

### Filename Generators
The filename generator determines where the scraped files are saved.
Expand Down
10 changes: 6 additions & 4 deletions lib/resource.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ function Resource (url, filename) {
this.url = url;
this.filename = filename;

this.assets = [];
this.type = null;
this.depth = 0;

this.parent = null;
this.children = [];

this.saved = false;
}

Expand All @@ -18,15 +20,15 @@ Resource.prototype.createChild = function createChild (url, filename) {
child.parent = this;
child.depth = ++currentDepth;

this.assets.push(child);
this.children.push(child);

return child;
};

Resource.prototype.updateChild = function updateChild (oldChild, newChild) {
var index = this.assets.indexOf(oldChild);
var index = this.children.indexOf(oldChild);
if (index >= 0) {
this.assets[index] = newChild;
this.children[index] = newChild;
}
};

Expand Down
18 changes: 9 additions & 9 deletions test/functional/base/base.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -70,19 +70,19 @@ describe('Functional base', function() {
result.should.be.instanceOf(Array).and.have.length(3);

result[0].should.have.properties({ url: 'http://example.com/', filename: 'index.html' });
result[0].should.have.properties('assets');
result[0].assets.should.be.instanceOf(Array).and.have.length(4);
result[0].assets[0].should.be.instanceOf(Resource);
result[0].should.have.properties('children');
result[0].children.should.be.instanceOf(Array).and.have.length(4);
result[0].children[0].should.be.instanceOf(Resource);

result[1].should.have.properties({ url: 'http://example.com/about', filename: 'about.html' });
result[1].should.have.properties('assets');
result[1].assets.should.be.instanceOf(Array).and.have.length(4);
result[1].assets[0].should.be.instanceOf(Resource);
result[1].should.have.properties('children');
result[1].children.should.be.instanceOf(Array).and.have.length(4);
result[1].children[0].should.be.instanceOf(Resource);

result[2].should.have.properties({ url: 'http://blog.example.com/', filename: 'blog.html' }); // url after redirect
result[2].should.have.properties('assets');
result[2].assets.should.be.instanceOf(Array).and.have.length(1);
result[2].assets[0].should.be.instanceOf(Resource);
result[2].should.have.properties('children');
result[2].children.should.be.instanceOf(Array).and.have.length(1);
result[2].children[0].should.be.instanceOf(Resource);

// should create directory and subdirectories
fs.existsSync(testDirname).should.be.eql(true);
Expand Down
6 changes: 3 additions & 3 deletions test/unit/scraper-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ describe('Scraper', function () {
}).catch(done);
});

it('should return array of objects with url, filename and assets', function(done) {
it('should return array of objects with url, filename and children', function(done) {
nock('http://first-url.com').get('/').reply(200, 'OK');
nock('http://second-url.com').get('/').reply(500);

Expand All @@ -154,8 +154,8 @@ describe('Scraper', function () {
s.load().then(function(res) {
res.should.be.instanceOf(Array);
res.should.have.length(2);
res[0].should.be.instanceOf(Resource).and.have.properties(['url', 'filename', 'assets']);
res[1].should.be.instanceOf(Resource).and.have.properties(['url', 'filename', 'assets']);
res[0].should.be.instanceOf(Resource).and.have.properties(['url', 'filename', 'children']);
res[1].should.be.instanceOf(Resource).and.have.properties(['url', 'filename', 'children']);
done();
}).catch(done);
});
Expand Down