Skip to content

Commit

Permalink
replace urlib-sync with urllib
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmrasf
dmrasf committed Dec 6, 2020
1 parent d4cebf7 commit 1240bf3
Show file tree
Hide file tree
Showing 5 changed files with 1,751 additions and 1,236 deletions.
177 changes: 88 additions & 89 deletions lib/books-generator.js
View file
@@ -1,6 +1,6 @@
'use strict';

var request = require('urllib-sync').request;
var urllib = require('urllib')
var ejs = require('ejs');
var xpath = require('xpath');
var path = require('path');
Expand All @@ -14,91 +14,87 @@ var log = require('hexo-log')({
silent: false
});

function resolv(url, timeout, headers) {

var response = '';
try {
response = request(url, {
function resolv(url, timeout) {
return new Promise((resolve, reject) => {
var response = '';
urllib.request(url, {
method: 'GET',
timeout: timeout,
dataType: 'xml',
headers: headers,
});
} catch (err) {
offline = true;
}

if (offline) {
return {
list: [],
next: ""
};
}

if(headers['Cookie'] instanceof Array && headers['Cookie'].length===0){
headers['Cookie'] = response.headers['set-cookie']
}

var doc = new Dom({
errorHandler: {
warning: function (e) {
},

error: function (e) {
},

fatalError: function (e) {
headers: {
referer: 'https://book.douban.com/chart',
host: 'book.douban.com',
"user-agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
}
}).then(result => {
response = result.res;
var doc = new Dom({
errorHandler: {
warning: function (e) {
},

error: function (e) {
},

fatalError: function (e) {
}
}
}).parseFromString(response.data.toString());
var items = xpath.select('//ul[@class="interest-list"]/li[@class="subject-item"]', doc);

var next = xpath.select('string(//span[@class="next"]/a/@href)', doc);
if (next.startsWith("/")) {
next = "https://book.douban.com" + next;
}
}
}).parseFromString(response.data.toString());


var items = xpath.select('//ul[@class="interest-list"]/li[@class="subject-item"]', doc);

var next = xpath.select('string(//span[@class="next"]/a/@href)', doc);
if (next.startsWith("/")) {
next = "https://book.douban.com" + next;
}
var list = [];
for (var i in items) {
var parser = new Dom().parseFromString(items[i].toString());
var title = xpath.select1('string(//div[@class="info"]/h2/a/@title)', parser);
var alt = xpath.select1('string(//div[@class="info"]/h2/a/@href)', parser);
var image = xpath.select1('string(//div[@class="pic"]/a/img/@src)', parser);

var pub = xpath.select1('string(//div[@class="pub"])', parser);

var updated = xpath.select1('string(//span[@class="date"])', parser);

var tags = xpath.select1('string(//span[@class="tags"])', parser);
tags = tags ? tags.substr(3) : '';

var recommend = xpath.select1('string(//div[@class="short-note"]/div/span[contains(@class,"rating")]/@class)', parser);
recommend = renderStar(recommend.substr(6, 1));
var comment = xpath.select1('string(//p[@class="comment"])', parser);
comment = comment ? comment : '';

//image = 'https://images.weserv.nl/?url=' + image.substr(8, image.length - 8) + '&w=100';

list.push({
title: title,
alt: alt,
image: image,
pub:pub,
updated:updated,
tags: tags,
recommend: recommend,
comment: comment
});
}

var list = [];
for (var i in items) {
var parser = new Dom().parseFromString(items[i].toString());
var title = xpath.select1('string(//div[@class="info"]/h2/a/@title)', parser);
var alt = xpath.select1('string(//div[@class="info"]/h2/a/@href)', parser);
var image = xpath.select1('string(//div[@class="pic"]/a/img/@src)', parser);

var pub = xpath.select1('string(//div[@class="pub"])', parser);

var updated = xpath.select1('string(//span[@class="date"])', parser);

var tags = xpath.select1('string(//span[@class="tags"])', parser);
tags = tags ? tags.substr(3) : '';

var recommend = xpath.select1('string(//div[@class="short-note"]/div/span[contains(@class,"rating")]/@class)', parser);
recommend = renderStar(recommend.substr(6, 1));
var comment = xpath.select1('string(//p[@class="comment"])', parser);
comment = comment ? comment : '';

//image = 'https://images.weserv.nl/?url=' + image.substr(8, image.length - 8) + '&w=100';

list.push({
title: title,
alt: alt,
image: image,
pub:pub,
updated:updated,
tags: tags,
recommend: recommend,
comment: comment
resolve({
list: list,
next: next
});

}).catch(_ => {
offline = 1;
resolve({
list: [],
next: ""
});
});
}

return {
'list': list,
'next': next
};
});
}

module.exports = function (locals) {
module.exports = async function (locals) {

var config = this.config;
if (!config.douban || !config.douban.book) {//当没有输入book信息时，不进行数据渲染。
Expand Down Expand Up @@ -126,26 +122,29 @@ module.exports = function (locals) {
var wishUrl = 'https://book.douban.com/people/' + config.douban.user + '/wish';

for (var nextWish = wishUrl; nextWish;) {
var resWish = resolv(nextWish, timeout, headers);
nextWish = resWish.next;
wish = wish.concat(resWish.list);
await resolv(nextWish, timeout).then(resWish => {
nextWish = resWish.next;
wish = wish.concat(resWish.list);
});
}

var readingUrl = 'https://book.douban.com/people/' + config.douban.user + '/do';

for (var nextreading = readingUrl; nextreading;) {
var resreading = resolv(nextreading, timeout, headers);
nextreading = resreading.next;
reading = reading.concat(resreading.list);
await resolv(nextreading, timeout).then(resreading => {
nextreading = resreading.next;
reading = reading.concat(resreading.list);
});
}


var readUrl = 'https://book.douban.com/people/' + config.douban.user + '/collect';

for (var nextread = readUrl; nextread;) {
var resread = resolv(nextread, timeout, headers);
nextread = resread.next;
read = read.concat(resread.list);
await resolv(nextread, timeout).then(resread => {
nextread = resread.next;
read = read.concat(resread.list);
});
}

var endTime = new Date().getTime();
Expand Down

0 comments on commit 1240bf3

Please sign in to comment.