modify to adapt github's new html

oGsLP · May 31, 2019 · 8eae6fc · 8eae6fc
1 parent 646e432
commit 8eae6fc
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 14 deletions.
diff --git a/lib/crawler/followCrawler.js b/lib/crawler/followCrawler.js
@@ -44,7 +44,7 @@ module.exports = function(username) {
 
 function crawlFollow(array, data) {
   let $ = cheerio.load(data.toString());
-  $(".col-9 >div")
+  $(".col-lg-9 >div")
     .filter(".position-relative")
     .children()
     .each((i, el) => {

diff --git a/lib/crawler/reposCrawler.js b/lib/crawler/reposCrawler.js
@@ -10,7 +10,7 @@ module.exports = function(username) {
     .then(response => {
       let html = response.data.toString(); // 获取网页内容
       const $ = cheerio.load(html); // 传入页面内容
-      $("#user-repositories-list li >.col-9").each(function() {
+      $("#user-repositories-list li >.col-lg-9").each(function() {
         let repo = {};
         repo.repoName = $(this)
           .find("h3 >a")

diff --git a/lib/crawler/userCrawler.js b/lib/crawler/userCrawler.js
@@ -10,8 +10,8 @@ module.exports = function(username) {
       let html = response.data.toString();
       const $ = cheerio.load(html);
       // Profile
-      let $profile = $(".h-card >div");
-      let $status = $($profile).filter(".user-status-container");
+      let $profile = $(".h-card >div").filter(".clearfix");
+      let $status = $($profile).find(".user-status-container");
       let [stat_emoji, stat_desc] = [
         $($status)
           .find("g-emoji")
@@ -24,7 +24,9 @@ module.exports = function(username) {
           .text()
           .trim()
       ];
+      stat_emoji = stat_emoji.substr(0, stat_emoji.length / 2);
       let $names = $($profile)
+        .find(".vcard-names-container")
         .filter(".vcard-names-container")
         .find("h1")
         .children();
@@ -37,7 +39,7 @@ module.exports = function(username) {
           .trim()
       ];
 
-      let $infos = $($profile).filter(".js-profile-editable-area");
+      let $infos = $($profile).find(".js-profile-editable-area");
       let description = $($infos)
         .find(".user-profile-bio>div")
         .text()

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "git-info-crawler",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "Crawl info from one git account",
   "homepage": "",
   "author": {

diff --git a/test.js b/test.js
@@ -15,8 +15,8 @@
 // const crawler = require("./../index");
 // crawler.crawlRepos("oGsLP");
 
-// const userCrawler = require("./lib/crawler/userCrawler");
-// userCrawler("NJUSSJ").then(data => console.log(data));
+const userCrawler = require("./lib/crawler/userCrawler");
+userCrawler("NJUSSJ").then(data => console.log(data));
 //
 // const emojiCrawler = require("./lib/crawler/emojiCrawler");
 //
@@ -26,6 +26,6 @@
 // const dbUtil = require("./lib/util/MongoDBUtil");
 // dbUtil.init();
 
-const crawlFollow = require("./lib/crawler/followCrawler");
-const printFollow = require("./lib/printer/followPrinter");
-crawlFollow("oGsLP").then(res => printFollow(res));
+// const crawlFollow = require("./lib/crawler/followCrawler");
+// const printFollow = require("./lib/printer/followPrinter");
+// crawlFollow("oGsLP").then(res => printFollow(res));
diff --git a/test/crawler/reposCrawler.test.js b/test/crawler/reposCrawler.test.js
@@ -9,10 +9,10 @@ describe("#reposCrawler", () => {
     return crawlRepos("oGsLP").then(data => {
       assert.strictEqual(data.username, "oGsLP");
       assert.deepStrictEqual(data.repos.pop(), {
-        repoName: "HotelSystem",
-        desc: "软工第二次大作业:酒店管理系统",
+        repoName: "Software-Engineering-Team-Work2",
+        desc: "",
         lang: "Java",
-        updated: "Nov 20, 2016"
+        updated: "Dec 21, 2017"
       });
     });
   }, 10000);