Merge pull request #28 from crawlab-team/develop

fixed crawlab-team/artipub#27
rpennacchio3mj5y · Mar 1, 2022 · 44997cb · 44997cb
2 parents ced7ed7 + 28196c9
commit 44997cb
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 66 deletions.
diff --git a/lib/BaseExecutor.js b/lib/BaseExecutor.js
@@ -3,50 +3,32 @@ const models = require('../models')
 const constants = require('../constants')
 
 class BaseExecutor {
-    constructor(task) {
-        this.task = task
-        this.platform = undefined
-        this.spider = undefined
-    }
+  constructor(task) {
+    this.task = task
+    this.platform = undefined
+    this.spider = undefined
+  }
 
-    async init() {
-        const task = this.task
+  async init() {
+    const task = this.task
 
-        // 平台
-        this.platform = await models.Platform.findOne({ _id: task.platformId })
-        const spiderName = this.platform.name
+    // 平台
+    this.platform = await models.Platform.findOne({ _id: task.platformId })
+    const spiderName = this.platform.name
 
-        let spider
-        if (spiderName === constants.platform.JUEJIN) {
-            spider = new spiders.JuejinSpider(task._id)
-        } else if (spiderName === constants.platform.SEGMENTFAULT) {
-            spider = new spiders.SegmentfaultSpider(task._id)
-        } else if (spiderName === constants.platform.JIANSHU) {
-            spider = new spiders.JianshuSpider(task._id)
-        } else if (spiderName === constants.platform.CSDN) {
-            spider = new spiders.CsdnSpider(task._id)
-        } else if (spiderName === constants.platform.ZHIHU) {
-            spider = new spiders.ZhihuSpider(task._id)
-        } else if (spiderName === constants.platform.OSCHINA) {
-          spider = new spiders.OschinaSpider(task._id)
-        } else if (spiderName === constants.platform.TOUTIAO) {
-          spider = new spiders.ToutiaoSpider(task._id)
-        } else if (spiderName === constants.platform.CNBLOGS) {
-          spider = new spiders.CnblogsSpider(task._id)
-        } else if (spiderName === constants.platform.V2EX) {
-          spider = new spiders.V2exSpider(task._id)
-        }
-        this.spider = spider
-    }
+    const Spider = require(`../spiders/${spiderName}`)
+    console.log(Spider)
+    this.spider = new Spider(task._id)
+  }
 
-    async run() {
-        // to be inherited
-    }
+  async run() {
+    // to be inherited
+  }
 
-    async start() {
-        await this.init()
-        await this.run()
-    }
+  async start() {
+    await this.init()
+    await this.run()
+  }
 }
 
 module.exports = BaseExecutor
diff --git a/routes/platform.js b/routes/platform.js
@@ -167,7 +167,11 @@ module.exports = {
       const platform = platforms[i]
       const Spider = require(`../spiders/${platform.name}`)
       const spider = new Spider(null, platform._id.toString())
-      await spider.checkCookieStatus()
+      try {
+        await spider.checkCookieStatus()
+      } catch (e) {
+        console.error(e)
+      }
     }
     await res.json({
       status: 'ok'

diff --git a/spiders/jianshu.js b/spiders/jianshu.js
@@ -2,30 +2,32 @@ const constants = require('../constants')
 const BaseSpider = require('./base')
 
 class JianshuSpider extends BaseSpider {
-    async afterGoToEditor() {
-        await this.page.evaluate(() => {
-            document.querySelectorAll('span').forEach(el => {
-                if (el.textContent.trim() === '新建文章') {
-                    el.click()
-                }
-            })
+  async afterGoToEditor() {
+    await this.page.evaluate(() => {
+      document.querySelectorAll('span')
+        .forEach(el => {
+          if (el.textContent.trim() === '新建文章') {
+            el.click()
+          }
         })
-        await this.page.waitFor(3000)
-    }
+    })
+    await this.page.waitFor(5000)
+  }
 
-    async afterInputEditor() {
-    }
+  async afterInputEditor() {
+  }
 
-    async afterPublish() {
-        // this.task.url = this.page.url()
-        this.task.updateTs = new Date()
-        this.task.status = constants.status.FINISHED
-        await this.article.save()
-    }
+  async afterPublish() {
+    this.task.url = this.page.url()
+    if (!this.task.url.match(/\/p\/\w+/)) return
+    this.task.updateTs = new Date()
+    this.task.status = constants.status.FINISHED
+    await this.article.save()
+  }
 
-    async fetchStats() {
-        // TODO: implement this method
-    }
+  async fetchStats() {
+    // TODO: implement this method
+  }
 }
 
 module.exports = JianshuSpider
diff --git a/spiders/v2ex.js b/spiders/v2ex.js
@@ -1,5 +1,3 @@
-const fs = require('fs')
-const path = require('path')
 const BaseSpider = require('./base')
 const constants = require('../constants')
 

diff --git a/spiders/wechat.js b/spiders/wechat.js
@@ -0,0 +1,30 @@
+const BaseSpider = require('./base')
+const constants = require('../constants')
+
+class WechatSpider extends BaseSpider {
+  async afterInputEditor() {
+  }
+
+  async publish() {
+    // 发布文章
+    // const elPub = await this.page.$(this.editorSel.publish)
+    // await elPub.click()
+    // await this.page.waitFor(20000)
+
+    // 后续处理
+    await this.afterPublish()
+  }
+
+  async afterPublish() {
+    // this.task.url = await this.page.url().replace('#reply0', '')
+    // if (!this.task.url.match(/\/t\/\d+/)) return
+    // this.task.updateTs = new Date()
+    // this.task.status = constants.status.FINISHED
+    // await this.task.save()
+  }
+
+  async fetchStats() {
+  }
+}
+
+module.exports = WechatSpider
diff --git a/spiders/zhihu.js b/spiders/zhihu.js
@@ -72,10 +72,14 @@ class ZhihuSpider extends BaseSpider {
 
   async publish() {
     // 发布文章
-    await this.page.evaluate(() => {
-      const el = document.querySelector('.PublishPanel-stepTwoButton')
-      el.click()
-    })
+    try {
+      await this.page.evaluate(() => {
+        const el = document.querySelector('.PublishPanel-stepTwoButton')
+        el.click()
+      })
+    } catch (e) {
+      // do nothing
+    }
     await this.page.waitFor(5000)
 
     // 后续处理