init commit

.gitignore

@@ -0,0 +1,5 @@
+node_modules
+.idea/
+.vscode/
+v8-compile-cache-0/
+utils/test_env.js

app.js

@@ -0,0 +1,45 @@
+var express = require('express');
+var path = require('path');
+var logger = require('morgan');
+var cookieParser = require('cookie-parser');
+var bodyParser = require('body-parser');
+
+
+var app = express();
+app.get('env') === 'production' || require('./utils/test_env');
+
+// require('./controller/twitter-like-fetch');
+require('./controller/pic-downloader');
+
+app.get('env') === 'production' ? app.disable('x-powered-by') : false;
+// view engine setup
+// app.set('views', path.join(__dirname, 'views'));
+// app.set('view engine', 'ejs');
+app.get('env') !== 'production' ? (app.use(logger('dev'))) : false;
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({extended: false}));
+app.use(cookieParser());
+// app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', require('./routes/index'));
+// app.use('/github', require('./routes/github'));
+// app.use('/bangumi', require('./routes/bangumi'));
+
+// catch 404 and forward to error handler
+app.use(function (req, res, next) {
+  var err = new Error('Not Found');
+  err.status = 404;
+  next(err);
+});
+
+// error handler
+app.use(function (err, req, res, next) {
+  err.status != 404 ? console.error(err) : console.error(req.originalUrl);
+  var errorMessage = { message: err.message };
+
+  res.status(err.status || 500);
+  req.app.get('env') !== 'production' ? (errorMessage.stack = err.stack) : false;
+  res.json(errorMessage);
+});
+
+module.exports = app;

bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('contribution:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '4000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

controller/bangumi-cache.js

@@ -0,0 +1,101 @@
+/**
+ * Created by xingo on 2017/08/10 .
+ */
+
+let errTimes = 0, MAX_ERR_TIMES = 10;
+let { currentUser } = require('../utils/environment.js');
+let taskMap = {};
+let getTask = (func) => {
+  return {
+    errTimes: 0,
+    func,
+    cycleTask: undefined
+  }
+};
+
+let { getBangumiTimelineJSON } = require('../controller/bangumi-contribution.js');
+function getBangumiContribution(username) {
+  return getBangumiTimelineJSON(username).then(json => {
+    exportModule.cache[username] || (exportModule.cache[username] = {});
+    return exportModule.cache[username].contribution = json;
+  });
+}
+
+taskMap.getBangumiContribution = getTask(getBangumiContribution);
+
+
+let { getRss: getBangumiRssJson } = require('../controller/bangumi-rss.js');
+function getBangumiRss(username) {
+  return getBangumiRssJson(username).then(json => {
+    exportModule.cache[username] || (exportModule.cache[username] = {});
+    return exportModule.cache[username].timeline = json;
+  });
+}
+
+taskMap.getBangumiRss = getTask(getBangumiRss);
+
+let taskArray = [];
+
+for (let taskName in taskMap) {
+  let task = taskMap[taskName];
+  let cycleTask = function (username) {
+    task.func(username).then(result => {
+      task.errTimes = 0;
+      return result;
+    }).catch(err => {
+      console.error(err);
+      if (task.errTimes >= MAX_ERR_TIMES) {
+        return;
+      }
+      task.errTimes++;
+      return cycleTask(username);
+    });
+  }
+  task.cycleTask = cycleTask;
+}
+
+let runAllCycleTaskFunc = (username) => {
+  for (let taskName in taskMap) {
+    taskMap[taskName].cycleTask(username);
+  }  
+}
+
+runAllCycleTaskFunc(currentUser);
+
+
+// function getBangumiContributionTask(username) {
+//   return getBangumiContribution(username).catch(err => {
+//     console.error(err);
+//     return getBangumiContributionTask(username);
+//   });
+// }
+
+// getBangumiContributionTask(currentUser);
+
+// function getBangumiRssTask(username) {
+//   return getBangumiRss(username).catch(err => {
+//     console.error(err);
+//     return getBangumiRssTask(username);
+//   });
+// }
+
+// getBangumiRssTask(currentUser);
+
+let moment = require('moment');
+function runCycleTask() {
+  setTimeout(() => {
+    exportModule.cache = {};
+    runAllCycleTaskFunc(currentUser);
+    runCycleTask();
+    exportModule.cache.time = moment().format('HH:mm:ss');
+  }, moment().endOf('day').add(1, 'hours').valueOf() - Date.now());
+}
+
+runCycleTask();
+
+let exportModule = {
+  cache: {},
+  taskMap
+};
+
+module.exports = exportModule;

controller/bangumi-contribution.js

@@ -0,0 +1,135 @@
+/**
+ * Created by xingo on 2017/08/10 .
+ */
+
+let httpClient = require('request');
+let domEnv = require('jsdom').env,
+    jQInit = require('jquery');
+
+function fetchTimeline(username, page) {
+  return new Promise((resolve, reject) => {
+    httpClient.get({
+      url: `https://bgm.tv/user/${username}/timeline?type=all&page=${page}&ajax=1`,
+      headers: {
+        'User-Agent': `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36`,
+        'Upgrade-Insecure-Requests': 1,
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,ja;q=0.2,zh-TW;q=0.2',
+        'Referer': `https://bgm.tv/user/${username}/timeline`,
+      }
+    }, (err, response, body) => {
+      if (err)
+        return reject(err);
+      resolve(body)
+    })
+  });
+}
+
+function fetchWindow(username) {
+  return new Promise((resolve, reject) => {
+    httpClient.get({
+      url: `https://bgm.tv/user/${username}/timeline`,
+      headers: {
+        'User-Agent': `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36`,
+        'Upgrade-Insecure-Requests': 1,
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,ja;q=0.2,zh-TW;q=0.2',
+      }
+    }, (err, response, body) => {
+      if (err)
+        return reject(err);
+      resolve(body)
+    })
+  }).then(body => {
+    return new Promise((resolve, reject) => {
+      domEnv(body, (errors, window) => {
+        if (errors)
+          return reject(errors);
+        let $ = jQInit(window);
+
+        resolve($);
+      })
+    })
+  })
+}
+
+
+let moment = require('moment');
+
+function setDate($, body, dateObj) {
+  let timeline = $(body);
+  let shouldStop = false;
+  timeline.children('h4.Header').each(function (index) {
+    let date = $(this).text();
+    switch (date) {
+      case '今天':
+        date = moment().format('YYYY-MM-DD');
+        break;
+      case '昨天':
+        date = moment().subtract(1, 'days').format('YYYY-MM-DD');
+        break;
+      default:
+        date = moment(date, 'YYYY-M-D').format('YYYY-MM-DD');
+        break;
+    }
+
+    if (moment().diff(moment(date), 'days') > 371) {
+      shouldStop = true;
+      return false;
+    }
+    if (dateObj[date])
+      dateObj[date] = dateObj[date] + $(this).next().children('li').length;
+    else
+      dateObj[date] = $(this).next().children('li').length;
+  });
+  return shouldStop;
+}
+
+function fetchTimelinePromise(username, page, jquery, dateObj) {
+  return fetchTimeline(username, page).then(body => {
+    if (setDate(jquery, body, dateObj)) {
+      return dateObj;
+    }
+    else {
+      return fetchTimelinePromise(username, page + 1, jquery, dateObj);
+    }
+  })
+}
+
+function getBangumiTimelineJSON(username) {
+  return fetchWindow(username).then(jquery => {
+    let page = 1;
+    let dateObj = {};
+
+    return fetchTimelinePromise(username, page, jquery, dateObj);
+  }).then(dateObj => {
+    let day = 0;
+    let json = {
+      max: 0,
+      data: [],
+    };
+    let momentObj = moment().subtract(day, 'days');
+    while (!(momentObj.day() == 6 && day >= 365)) {
+      let date = momentObj.format('YYYY-MM-DD');
+      let count = dateObj[date];
+      count = count ? count : 0;
+      json.data.unshift({
+        count: count ? count : 0,
+        date: date,
+      });
+
+      if (count > json.max)
+        json.max = count;
+
+      momentObj = moment().subtract(++day, 'days');
+    }
+    return json;
+  })
+}
+
+
+module.exports = {
+  getBangumiTimelineJSON
+}