auto detect tag as company stock code TryGhost#5

core/server/data/finpub/company.js

@@ -0,0 +1,23 @@
+
+const companies = {};
+
+var val = require('./companies.json');
+for (let i = 0; i < val.length; i++) {
+    let c = val[i];
+    companies[c.Code.toLowerCase()] = {code: c.Code, name: c.Name, exchange: c.Exchange, industry: c.IndustryName};
+}
+// free mem
+delete val;
+
+
+const CompanyHandler = {
+    get: function(code) {
+        console.log('check code ', code, companies[code]);
+        if (code) {
+            return companies[code.toLowerCase()];
+        }
+        return null;
+    }
+}
+
+module.exports = CompanyHandler;

core/server/models/tag.js

@@ -1,4 +1,5 @@
 const ghostBookshelf = require('./base');
+const CompanyHelper = require('../data/finpub/company');
 
 let Tag, Tags;
 
@@ -86,6 +87,30 @@ Tag = ghostBookshelf.Model.extend({
             actor_id: actor.id,
             actor_type: actor.type
         };
+    },
+
+    onCreating: function onCreating(model, attr, options) {
+        // overwrite onCreating to automatically add meta_title 
+        // and meta_description
+        console.log('----------------------------> on creating tag ', this.get('name'));  
+        let company = CompanyHelper.get(this.get('name'));
+        if (company) {
+
+            if (!options.importing || (options.importing && !this.get('meta_title'))) {
+                this.set('meta_title', String(company.code + ' - ' + company.name));
+            }
+
+            if (!options.importing || (options.importing && !this.get('meta_description'))) {
+                this.set('meta_description', String(company.name) + ' (' + company.exchange + ')');
+            }
+
+            if (!options.importing || (options.importing && !this.get('description'))) {
+                this.set('description', String(company.name) + ' (' + company.exchange + ')');
+            }
+        }
+
+        // call parent
+        return ghostBookshelf.Model.prototype.onCreating.apply(this, arguments);
     }
 }, {
     orderDefaultOptions: function orderDefaultOptions() {

scripts/crawler.js

@@ -0,0 +1,59 @@
+// const fetch = require("node-fetch");
+const request = require('request');
+const fs = require('fs');
+
+function _fetchCompanies(page, pageSize) {
+    console.log('fetch companies page = ' + page + ' pageSize = ' + pageSize);
+    let url = 'https://finance.vietstock.vn/data/corporateaz',   
+        data = `catID=0&industryID=0&page=${page}&pageSize=${pageSize}&type=0&code=&businessTypeID=0&orderBy=Code&orderDir=ASC`,
+        config = {
+            url: url,
+            method: 'post',
+            body: data,
+            headers: {
+                'Content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
+                'Referrer': 'https://finance.vietstock.vn/doanh-nghiep-a-z?page=3',
+                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
+            },
+            credentials: 'include'
+        };
+    return new Promise(resolve => {
+        request(config, (err, resp, body) => {
+            resolve(JSON.parse(body));
+        })
+
+    }); 
+
+}
+
+function fetchCompanies() {
+    let page = 1,
+        pageSize = 50,
+        result = [];
+
+
+    return new Promise(resolve => {
+        let callback = companies => {
+            if (companies && companies.length > 0) {
+                console.log('fetched ' + companies.length + '. Continue');
+                result = result.concat(companies);
+                page = page + 1;
+                _fetchCompanies(page, pageSize).then(callback);
+            }
+            else {
+                resolve(result);
+            }
+        };
+
+        _fetchCompanies(page, pageSize)
+            .then(callback);        
+    })
+
+}
+
+fetchCompanies().then(companies => {
+    console.log('total companies = ', companies.length);
+    fs.writeFile('../core/server/data/crawled/companies.json', JSON.stringify(companies), err => {
+        if (err) throw err;
+    });
+})