Add init price

functions/config/adayroi.com.js

@@ -1,6 +1,7 @@
 const { regexProcess } = require('../utils/parser/utils')
 const chrome = require('chrome-aws-lambda')
 const puppeteer = require('puppeteer-core')
+const { initDataJajum } = require('../utils/fetch')
 
 let browser = null
 
@@ -61,5 +62,7 @@ module.exports = {
     let { name, description, image } = json
     let priceCurrency = offers.priceCurrency || ''
     return { name, description, currency: priceCurrency, image }
-  }
+  },
+
+  init_data: async params => initDataJajum('adayroi.com', params)
 }

functions/config/shopee.vn.js

@@ -29,5 +29,7 @@ module.exports = {
     let image = `https://cf.shopee.vn/file/${info.images[0]}`
     const { name, description, currency } = info
     return { name, description, currency, image }
-  }
+  },
+
+  init_data: async () => null
 }

functions/config/tiki.vn.js

@@ -1,8 +1,8 @@
 const { URL } = require('url')
 const assert = require('assert')
-
-const { regexProcess, fetchContent } = require('../utils/parser/utils')
 const { JSDOM } = require('jsdom')
+const { regexProcess, fetchContent } = require('../utils/parser/utils')
+const { initDataJajum } = require('../utils/fetch')
 
 const DEBUG = false
 
@@ -79,5 +79,10 @@ module.exports = {
       image
     }
   },
-  format_product_info: json => json
+  format_product_info: json => json,
+
+  init_data: async (params) => {
+    console.log(initDataJajum)
+    return await initDataJajum('tiki.vn', params)
+  }
 }

functions/modules/addUrl.js

@@ -72,10 +72,15 @@ module.exports = httpsFunctions.onRequest(async (req, res) => {
         console.log('Url exists, subscribe email, update info')
         // Subscribe email
         let subRef = urlDoc.collection(collection.SUBSCRIBE).doc(email)
+
+        // TODO: check exist subscribe
         batch.set(subRef, {
             email,
             active: false,
-            create_at: FieldValue.serverTimestamp()
+            create_at: FieldValue.serverTimestamp(),
+            expect_when: 'down',
+            expect_price: 0,
+            methods: 'email'
         }, {
             merge: true
         })
@@ -87,7 +92,9 @@ module.exports = httpsFunctions.onRequest(async (req, res) => {
         data['last_update_at'] = FieldValue.serverTimestamp()
         data['info'] = info
 
-        batch.set(urlDoc, data, { merge: true })
+        batch.set(urlDoc, data, {
+            merge: true
+        })
         batch.commit().then(() => {
             data['is_update'] = true
             return res.json({
@@ -103,66 +110,68 @@ module.exports = httpsFunctions.onRequest(async (req, res) => {
 
     // New url 
     batch.set(urlDoc, {
-            url,
-            domain: domainOf(url),
-            info,
-            number_of_add: 1, // How many time this url is added?
-            raw_count: 0,
-            created_at: FieldValue.serverTimestamp(),
-            last_pull_at: null,
-            add_by: email,
+        url,
+        domain: domainOf(url),
+        info,
+        number_of_add: 1, // How many time this url is added?
+        raw_count: 0,
+        created_at: FieldValue.serverTimestamp(),
+        last_pull_at: null,
+        add_by: email,
+    }, {
+        merge: true
+    })
+
+
+    // Update Metadata
+    let statisticDoc = db.collection(collection.METADATA).doc('statistics')
+    statisticDoc.get().then(doc => {
+        const urlCount = parseInt(doc.get('url_count') || 0) + 1;
+        batch.set(statisticDoc, {
+            url_count: urlCount
         }, {
             merge: true
         })
+    })
+
+    // Subscribe email
+    console.log(`Subscribe ${email}`)
+    batch.set(urlDoc.collection(collection.SUBSCRIBE).doc(email), {
+        email,
+        active: true,
+        expect_when: 'down',
+        expect_price: 0,
+        methods: 'email',
+        create_at: FieldValue.serverTimestamp()
+    }, {
+        merge: true
+    })
+
+    let initData = await initProductDataFromUrl(url)
+    console.log('initData', initData)
+
+    // Fetch the first data
+    if (initData) {
+        initData.map(item => {
+            let rawRef = urlDoc.collection(collection.RAW_DATA).doc()
+            batch.set(rawRef, item)
+        })
+    }
 
-
-            // Update Metadata
-            let statisticDoc = db.collection(collection.METADATA).doc('statistics')
-            statisticDoc.get().then(doc => {
-                const urlCount = parseInt(doc.get('url_count') || 0) + 1;
-                batch.set(statisticDoc, { url_count: urlCount }, { merge: true })
-            })
-
-            // Subscribe email
-            console.log(`Subscribe ${email}`)
-            batch.set(urlDoc.collection(collection.SUBSCRIBE).doc(email), {
-                email,
-                active: true,
-                expect_when: 'down',
-                expect_price: 0,
-                methods: 'email',
-                create_at: FieldValue.serverTimestamp()
-            }, {
-                merge: true
-            })
-
-            let initData = await initProductDataFromUrl(url)
-            console.log('initData', initData)
-
-            // Fetch the first data
-            if (initData) {
-                initData.map(item => {
-                    let rawRef = urlDoc.collection(collection.RAW_DATA).doc()
-                    batch.set(rawRef, item)
-                })
-            }
-
-            batch.commit(() => {
-                console.log(`Init data with ${initData.length} items`)
-            })
-
-            const pullDataUrl = urlFor('pullData', {
-                region: 'asia',
-                url: url,
-                token: ADMIN_TOKEN
-            })
-            fetch(pullDataUrl)
-            console.log(`Fetch the first data ${pullDataUrl}`)
-
-            // Response
-            urlDoc.get().then(snapshot => res.json({
-                id: snapshot.id,
-                ...snapshot.data()
-            }))
+    // Commit and fresh pull
+    await batch.commit()
+    const pullDataUrl = urlFor('pullData', {
+        region: 'asia',
+        url: url,
+        token: ADMIN_TOKEN
+    })
+    fetch(pullDataUrl)
+    console.log(`Fetch the first data ${pullDataUrl}`)
+
+    // Response
+    urlDoc.get().then(snapshot => res.json({
+        id: snapshot.id,
+        ...snapshot.data()
+    }))
 
 })

functions/modules/pullData.js

@@ -5,7 +5,6 @@ const {
   asiaRegion,
   db,
   pullProductDataFromUrl,
-  normalizeUrl,
   documentIdFromHashOrUrl,
   collection,
   validateToken,

functions/utils/fetch.js

@@ -0,0 +1,34 @@
+
+const { fetchContent } = require('../utils/parser/utils')
+const { JSDOM } = require('jsdom')
+
+// tiki.vn sendo.vn adayroi.com lotte.vn lazada.vn
+
+const initDataJajum = async (domain, params) => {
+    const url = `https://jajum.com/products/${domain}/-p${params.product_id}`
+    console.log(`fetch data from ${url}`)
+    let html = await fetchContent(url)
+    if (!html) return null
+    const dom = new JSDOM(html)
+
+    let scripts = dom.window.document.querySelectorAll('script')
+    for (let script of scripts) {
+      let content = script.textContent
+      if (content && content.indexOf('datasets') > -1) {
+        content = content.split(`"data":`)[1].split(`,"borderColor"`)[0]
+        content = JSON.parse(content)
+        console.log(content)
+
+        let raw = content.map(row => {
+          return { datetime: new Date(row.x), price: row.y }
+        })
+
+        return raw
+      }
+    }
+    return null
+}
+
+module.exports = {
+    initDataJajum
+}

functions/utils/index.js

@@ -17,28 +17,14 @@ const {
   collection
 } = require('./constants')
 
-const {
-  getConfig,
-  getSortKey,
-  functionsUrl,
-  functionsUrlAsia,
-  hostingUrl,
-  IS_PROD
-} = require('./config')
+const configUtils = require('./config')
 
-const {
-  getDeepLink
-} = require('./accesstrade')
+const accessTradeUtils = require('./accesstrade')
+const formaterUtils = require('./formater')
+const { normalizeUrl } = require('./formater')
+const fetchUtils = require('./fetch')
 
-const {
-  normalizeUrl,
-  hash,
-  formatPrice,
-  cleanEmail,
-  pullProductDataFromUrl,
-  urlFor,
-  redashFormat
-} = require('./formater')
+console.log('xxxxxx fetchUtils', fetchUtils)
 
 // Setting DB
 admin.initializeApp(functions.config().firebase)
@@ -149,24 +135,14 @@ module.exports = {
   querystring,
 
   // Accesstrade deeplink
-  getDeepLink,
+  ...accessTradeUtils,
 
   // Get Firebase Functions env config
-  getConfig,
-  functionsUrl,
-  functionsUrlAsia,
-  hostingUrl,
-  IS_PROD,
+  ...configUtils,
 
   // Formater
-  normalizeUrl,
-  hash,
-  formatPrice,
-  cleanEmail,
-  pullProductDataFromUrl,
-  urlFor,
-  redashFormat,
-
+  ...formaterUtils,
+
   resError,
   getUserFromToken,
 
@@ -203,11 +179,12 @@ module.exports = {
    * @return {bool}
    */
   validateToken: token => {
-    const adminToken = getConfig('admin_token')
+    const adminToken = configUtils.getConfig('admin_token')
     return token && adminToken === token
   },
 
-  getSortKey,
   verifyUserTokenId,
-  fetchRetry
+  fetchRetry,
+
+  ...fetchUtils
 }