README + unnecessary fields removed

README.md

@@ -0,0 +1,77 @@
+# First Progressive Web App (PWA) for BigCommerce
+This projects bring You the BigCommerce support as a backend platform for [Vue Storefront - first Progressive Web App for e-Commerce](https://github.com/DivanteLtd/vue-storefront). 
+
+Vue Storefront is a standalone PWA storefront for your eCommerce, possible to connect with any eCommerce backend (eg. Magento, Pimcore, WooCommerce, BigCommerce or Shopware) through the API.
+
+[![See how it works!](/DivanteLtd/vue-storefront/raw/master/doc/media/Fil-Rakowski-VS-Demo-Youtube.png)](https://www.youtube.com/watch?v=L4K-mq9JoaQ)
+
+Sign up for a demo at https://vuestorefront.io/.
+
+# BigCommerce data bridge
+Vue Storefront is platform agnostic - which mean: it can be connected to virtually any eCommerce CMS. This project is a data connector for *BigCommerce eCommerce Framework*.
+
+This integration is currently at **Proof of Concept** stage and it's not ready for production deplyoment. 
+
+Ready made features:
+- Simple products support.
+- Configurable product support,
+- Media import,
+- Configurable options,
+- Product Variants
+
+TODO:
+- Custom product options,
+- Customer account,
+- Checkout + order,
+- Shopping cart sync,
+- Add on-demand indexation based on BigCommerce web-hooks,
+- Add delta-main indexing scheme indexing records modified after ...
+
+## Installation guide
+
+First, please do install Vue Storefront. Here You can find the [official installation guide](https://divanteltd.github.io/vue-storefront/guide/installation/linux-mac.html). 
+
+Requirements:
+- Node 10,
+- Yarn package manager
+
+Then please execute the following steps:
+```bash
+git clone https://github.com/DivanteLtd/bigcommerce2vuestorefront.git
+cd bigcommerce2vuestorefront
+yarn install
+```
+
+Please do setup the BigCommerce [API credentials](https://developer.bigcommerce.com/api/#api-documentation) by editing `config.js`:
+
+```js
+module.exports = Object.freeze({
+  bc: {
+    clientId: rocess.env.BC_API_CLIENT_ID || 'atiyjoyxaq65lfjyrriu4q10m0599yn',
+    secret: rocess.env.BC_API_SECRET || 'bwfdv6glwb72nhgpqd4nnikpdr9jiiv',
+    accessToken: rocess.env.BC_API_ACCESS_TOKEN || 'mjhbzys8zcwjjdf3jjzsm57bt0w55ot',
+    storeHash: rocess.env.BC_API_STORE_HASH ||'txjxffgep6',
+    responseType: 'json',
+    apiVersion: 'v3'
+  },
+  db: {
+    driver: 'elasticsearch',
+    url: process.env.DATABASE_URL || 'http://localhost:9200',
+    indexName:  process.env.INDEX_NAME || 'vue_storefront_catalog'
+  }
+})
+```
+
+You can use the ENV variables instead:
+
+```bash
+export BC_API_CLIENT_ID=atiyjoyxaq65lfjyrriu4q10m0599yn
+export BC_API_SECRET=bwfdv6glwb72nhgpqd4nnikpdr9jiiv
+export BC_API_ACCESS_TOKEN=mjhbzys8zcwjjdf3jjzsm57bt0w55ot
+export BC_API_STORE_HASH=txjxffgep6
+node cli.js products
+node cli.js categories
+```
+
+**Important note:** please do test this data bridge with Vue Storefront 1.6.
+

src/common/removeQueryString.js

@@ -0,0 +1,13 @@
+const url = require('url');
+
+const removeQueryString = (sourceUrl) => {
+    // split url into distinct parts
+    // (full list: https://nodejs.org/api/url.html#url_url_parse_urlstr_parsequerystring_slashesdenotehost)
+    var obj = url.parse(sourceUrl);
+    // remove the querystring
+    obj.search = obj.query = "";
+    // reassemble the url
+    return url.format(obj);
+  }
+
+module.exports = removeQueryString

src/importer/products.js

@@ -3,14 +3,23 @@ const sendToElastic = require('../common/sendToElastic.js')
 
 
 const importer = ({ config, elasticClient, apiConnector }) => {
-  apiConnector(config.bc).get('/catalog/products?include=variants,images&limit=1000').then(
+  const baseUrl = '/catalog/products?include=variants,images'
+
+  const fetchApi = (url) => {
+    return apiConnector(config.bc).get(url).then(
     (result) => {
       for (let product of result.data) {
           productTemplate.fill(product, { apiConnector, elasticClient, config }).then(converted => {
             sendToElastic(converted, 'product', { config, elasticClient })
           })
         }
+        if (result.meta.pagination.links.next) {
+          console.log('Switching the page to ' + result.meta.pagination.links.next)
+          fetchApi(baseUrl + result.meta.pagination.links.next)
+        }
     })
+  }
+  fetchApi(baseUrl).catch(err => console.error(err))
 
   function importProducts() {
     console.log('produts are being imported...')

src/templates/category.js

@@ -1,4 +1,5 @@
 const config = require('../../config')
+const removeQueryString = require('../common/removeQueryString')
 
 const extractSubcategories = async (parent_id, apiConnector) => {
 
@@ -8,16 +9,12 @@ const extractSubcategories = async (parent_id, apiConnector) => {
     for (let child of parsed) {
 
       let childData = {
-        "entity_type_id": 3,
-        "attribute_set_id": 0,
         "parent_id": parent_id,
-        "created_at": "2018-10-12",
-        "updated_at": "2018-10-12",
-        "position": 0,
-        "children_count": 1,
-        "available_sort_by": null,
+        "position": child.sort_order,
+        "children_count": 1,// TODO: update children count
         "include_in_menu": 1,
         "name": child.name,
+        "url_key": child.custom_url ? removeQueryString(child.custom_url.url) : '',
         "id": child.id,
         "children_data": child.id !== parent_id && await extractSubcategories(child.id, apiConnector)
       }
@@ -36,33 +33,28 @@ const fill = async (source, { apiConnector, elasticClient }) => {
   let {
     id,
     description,
+    is_visible,
     name,
-    parent,
-    slug,
+    parent_id,
+    custom_url,
+    sort_order
 
   } = source
-
-  console.log(slug)
   let children = await extractSubcategories(parseInt(id), apiConnector)
   let output = {
-    "entity_type_id": 3,
-    "attribute_set_id": 0,
-    "parent_id": parent,
-    "created_at": "2018-10-12",
-    "updated_at": "2018-10-12",
-    "is_active": true,
-    "position": 0,
-    "level": 2,
+    "parent_id": parent_id,
+    "is_active": is_visible,
+    "position": sort_order,
+    "level": 2, // level 1 = root category
     "children_count": children.length,
-    "product_count": 1,
+    "product_count": 1, // TODO: update this value properly
     "available_sort_by": null,
     "include_in_menu": 1,
     "name": name,
     "id": id,
     "children_data": children,
-    "is_anchor": "1",
     "path": `1/${id}`,
-    "url_key": slug
+    "url_key": removeQueryString(custom_url.url)
   };
 
   /*let childrenData =

src/templates/product.js

@@ -1,14 +1,11 @@
 const moment = require('moment')
-const url = require('url');
+const removeQueryString = require('../common/removeQueryString')
 
-const removeQueryString = (sourceUrl) => {
-  // split url into distinct parts
-  // (full list: https://nodejs.org/api/url.html#url_url_parse_urlstr_parsequerystring_slashesdenotehost)
-  var obj = url.parse(sourceUrl);
-  // remove the querystring
-  obj.search = obj.query = "";
-  // reassemble the url
-  return url.format(obj);
+const trim = (s, t) => {
+  var tr, sr
+  tr = t.split('').map(e => `\\\\${e}`).join('')
+  sr = s.replace(new RegExp(`^[${tr}]+|[${tr}]+$`, 'g'), '')
+  return sr
 }
 
 const extractCategories = (categories, apiConnector) => {
@@ -32,73 +29,64 @@ const customFields = async (product, { apiConnector, elasticClient, config }) =>
   return apiConnector(config.bc).get('/catalog/products/' + product.id + '/custom-fields').catch(err => console.error(err))
 }
 const fill =  (source, { apiConnector, elasticClient, config }) =>  new Promise( (resolve, reject) => {
+    const filter_options = {}
     let output = {
       "category_ids": source.categories,
-      "entity_type_id": 4,
-      "attribute_set_id": 11,
       "type_id": "simple", // TODO: add othe product types
       "sku": source.sku,
       "has_options": source.options && source.options.length > 0, // todo
       "required_options": 0,
-      "created_at": moment(source.created_at).toJSON(),
-      "updated_at": moment(source.updated_at).toJSON(),
+//      "created_at": moment(source.created_at).toJSON(),
+//      "updated_at": moment(source.updated_at).toJSON(),
       "status": 1,
-      "accessories_size": null,
       "visibility": source.is_visible ? 4 : 0,
       "tax_class_id": source.tax_class_id,
-      "is_recurring": false,
       "description": source.description,
-      "meta_keyword": null,
-      "short_description": "",
       "name": source.name,
-      "meta_title": null,
       "image": source.images ? removeQueryString(source.images[0].url_standard) : '',
-      "meta_description": null,
       "thumbnail": source.images ? removeQueryString(source.images[0].url_thumbnail) : '',
       "media_gallery": source.images ? source.images.map(si => {  return { image: removeQueryString(si.url_standard) } }) : null,
-      "url_key": source.custom_url.url,
-      "country_of_manufacture": null,
-      "url_path": source.custom_url.url,
-      "image_label": null,
-      "small_image_label": null,
-      "thumbnail_label": null,
-      "gift_wrapping_price": null,
+      "url_key": trim(source.custom_url.url, '/'),
+      "url_path": trim(source.custom_url.url, '/'),
       "weight": source.weight,
       "price": source.price,
       "special_price": null,
-      "msrp": null,
       "news_from_date": null,
       "news_to_date": null,
       "special_from_date": null,
       "special_to_date": null,
-      "is_salable": true,
       "stock_item": {
         "is_in_stock": source.inventory_tracking === 'none' ? true : source.inventory_level > 0
       },
       "id": source.id,
       "category": extractCategories(source.categories),
-      "stock":{
+      "stock":{ // TODO: Add stock quantity - real numbers
         "is_in_stock": true
       },
       "configurable_children": source.variants ? source.variants.map(sourceVariant => {
         let child = {
           "sku": sourceVariant.sku,
-          "price": sourceVariant.price,
+          "price": sourceVariant.price ? sourceVariant.price : source.price,
           "image": removeQueryString(sourceVariant.image_url),
           "is_salable": !sourceVariant.purchasing_disabled,
           "product_id": source.id
         }
         sourceVariant.option_values.map((ov) => {
+          if (!filter_options[ov.option_display_name + '_options']) filter_options[ov.option_display_name + '_options'] = new Set() // we need to aggregate the options from child items
+          filter_options[ov.option_display_name + '_options'].add(ov.label)
+          child[ov.option_display_name] = ov.label
           child['prodopt-' + ov.option_id] = ov.id // our convention is to store the product options as a attributes with the names = prodopt-{{option_id}}
         })
         return child
       }) : null
     }
+    for (let key in filter_options) {
+      output[key] = Array.from(filter_options[key])
+    }
     const subPromises = []
     subPromises.push(options(source, { apiConnector, elasticClient, config }).then(productOptions => {
       if (productOptions && productOptions.data.length >0) {
         output.type_id = "configurable"
-        console.log(productOptions)
         output.configurable_options = productOptions.data.map(po => {
           return { // TODO: we need to populate product's : product.color_options and product.size_options to make forntend filters work properly
             id: po.id,// TODO: LETS STORE THE ATTRIBUTES DICTIONARY JUST FOR attr config / type - we don't need the available options (which is risky updating Elastic)
@@ -130,6 +118,7 @@ const fill =  (source, { apiConnector, elasticClient, config }) =>  new Promise(
             label: po.name
           }
         })
+        output[po.name] = po.value
       }
     }))
 
@@ -139,7 +128,7 @@ const fill =  (source, { apiConnector, elasticClient, config }) =>  new Promise(
 
     Promise.all(subPromises).then((results) => {
         // console.log(output)
-        console.log('Product ' + output.name + ' - ' + output.sku + ' ' + output.type_id +': imported!')
+        console.log('Product ' + output.name + ' - ' + output.sku + ' ' + output.type_id + ' - price: ' + output.price + ': imported!')
 
         resolve(output)
       }