fix: dataset autoreload with github api conditional request

api/github/api.js

@@ -1,8 +1,10 @@
 const helpersFactory = require('./helpers')
 const octokitFactory = require('@octokit/rest')
+const debug = require('debug')('editor:api')
 
 module.exports = class GitHubAPIv3 {
   constructor ({ forge, editor, ontology }) {
+    debug('new GitHubAPIv3')
     this.branch = editor.github.branch
     this.owner = editor.github.owner
     this.repo = editor.github.repo
@@ -25,6 +27,8 @@ module.exports = class GitHubAPIv3 {
     const { getRefSHA, getFileSHA } = helpersFactory(this.__octokit)
     this.__getRefSHA = getRefSHA
     this.__getFileSHA = getFileSHA
+
+    this.__cache = new Map()
   }
 
   async createBranch () {
@@ -49,12 +53,42 @@ module.exports = class GitHubAPIv3 {
     }
   }
 
+  async getHeadSHA ({ branch = this.branch } = {}) {
+    const owner = this.owner
+    const repo = this.repo
+    const ref = `heads/${branch}`
+    const response = await this.__octokit.repos.getCommitRefSha({ owner, repo, ref })
+    return response.data.sha
+  }
+
   async getFile ({ path = this.ontologyPath, branch = this.branch } = {}) {
     const owner = this.owner
     const repo = this.repo
-    const ref = `heads/${this.branch}`
-    const result = await this.__octokit.repos.getContents({ owner, repo, path, ref })
-    const content = Buffer.from(result.data.content, 'base64').toString()
+    const ref = `heads/${branch}`
+    const query = { owner, repo, path, ref }
+
+    const cached = this.__cache.get(`${ref}/${path}`)
+    const headers = cached ? { 'If-Modified-Since': cached.lastModified } : {}
+
+    let content
+    try {
+      const response = await this.__octokit.repos.getContents({ ...query, headers })
+      content = Buffer.from(response.data.content, 'base64').toString()
+      const lastModified = response.headers['last-modified']
+      this.__cache.set(`${ref}/${path}`, { lastModified, content })
+    }
+    catch (error) {
+      // '304 not modified', let's serve cached version
+      if (error.status === 304) {
+        debug(`github sent 304 for file ${ref}/${path}`)
+        content = cached.content
+      }
+      else {
+        debug(`rate limit: ${error.headers['x-ratelimit-remaining']}`)
+        throw error
+      }
+    }
+
     return content
   }
 
@@ -71,7 +105,7 @@ module.exports = class GitHubAPIv3 {
       repo
     })
 
-    const result = await this.__octokit.repos.updateFile({
+    const response = await this.__octokit.repos.updateFile({
       content: Buffer.from(content).toString('base64'),
       path,
       owner,
@@ -83,14 +117,14 @@ module.exports = class GitHubAPIv3 {
       author
     })
 
-    return result
+    return response
   }
 
   async createPR ({ title, body, branch } = {}) {
     const owner = this.owner
     const repo = this.repo
 
-    const result = await this.__octokit.pullRequests.create({
+    const response = await this.__octokit.pullRequests.create({
       head: branch,
       base: this.branch,
       maintainer_can_modify: true,
@@ -101,24 +135,24 @@ module.exports = class GitHubAPIv3 {
     })
 
     return {
-      number: result.data.number
+      number: response.data.number
     }
   }
 
   async mergePR ({ number, sha } = {}) {
     const owner = this.owner
     const repo = this.repo
 
-    const result = await this.__octokit.pullRequests.merge({
+    const response = await this.__octokit.pullRequests.merge({
       owner,
       repo,
       number,
       sha
     })
 
     return {
-      success: !!result.data.merged, // true | undefined => bool
-      message: result.data.message // shown to user if !success
+      success: !!response.data.merged, // true | undefined => bool
+      message: response.data.message // shown to user if !success
     }
   }
 

api/github/index.js

@@ -1,22 +1,13 @@
 const _ = require('lodash')
 const Router = require('express').Router
 const axios = require('axios')
-const apicache = require('apicache')
 const gql = require('graphql-tag')
-const debug = require('debug')('editor:api')
 const apolloClientFactory = require('../getApolloClient')
 const GitHubAPIv3 = require('./api')
 
 module.exports = async function (editorConfig) {
   const router = Router()
   const api = new GitHubAPIv3(editorConfig)
-  const onlyStatus200 = (req, res) => res.statusCode === 200
-  const cache = (duration) => apicache.middleware(duration, onlyStatus200)
-
-  process.on('SIGHUP', () => {
-    apicache.clear()
-    debug('GitHub API: apicache cleared')
-  })
 
   const anonApolloClient = await apolloClientFactory()
   const getApolloClientForUser = async (req) => apolloClientFactory({
@@ -25,29 +16,13 @@ module.exports = async function (editorConfig) {
     getAuth: () => req.get('Authorization'),
     ssr: true
   })
-  const ontologyFilename = editorConfig.ontology.ontologyRawUrl.substr(editorConfig.ontology.ontologyRawUrl.lastIndexOf('/') + 1)
-  const structureFilename = editorConfig.ontology.structureRawUrl.substr(editorConfig.ontology.structureRawUrl.lastIndexOf('/') + 1)
 
   router.get('/', (req, res, next) => {
     res.send('Ontology Editor currently using GitHub')
   })
 
-  router.get('/cache', (req, res) => {
-    res.json(apicache.getIndex())
-  })
-
-  router.get('/blob/:branch/:file', cache('5 minutes'), async (req, res, next) => {
+  router.get('/blob/:file', async (req, res, next) => {
     const path = req.params.file
-    const branch = req.params.branch
-    const content = await api.getFile({ path, branch })
-    res.type('application/n-triples')
-
-    res.send(content)
-  })
-
-  router.get('/blob/:file', cache('5 minutes'), async (req, res, next) => {
-    const path = req.params.file
-    req.apicacheGroup = `file:${path}`
     const content = await api.getFile({ path })
     res.type('application/n-triples')
 
@@ -227,10 +202,6 @@ module.exports = async function (editorConfig) {
   })
 
   router.post('/proposal/merge', async (req, res, next) => {
-    [ontologyFilename, structureFilename].forEach((file) => {
-      apicache.clear(`file:${file}`)
-    })
-
     const { threadId, number } = req.body
 
     try {
@@ -272,9 +243,6 @@ module.exports = async function (editorConfig) {
   })
 
   router.post('/proposal/close', async (req, res, next) => {
-    [ontologyFilename, structureFilename].forEach((file) => {
-      apicache.clear(`file:${file}`)
-    })
 
     const { threadId, number, status } = req.body
 

components/admin/AdminProposalList.vue

@@ -248,8 +248,8 @@ export default {
       catch (err) {
         this.working = false
         console.error(err)
-        this.$sentry.captureException(err)
         this.$toast.error(`Error: ${err.response.data.message || err.message}`, toastClose).goAway(1600)
+        this.$sentry.captureException(err)
       }
     },
     async reject (proposal, status = 'REJECTED') {

components/fallback/ClassProposals.vue

@@ -89,6 +89,7 @@ export default {
         }
       },
       fetchPolicy: 'no-cache',
+      pollInterval: 1000 * 15,
       result ({ data, loading }) {
         if (!loading) {
           const proposals = _get(data, 'proposals.proposals', [])

components/fallback/PropertyProposals.vue

@@ -97,6 +97,7 @@ export default {
         }
       },
       fetchPolicy: 'no-cache',
+      pollInterval: 1000 * 15,
       result ({ data, loading }) {
         if (!loading) {
           const proposals = _get(data, 'proposals.proposals', [])

nuxt.config.js

@@ -49,6 +49,7 @@ module.exports = async () => {
     * Order matters!
     */
     plugins: [
+      '@/plugins/sentry',
       '@/plugins/libs/rdf',
       '@/plugins/models/Class',
       '@/plugins/models/Property',
@@ -136,7 +137,6 @@ module.exports = async () => {
     ** Module config: sentry
     */
     sentry: {
-      dsn: process.env.SENTRY_DSN,
       config: {
         release: version
       }

plugins/nuxt-init.client.js

@@ -1,5 +1,8 @@
 export default async (context) => {
   if (process.browser) {
     await context.store.dispatch('nuxtClientInit', context)
+    setInterval(() => {
+      context.store.dispatch('graph/RELOAD_DATASET')
+    }, 10000)
   }
 }

plugins/sentry.js

@@ -0,0 +1,9 @@
+export default ({ app }, inject) => {
+  if (!process.SENTRY_DSN) {
+    inject('sentry', { captureException })
+  }
+}
+
+function captureException (error) {
+  console.error(error)
+}

store/index.js

@@ -1,5 +1,7 @@
 import * as VueDeepSet from 'vue-deepset'
 
+let datasetPolling = false
+
 export const state = () => ({
   authProcessDone: false
 })
@@ -28,6 +30,12 @@ export const actions = {
     if (req && req.structure) {
       commit('graph/structureInit', req.structure)
     }
+    if (!datasetPolling && process.server) {
+      datasetPolling = setInterval(() => {
+        console.log(new Date())
+        dispatch('graph/RELOAD_DATASET')
+      }, 5000)
+    }
   },
   async nuxtClientInit ({ commit, dispatch }, context) {
     commit('class/NEW')

trifid/dataset-fetch-client.js

@@ -5,8 +5,8 @@ import axios from 'axios'
 
 const debug = require('debug')('editor:trifid')
 
-// * used by the store to refresh client-side dataset after a merge
-// only used client-side, the other one cannot be used client-side because of
+// * used by the store to refresh client-side dataset after a merge.
+// Only used client-side, the other one cannot be used client-side because of
 // knex
 export default async function fetchDataset (editorConfig) {
   const ontologyFilename = editorConfig.ontology.ontologyRawUrl.substr(editorConfig.ontology.ontologyRawUrl.lastIndexOf('/') + 1)