Get cached linkchecker result before hitting API (#1235)

opendatateam · Oct 24, 2017 · aefa821 · aefa821
1 parent 51e043a
commit aefa821
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 - Introduce `udata search index` commmand to replace both deprecated `udata search init` and `udata search reindex` commands. They will be removed in udata 1.4. [#1233](https://github.com/opendatateam/udata/pull/1233)
 - Rollback oauthlib from 2.0.5 to 2.0.2, pending a permanent solution [#1237](https://github.com/opendatateam/udata/pull/1237)
+- Get cached linkchecker result before hitting API [#1235](https://github.com/opendatateam/udata/pull/1235)
 
 ## 1.2.0 (2017-10-20)
 

diff --git a/js/config.js b/js/config.js
@@ -69,6 +69,11 @@ export const csrf_token = _meta('csrf-token');
  */
 export const check_urls = _jsonMeta('check-urls');
 
+/**
+ * What is the link checker result cache duration?
+ */
+export const check_urls_cache_duration = _jsonMeta('check-urls-cache-duration');
+
 /**
  * The API root/base URL
  */
@@ -168,6 +173,7 @@ export default {
     auth_url,
     sentry,
     check_urls,
+    check_urls_cache_duration,
     is_territory_enabled,
     is_delete_me_enabled,
     hidpi,

diff --git a/js/front/dataset/index.js b/js/front/dataset/index.js
@@ -39,6 +39,13 @@ new Vue({
             userReuses: []
         };
     },
+    computed: {
+        limitCheckDate() {
+            const limitDate = new Date();
+            limitDate.setSeconds(limitDate.getSeconds() - config.check_urls_cache_duration);
+            return limitDate;
+        }
+    },
     ready() {
         this.loadCoverageMap();
         this.checkResources();
@@ -140,6 +147,35 @@ new Vue({
             }
         },
 
+        /**
+         * Get a cached checked result from extras if fresh enough
+         * @param  {Object} resource A resource as extracted from JSON-LD
+         */
+        getCachedCheck(resource) {
+            const extras = resource.extras;
+            if (extras['check:date']) {
+                const checkDate = new Date(extras['check:date']);
+                if (checkDate >= this.limitCheckDate) {
+                    return Object.keys(extras).reduce((obj, key) => {
+                        if (key.startsWith('check:')) {
+                            obj[key] = extras[key];
+                        }
+                        return obj;
+                    }, {});
+                }
+            }
+        },
+
+        /**
+         * Get cached check or API check
+         * @param  {Object} resource A resource element from DOM
+         * @param  {String} checkurl The API check url
+         */
+        getResourceCheckStatus(resource_el, checkurl) {
+            const cachedCheck = this.getCachedCheck(resource_el);
+            return (cachedCheck && Promise.resolve(cachedCheck)) || this.$api.get(checkurl);
+        },
+
         /**
          * Asynchronously check a displayed resource status
          * @param  {Object} resource A resource as extracted from JSON-LD
@@ -153,7 +189,7 @@ new Vue({
                 el.classList.add('format-label-warning');
                 el.setTooltip(this._('The server may be hard to reach (FTP).'), true);
             } else {
-                this.$api.get(checkurl)
+                this.getResourceCheckStatus(resource, checkurl)
                 .then((res) => {
                     const status = res['check:status'];
                     if (status >= 200 && status < 400) {

diff --git a/udata/core/dataset/models.py b/udata/core/dataset/models.py
@@ -90,6 +90,16 @@
 MAX_DISTANCE = 2
 
 
+def get_json_ld_extra(key, value):
+    '''Serialize an extras key, value pair into JSON-LD'''
+    value = value.serialize() if hasattr(value, 'serialize') else value
+    return {
+        '@type': 'http://schema.org/PropertyValue',
+        'name': key,
+        'value': value,
+    }
+
+
 class License(db.Document):
     # We need to declare id explicitly since we do not use the default
     # value set by Mongo.
@@ -228,6 +238,8 @@ def json_ld(self):
             'dateCreated': self.created_at.isoformat(),
             'dateModified': self.modified.isoformat(),
             'datePublished': self.published.isoformat(),
+            'extras': [get_json_ld_extra(*item)
+                       for item in self.extras.items()],
         }
 
         if 'views' in self.metrics:
@@ -539,7 +551,7 @@ def json_ld(self):
             'contributedDistribution': [
                 resource.json_ld for resource in self.community_resources
             ],
-            'extras': [self.get_json_ld_extra(*item)
+            'extras': [get_json_ld_extra(*item)
                        for item in self.extras.items()],
         }
 
@@ -561,16 +573,6 @@ def json_ld(self):
 
         return result
 
-    @staticmethod
-    def get_json_ld_extra(key, value):
-
-        value = value.serialize() if hasattr(value, 'serialize') else value
-        return {
-            '@type': 'http://schema.org/PropertyValue',
-            'name': key,
-            'value': value,
-        }
-
 
 pre_save.connect(Dataset.pre_save, sender=Dataset)
 post_save.connect(Dataset.post_save, sender=Dataset)

diff --git a/udata/templates/macros/metadata.html b/udata/templates/macros/metadata.html
@@ -47,6 +47,7 @@
 {% endif %}
 
 <meta name="check-urls" content="{{ config.LINKCHECKING_ENABLED|tojson }}" />
+<meta name="check-urls-cache-duration" content="{{ config.LINKCHECKING_CACHE_DURATION }}" />
 <meta name="territory-enabled" content="{{ 'true' if config.ACTIVATE_TERRITORIES else 'false' }}">
 
 <meta name="delete-me-enabled" content="{{ 'true' if config.DELETE_ME else 'false' }}">