Validate twitter URLS start with https://twitter.com in landscape.yml; …

…closes #195 (#311)

* Validate twitter URLS start with https://twitter.com in landscape.yml

* Remove query string from twitter URLs

* When twitter is set to null ignore twitter url from crunchbase

package.json

@@ -12,8 +12,8 @@
     "open:dist": "babel-node tools/distServer.js",
     "lint": "esw webpack.config.* src tools --color",
     "lint:watch": "npm run lint -- --watch",
-    "fetch": "babel-node tools/validateLandscapeKeys && babel-node tools/addExternalInfo.js && npm run yaml2json",
-    "update": "(rm /tmp/landscape.json || true) && babel-node tools/validateLandscapeKeys && npm run remove-quotes && LEVEL=medium babel-node tools/addExternalInfo.js && npm run yaml2json && babel-node tools/calculateNumberOfTweets",
+    "fetch": "babel-node tools/validateLandscape && babel-node tools/addExternalInfo.js && npm run yaml2json",
+    "update": "(rm /tmp/landscape.json || true) && babel-node tools/validateLandscape && npm run remove-quotes && LEVEL=medium babel-node tools/addExternalInfo.js && npm run yaml2json && babel-node tools/calculateNumberOfTweets",
     "yaml2json": "babel-node tools/generateJson.js",
     "remove-quotes": "babel-node tools/removeQuotes",
     "check-links": "pkill -9 chrome; pkill -9 chrome; babel-node tools/checkLinks",

specs/tools/actualTwitter.spec.js

@@ -0,0 +1,63 @@
+import actualTwitter from '../../tools/actualTwitter';
+
+describe('Twitter URL', () => {
+  describe('when crunchbase data not set', () => {
+    const node = { twitter: 'https://twitter.com/foo' };
+
+    test('returns URL from node', async () => {
+      expect(actualTwitter(node, null)).toBe(node.twitter)
+    })
+  });
+
+  describe('when node does not have twitter URL', () => {
+    const crunchbaseData = { twitter: 'https://twitter.com/foo' };
+
+    test('returns URL from node', async () => {
+      expect(actualTwitter({}, crunchbaseData)).toBe(crunchbaseData.twitter)
+    })
+  });
+
+  describe('when node has twitter URL set to null', () => {
+    const crunchbaseData = { twitter: 'https://twitter.com/foo' };
+    const node = { twitter: null };
+
+    test('returns undefined', async () => {
+      expect(actualTwitter(node, crunchbaseData)).toBe(undefined)
+    })
+  });
+
+  describe('when both node and crunchbase have twitter URL', () => {
+    const node = { twitter: 'https://twitter.com/main' };
+    const crunchbaseData = { twitter: 'https://twitter.com/other' };
+
+    test('returns URL from node', async () => {
+      expect(actualTwitter(node, crunchbaseData)).toBe(node.twitter)
+    })
+  });
+
+  describe('when twitter URL is not set anywhere', () => {
+    const node = {};
+    const crunchbaseData = {};
+
+    test('returns undefined', async () => {
+      expect(actualTwitter(node, crunchbaseData)).toBe(undefined)
+    })
+  });
+
+  describe('cleaning up twitter URL', () => {
+    test('replaces http with https', async () => {
+      const node = { twitter: 'http://twitter.com/foo' };
+      expect(actualTwitter(node)).toBe('https://twitter.com/foo')
+    });
+
+    test('removes www', async () => {
+      const node = { twitter: 'https://www.twitter.com/foo' };
+      expect(actualTwitter(node)).toBe('https://twitter.com/foo')
+    });
+
+    test('query string', async () => {
+      const node = { twitter: 'https://twitter.com/foo?omg' };
+      expect(actualTwitter(node)).toBe('https://twitter.com/foo')
+    });
+  });
+});

tools/actualTwitter.js

@@ -1,25 +1,8 @@
-import _ from 'lodash';
 export default function actualTwitter(node, crunchbaseEntry) {
-  const url = (function() {
-      if (_.isUndefined(node.twitter)) {
-        return (crunchbaseEntry || {}).twitter;
-      }
-      return node.twitter;
-  })();
-  if (_.isEmpty(url)) {
-    return null;
+  const twitterUrl = 'twitter' in node ? node.twitter : (crunchbaseEntry || {}).twitter;
+
+  if (twitterUrl) {
+    return twitterUrl.replace(/^http(?:s)?\:\/\/(?:www\.)?/, 'https://')
+                     .replace(/\?.*/, '');
   }
-  const fixedUrl = (function() {
-    if (url.indexOf('http://twitter.com/') === 0) {
-      return url.replace('http://twitter.com/', 'https://twitter.com/');
-    }
-    if (url.indexOf('https://www.twitter.com/') === 0) {
-      return url.replace('https://www.twitter.com/', 'https://twitter.com/');
-    }
-    if (url.indexOf('http://twitter.com/') === 0) {
-      return url.replace('http://twitter.com/', 'https://twitter.com/');
-    }
-    return url;
-  })();
-  return fixedUrl;
 }

tools/validateLandscapeKeys.js → tools/validateLandscape.js

@@ -2,11 +2,13 @@ import process from 'process';
 import path from 'path';
 import { projectPath } from './settings';
 const source = require('js-yaml').safeLoad(require('fs').readFileSync(path.resolve(projectPath,'landscape.yml')));
+import actualTwitter from './actualTwitter';
 const traverse = require('traverse');
 const _ = require('lodash');
 
 console.info('Processing the tree');
-const errors = [];
+let errors = [];
+let hasInvalidKeys = false;
 
 const allowedKeys = [
   'name',
@@ -24,10 +26,20 @@ const allowedKeys = [
   'open_source'
 ];
 
-const categoryKeys = [
-  'name',
-  'subcategores'
-];
+const addKeyError = (title, key) => {
+  hasInvalidKeys = true;
+  errors.push(`${title} has an unknown key: ${key}`);
+}
+
+const validateTwitterUrl = (item) => {
+  if (item.twitter) {
+    const normalizedTwitterUrl = actualTwitter(item);
+
+    if (normalizedTwitterUrl.indexOf("https://twitter.com/") === -1) {
+      errors.push(`item ${item.name} has an invalid twitter URL: ${item.twitter}`);
+    }
+  }
+}
 
 function checkItem(item) {
   if (item.item !== null) {
@@ -42,10 +54,10 @@ function checkItem(item) {
     return allowedKeys.indexOf(key) === -1
   });
   wrongKeys.forEach(function(key) {
-    errors.push(`entry ${item.name} has an unkown key: ${key}`);
+    addKeyError(`item ${item.name}`, key);
   });
 
-
+  validateTwitterUrl(item);
 }
 
 function checkCategoryEntry(item) {
@@ -60,7 +72,7 @@ function checkCategoryEntry(item) {
     return ['category', 'name', 'subcategories'].indexOf(key) === -1;
   });
   wrongKeys.forEach(function(key) {
-    errors.push(`category entry ${item.name} has an unkown key: ${key}`);
+    addKeyError(`category ${item.name}`, key);
   });
 }
 
@@ -76,7 +88,7 @@ function checkSubcategoryEntry(item) {
     return ['subcategory', 'name', 'items'].indexOf(key) === -1;
   });
   wrongKeys.forEach(function(key) {
-    errors.push(`subcategory entry ${item.name} has an unkown key: ${key}`);
+    addKeyError(`subcategory ${item.name}`, key);
   });
 }
 
@@ -92,13 +104,12 @@ _.each(rootElement, function(category) {
   });
 });
 
-
-
-
 errors.forEach(function(error) {
   console.info('FATAL: ', error);
 });
 if (errors.length > 0) {
-  console.info('Valid item keys are', JSON.stringify(allowedKeys));
+  if (hasInvalidKeys) {
+    console.info('Valid item keys are', JSON.stringify(allowedKeys));
+  }
   process.exit(1);
 }