Add tool to look for keys in app and src and make sure they are in /e…

…n/messages.json

.gitignore

@@ -18,6 +18,7 @@ yarn-error.log
 .ltk
 tools/i18n_results
 tools/leet/*.json
+tools/token_results
 
 ## Cliqz
 cliqz/

package.json

@@ -87,6 +87,7 @@
     "eslint-plugin-import": "^2.20.2",
     "eslint-plugin-jsx-a11y": "^6.2.3",
     "eslint-plugin-react": "^7.19.0",
+    "find-in-files": "^0.5.0",
     "fs-extra": "^9.0.0",
     "jest": "^25.4.0",
     "jest-fetch-mock": "^3.0.3",

tools/i18n-checker.js

@@ -84,7 +84,13 @@ function validateJson(paths) {
 		let hasError = false;
 		paths.forEach((path) => {
 			try {
-				jsonfile.readFileSync(`${path}`);
+				const file = jsonfile.readFileSync(`${path}`);
+				Object.keys(file).forEach((key) => {
+					if (!/^\w*$/.test(key)) {
+						hasError = true;
+						console.log('Error: file %s has invalid key "%s".', path, key);
+					}
+				});
 			} catch (err) {
 				hasError = true;
 				console.log('Error: file "%s" is not valid JSON.', path);

tools/token-checker.js

@@ -0,0 +1,144 @@
+/**
+ * Token Checker
+ *
+ * Ghostery Browser Extension
+ * http://www.ghostery.com/
+ *
+ * Copyright 2020 Ghostery, Inc. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0
+ */
+
+/* eslint import/no-extraneous-dependencies: 0 */
+/* eslint no-console: 0 */
+
+console.time('token-checker');
+
+const fs = require('fs-extra');
+const jsonfile = require('jsonfile');
+const findInFiles = require('find-in-files');
+
+// Constants
+const DEFAULT_LOCALE_PATH = './_locales/en/messages.json';
+const SEARCH_DIRECTORIES = ['app', 'databases', 'src', 'test', 'tools'];
+const FIND_TOKEN_REGEX = /\Wt\(['|"|`](\w*)['|"|`]\)/;
+const UNDEFINED_TOKEN_FILE = './tools/token_results/undefined_tokens.txt';
+
+// Empty tools/token_results directory
+fs.emptyDirSync('./tools/token_results');
+
+/**
+ * Outputs the contents of an object to a .txt file.
+ * @param  string fileName       The location of the file we will output to
+ * @param  object resultsObject  An object with the data we will output
+ * @return none
+ */
+function recordResults(fileName, resultsObject) {
+	const stream = fs.createWriteStream(fileName);
+	stream.once('open', () => {
+		Object.keys(resultsObject).forEach((file) => {
+			stream.write(`'${file}' has missing tokens:\n`);
+			Object.keys(resultsObject[file]).forEach((token) => {
+				stream.write(`\t${token}\n`);
+			});
+			stream.write('\n');
+		});
+		stream.end();
+	});
+}
+
+/**
+ * Gathers the tokens used in a directory
+ * @param  directory   directory to search for tokens
+ * @const  RegEx       FIND_TOKEN_REGEX  regex used to find tokens
+ * @return Promise     Resolves with an object of all tokens and the files
+ *                     in which those tokens were found
+ */
+function findTokensInDirectory(directory) {
+	return new Promise((resolve) => {
+		const dirTokens = {};
+		findInFiles.find(FIND_TOKEN_REGEX, directory).then((dirResults) => {
+			const fileNames = Object.keys(dirResults);
+			for (let i = 0; i < fileNames.length; i++) {
+				const fileName = fileNames[i];
+				const fileMatches = dirResults[fileName].matches;
+				for (let j = 0; j < fileMatches.length; j++) {
+					const match = fileMatches[j];
+					const token = match.substr(4, match.length - 6);
+					if (!dirTokens.hasOwnProperty(token)) {
+						dirTokens[token] = { files: {} };
+					}
+					dirTokens[token].files[fileName] = true;
+				}
+			}
+			resolve(dirTokens);
+		});
+	});
+}
+
+/**
+ * Gathers the tokens used in a directory
+ * @param  dirsTokens   Resolved object of findTokensInDirectory.
+ *                      Object of tokens and files in which the tokens were found.
+ * @const  string       DEFAULT_LOCALE_PATH  The location of the default locale JSON file
+ * @return Promise      Resolves with an object of files that have tokens not found
+ *                      in DEFAULT_LOCALE_PATH's messages.json file
+ */
+function compileUndefinedTokens(dirsTokens) {
+	const defaultLocaleJson = jsonfile.readFileSync(DEFAULT_LOCALE_PATH);
+	const undefinedTokenFiles = {};
+	return new Promise((resolve) => {
+		for (let i = 0; i < dirsTokens.length; i++) {
+			const dirTokens = dirsTokens[i];
+			const dirTokensArr = Object.keys(dirTokens);
+			for (let j = 0; j < dirTokensArr.length; j++) {
+				const token = dirTokensArr[j];
+				if (!defaultLocaleJson.hasOwnProperty(token)) {
+					const files = Object.keys(dirTokens[token].files);
+					for (let k = 0; k < files.length; k++) {
+						const file = files[k];
+						if (!undefinedTokenFiles.hasOwnProperty(file)) {
+							undefinedTokenFiles[file] = {};
+						}
+						undefinedTokenFiles[file][token] = true;
+					}
+				}
+			}
+		}
+		resolve(undefinedTokenFiles);
+	});
+}
+
+/**
+ * Checks for missing tokens throughout the project. Writes the list of missing tokens to a file.
+ * Does not check for tokens that are defined using variables.
+ * @const  array       SEARCH_DIRECTORIES  directories to search for tokens
+ * @const  string      UNDEFINED_TOKEN_FILE  The file where we should write the tokens
+ * @return Promise     Resolves or Rejects depending on whether there are undefined tokens
+ */
+function findUndefinedTokens() {
+	return new Promise((resolve, reject) => {
+		Promise.all(
+			SEARCH_DIRECTORIES.map(directory => findTokensInDirectory(directory))
+		).then(compileUndefinedTokens).then((undefinedTokenFiles) => {
+			const undefinedTokenFilesArr = Object.keys(undefinedTokenFiles);
+			if (undefinedTokenFilesArr.length >= 1) {
+				console.log('Error: undefined tokens were found. See them in `%s`.', UNDEFINED_TOKEN_FILE);
+				recordResults(UNDEFINED_TOKEN_FILE, undefinedTokenFiles);
+				reject();
+			} else {
+				console.log('Scanned all directories for undefined tokens, none found.');
+				resolve();
+			}
+		});
+	});
+}
+
+// Main
+findUndefinedTokens().catch(() => {
+	console.log('Errors found. Fix the files and run `node tools/token-checker` to re-validate translation tokens.');
+}).then(() => {
+	console.timeEnd('token-checker');
+});

yarn.lock

@@ -3569,6 +3569,14 @@ find-cache-dir@^2.1.0:
     make-dir "^2.0.0"
     pkg-dir "^3.0.0"
 
+find-in-files@^0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/find-in-files/-/find-in-files-0.5.0.tgz#8e5a20ffb562e0a47cb916b7f7b821717025e691"
+  integrity sha512-VraTc6HdtdSHmAp0yJpAy20yPttGKzyBWc7b7FPnnsX9TOgmKx0g9xajizpF/iuu4IvNK4TP0SpyBT9zAlwG+g==
+  dependencies:
+    find "^0.1.5"
+    q "^1.0.1"
+
 find-up@^1.0.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f"
@@ -3599,6 +3607,13 @@ find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
+find@^0.1.5:
+  version "0.1.7"
+  resolved "https://registry.yarnpkg.com/find/-/find-0.1.7.tgz#c86c87af1ab18f222bbe38dec86cbc760d16a6fb"
+  integrity sha1-yGyHrxqxjyIrvjjeyGy8dg0Wpvs=
+  dependencies:
+    traverse-chain "~0.1.0"
+
 findup-sync@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1"
@@ -6848,6 +6863,11 @@ punycode@^2.1.0, punycode@^2.1.1:
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
   integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
 
+q@^1.0.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
+  integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=
+
 qs@~6.5.2:
   version "6.5.2"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
@@ -8452,6 +8472,11 @@ tr46@^1.0.1:
   dependencies:
     punycode "^2.1.0"
 
+traverse-chain@~0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/traverse-chain/-/traverse-chain-0.1.0.tgz#61dbc2d53b69ff6091a12a168fd7d433107e40f1"
+  integrity sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE=
+
 treeify@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/treeify/-/treeify-1.1.0.tgz#4e31c6a463accd0943879f30667c4fdaff411bb8"