Tool/unused i18n token finder

tools/i18n-checker.js

@@ -24,7 +24,7 @@ const oboe = require('oboe');
 const LOCALES_FOLDER = './_locales';
 const GATHER_FILE_PATHS_EXCEPTIONS = ['.DS_Store'];
 const LANG_FILES_COUNT = 14;
-const DEFAULT_LOCALE_PATH = '../_locales/en/messages.json';
+const DEFAULT_LOCALE_PATH = './_locales/en/messages.json';
 const DUPLICATE_TOKENS_FILE = './tools/i18n_results/duplicate_tokens.txt';
 const MISSING_TOKENS_FILE = './tools/i18n_results/missing_tokens.txt';
 const EXTRA_TOKENS_FILE = './tools/i18n_results/extra_tokens.txt';
@@ -175,7 +175,7 @@ function findMissingKeys(paths) {
 		let hasMissingKeys = false;
 		const missingKeys = {};
 		paths.forEach((path) => {
-			const localeJson = jsonfile.readFileSync(`.${path}`);
+			const localeJson = jsonfile.readFileSync(`${path}`);
 			const locale = path.match(/_locales\/(.*)\/messages.json/)[1];
 			missingKeys[locale] = [];
 			Object.keys(defaultLocaleJson).forEach((key) => {
@@ -210,7 +210,7 @@ function findExtraKeys(paths) {
 		let hasExtraKeys = false;
 		const extraKeys = {};
 		paths.forEach((path) => {
-			const localeJson = jsonfile.readFileSync(`.${path}`);
+			const localeJson = jsonfile.readFileSync(`${path}`);
 			const locale = path.match(/_locales\/(.*)\/messages.json/)[1];
 			extraKeys[locale] = [];
 			Object.keys(localeJson).forEach((key) => {
@@ -243,7 +243,7 @@ function findMalformedKeys(paths) {
 		let hasMalformedKeys = false;
 		const malformedKeys = {};
 		paths.forEach((path) => {
-			const localeJson = jsonfile.readFileSync(`.${path}`);
+			const localeJson = jsonfile.readFileSync(`${path}`);
 			const locale = path.match(/_locales\/(.*)\/messages.json/)[1];
 			malformedKeys[locale] = [];
 			Object.keys(localeJson).forEach((key) => {
@@ -278,7 +278,7 @@ function findMissingPlaceholders(paths) {
 		let hasMissingPlaceholders = false;
 		const missingPlaceholders = {};
 		paths.forEach((path) => {
-			const localeJson = jsonfile.readFileSync(`.${path}`);
+			const localeJson = jsonfile.readFileSync(`${path}`);
 			const locale = path.match(/_locales\/(.*)\/messages.json/)[1];
 			missingPlaceholders[locale] = [];
 			Object.keys(defaultLocaleJson).forEach((key) => {
@@ -322,7 +322,7 @@ function findExtraPlaceholders(paths) {
 		let hasExtraPlaceholders = false;
 		const extraPlaceholders = {};
 		paths.forEach((path) => {
-			const localeJson = jsonfile.readFileSync(`.${path}`);
+			const localeJson = jsonfile.readFileSync(`${path}`);
 			const locale = path.match(/_locales\/(.*)\/messages.json/)[1];
 			extraPlaceholders[locale] = [];
 			Object.keys(localeJson).forEach((key) => {
@@ -364,7 +364,7 @@ function findMalformedPlaceholders(paths) {
 		let hasMalformedPlaceholders = false;
 		const malformedPlaceholders = [];
 		paths.forEach((path) => {
-			const localeJson = jsonfile.readFileSync(`.${path}`);
+			const localeJson = jsonfile.readFileSync(`${path}`);
 			const locale = path.match(/_locales\/(.*)\/messages.json/)[1];
 			malformedPlaceholders[locale] = [];
 			Object.keys(localeJson).forEach((key) => {

tools/unused-i18n-token-finder.js

@@ -0,0 +1,126 @@
+/**
+ * Possibly Unused i18n Token Finder
+ * Key word: POSSIBLY
+ *
+ * Looks for i18n tokens that MAY be unused by the code
+ * Since some tokens are generated dynamically, the list generated by this script
+ * should ALWAYS be verified manually before removing any of the tokens in it
+ *
+ * Ghostery Browser Extension
+ * http://www.ghostery.com/
+ *
+ * Copyright 2019 Ghostery, Inc. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0
+ */
+
+console.time('unused-i18n-token-finder');
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+const fs = require('fs');
+// eslint-disable-next-line import/no-extraneous-dependencies
+const jsonfile = require('jsonfile');
+
+// Constants
+const DEFAULT_LOCALE_TOKENS_FILE = './_locales/en/messages.json';
+const UNUSED_TOKENS_DIR = './tools/i18n_results';
+const UNUSED_TOKENS_FILENAME = 'unused_tokens.txt';
+
+function saveListOfUnusedTokensToFile(unusedTokens) {
+	const filepath = `${UNUSED_TOKENS_DIR}/${UNUSED_TOKENS_FILENAME}`;
+
+	if (!fs.existsSync(UNUSED_TOKENS_DIR)) {
+		fs.mkdirSync(UNUSED_TOKENS_DIR);
+	}
+
+	fs.writeFileSync(
+		filepath,
+		unusedTokens.join('\n')
+	);
+}
+
+function findUnusedTokens(tokens, filepaths) {
+	tokens = tokens.map(token => ({ value: token, isUsed: false }));
+
+	filepaths.forEach((filepath) => {
+		const fileContents = fs.readFileSync(filepath, 'utf8');
+		tokens.forEach((token) => {
+			if (token.isUsed) { return; }
+
+			// THE TEST
+			if (fileContents.includes(`t('${token.value}`)) {
+				token.isUsed = true;
+			}
+		});
+	});
+
+	const unusedTokens =
+		(tokens.filter(token => token.isUsed === false))
+			.map(token => token.value);
+
+	return unusedTokens;
+}
+
+/**
+ * Recursively collect the filepaths of files that
+ * satisfy the supplied extension and file system location conditions
+ * @param [Array|object] whereToLookAndForWhatExtensions
+ * @param [string Array] filepaths							The matching filepaths
+ * @returns [string Array] filepaths						The matching filepaths
+ */
+function getFilepaths(whereToLookAndForWhatExtensions, filepaths = []) {
+	const target = whereToLookAndForWhatExtensions;
+
+	if (Array.isArray(target)) {
+		target.forEach((t) => {
+			filepaths = getFilepaths(t, filepaths);
+		});
+	} else {
+		const dirEntries = fs.readdirSync(target.dir, { withFileTypes: true });
+
+		dirEntries.forEach((dirEntry) => {
+			if (dirEntry.isDirectory()) {
+				filepaths = getFilepaths({
+					dir: `${target.dir}/${dirEntry.name}`,
+					extensions: target.extensions
+				}, filepaths);
+			} else if (dirEntry.isFile()) {
+				if (target.extensions.some(extension => dirEntry.name.endsWith(extension))) {
+					filepaths.push(`${target.dir}/${dirEntry.name}`);
+				}
+			}
+		});
+	}
+
+	return filepaths;
+}
+
+function getJSONKeys(filepath) {
+	const json = jsonfile.readFileSync(filepath);
+	return Object.keys(json);
+}
+
+saveListOfUnusedTokensToFile(
+	findUnusedTokens(
+		getJSONKeys(DEFAULT_LOCALE_TOKENS_FILE),
+		getFilepaths(
+			[
+				// Overly broad, but we favor simplicity since there is no compelling reason here to favor performance / efficiency
+				// Also, we prefer that unused tokens be incorrectly reported as used than vice versa
+				{ dir: './app', extensions: ['.jsx', '.js'] },
+				{ dir: './src', extensions: ['.js'] },
+			]
+		)
+	)
+);
+
+console.log('\nPLEASE NOTE:');
+console.log('Since some i18n tokens are generated dynamically,')
+console.log('and since some others are formatted in a non-standard way,');
+console.log('the list generated by this script should ALWAYS');
+console.log('be verified manually before removing any of the tokens in it.');
+console.log('\nThe results are in ./tools/i18n_results/unused_tokens.txt\n');
+
+console.timeEnd('unused-i18n-token-finder');