Merge pull request #5061 from newrelic/ru/exclude-file

feat: exclusion file and tests

loadershim.js

@@ -1,3 +1,7 @@
+process.env.HUMAN_TRANSLATION_PROJECT_ID = 'HT_ID';
+process.env.MACHINE_TRANSLATION_PROJECT_ID = 'MT_ID';
+process.env.DB_CONNECTION_INFO = '{}';
+
 global.___loader = {
   enqueue: jest.fn(),
 };

scripts/actions/__tests__/add-files-to-translation-queue.test.js

@@ -0,0 +1,219 @@
+'use strict';
+const { describe, expect, test, beforeEach } = require('@jest/globals');
+
+const fs = require('fs');
+const frontmatter = require('@github-docs/frontmatter');
+const { getLocalizedFileData } = require('../add-files-to-translation-queue');
+
+const MOCK_CONSTANTS = {
+  LOCALE_IDS: {
+    jp: 'ja-JP',
+    kr: 'ko-KR',
+  },
+};
+
+jest.mock('fs');
+jest.mock('path');
+jest.mock('@github-docs/frontmatter');
+jest.mock('../translation_workflow/database');
+jest.mock('../utils/vendor-request');
+
+const mockMdx = (translate = []) => {
+  return `---
+title: A test file
+${translate.length ? `translate:\n  - ${translate.join('\n  - ')}` : ''}
+---
+This is a test file
+`;
+};
+
+const mockReadFileSync = (translate = []) => {
+  const mdx = mockMdx(translate);
+  fs.readFileSync.mockReturnValueOnce(mdx);
+};
+
+describe('add-files-to-translation-queue tests', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  const setup = () => {
+    const EXCLUSIONS = {
+      excludePath: { jp: ['excluded/path'], kr: ['excluded/path'] },
+      excludeType: { jp: ['excludedType'], kr: ['excludedType'] },
+    };
+    const originalAdd = jest.requireActual('../utils/constants.js');
+
+    jest.doMock('../utils/constants.js', () => {
+      return { ...originalAdd, LOCALE_IDS: MOCK_CONSTANTS.LOCALE_IDS };
+    });
+
+    jest.mock('../utils/helpers');
+    const { getExclusions } = require('../utils/helpers');
+
+    getExclusions.mockReturnValue(EXCLUSIONS);
+  };
+
+  describe('Queue translations', () => {
+    test('Adds the Human Translation Project Id for locale under `translate` in frontmatter', async () => {
+      const file = { filename: '/content/bar.mdx' };
+      mockReadFileSync(['jp']);
+      frontmatter.mockReturnValueOnce({ data: { translate: ['jp'] } });
+      const toBeTranslated = getLocalizedFileData(file);
+
+      expect(toBeTranslated).toEqual([
+        { filename: '/content/bar.mdx', locale: 'ja-JP', project_id: 'HT_ID' },
+      ]);
+    });
+
+    test('Adds the Machine Translation Project Id when there is no `translate` in frontmatter', async () => {
+      const file = { filename: '/content/bar.mdx' };
+      mockReadFileSync();
+      frontmatter.mockReturnValueOnce({ data: {} });
+      const toBeTranslated = getLocalizedFileData(file);
+
+      expect(toBeTranslated).toEqual([
+        { filename: '/content/bar.mdx', locale: 'ja-JP', project_id: 'MT_ID' },
+      ]);
+    });
+
+    test('Adds the relevant Project Id when there are multiple locales and only one `translate` in frontmatter', async () => {
+      const file = { filename: '/content/bar.mdx' };
+      const originalAdd = jest.requireActual('../utils/constants.js');
+
+      jest.doMock('../utils/constants.js', () => {
+        return { ...originalAdd, LOCALE_IDS: MOCK_CONSTANTS.LOCALE_IDS };
+      });
+
+      const {
+        getLocalizedFileData,
+      } = require('../add-files-to-translation-queue');
+      const fs = require('fs');
+      const frontmatter = require('@github-docs/frontmatter');
+
+      jest.doMock('fs');
+      jest.doMock('@github-docs/frontmatter');
+
+      const mdx = mockMdx(['jp']);
+      fs.readFileSync.mockReturnValueOnce(mdx);
+      frontmatter.mockReturnValueOnce({ data: { translate: ['jp'] } });
+
+      const toBeTranslated = getLocalizedFileData(file);
+      expect(toBeTranslated).toEqual([
+        {
+          filename: '/content/bar.mdx',
+          locale: 'ja-JP',
+          project_id: 'HT_ID',
+        },
+        {
+          filename: '/content/bar.mdx',
+          locale: 'ko-KR',
+          project_id: 'MT_ID',
+        },
+      ]);
+    });
+
+    test('Doesnt exclude any files from translation', async () => {
+      const files = [
+        { filename: 'included/path/content/bar.mdx', locale: 'ja-JP' },
+        { filename: 'included/path/content/foo.mdx', locale: 'ko-KR' },
+      ];
+      setup();
+      const { excludeFiles } = require('../add-files-to-translation-queue');
+
+      const includedFiles = excludeFiles(files);
+
+      expect(includedFiles).toEqual([
+        { filename: 'included/path/content/bar.mdx', locale: 'ja-JP' },
+        { filename: 'included/path/content/foo.mdx', locale: 'ko-KR' },
+      ]);
+    });
+
+    test('Excludes files under a set path', async () => {
+      const files = [
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ko-KR',
+        },
+        {
+          filename: 'excluded/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ja-JP',
+        },
+      ];
+      setup();
+
+      const { excludeFiles } = require('../add-files-to-translation-queue');
+      const includedFiles = excludeFiles(files);
+
+      expect(includedFiles).toEqual([
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ko-KR',
+        },
+      ]);
+    });
+
+    test('Excludes specific file types', async () => {
+      const files = [
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'excludedType',
+          locale: 'ko-KR',
+        },
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ja-JP',
+        },
+      ];
+      setup();
+
+      const { excludeFiles } = require('../add-files-to-translation-queue');
+      const includedFiles = excludeFiles(files);
+
+      expect(includedFiles).toEqual([
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ja-JP',
+        },
+      ]);
+    });
+
+    test('Excludes files and file types under a set path', async () => {
+      const files = [
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ko-KR',
+        },
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'excludedType',
+          locale: 'ja-JP',
+        },
+        {
+          filename: 'excluded/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ja-JP',
+        },
+      ];
+      setup();
+
+      const { excludeFiles } = require('../add-files-to-translation-queue');
+      const includedFiles = excludeFiles(files);
+
+      expect(includedFiles).toEqual([
+        {
+          filename: 'included/path/content/bar.mdx',
+          contentType: 'doc',
+          locale: 'ko-KR',
+        },
+      ]);
+    });
+  });
+});

scripts/actions/add-files-to-translation-queue.js

@@ -1,7 +1,6 @@
 const fs = require('fs');
 const path = require('path');
 const frontmatter = require('@github-docs/frontmatter');
-
 const {
   getTranslations,
   addTranslation,
@@ -10,15 +9,13 @@ const {
 const { fetchPaginatedGHResults } = require('./utils/github-api-helpers');
 const checkArgs = require('./utils/check-args');
 const { prop } = require('../utils/functional');
+const { LOCALE_IDS } = require('./utils/constants');
+const { getExclusions } = require('./utils/helpers');
 
 const STATUS = {
   PENDING: 'PENDING',
 };
 
-const LOCALE_IDS = {
-  jp: 'ja-JP',
-};
-
 const translationDifference = (pendingFiles, prChanges) =>
   prChanges.filter(
     (file) =>
@@ -35,6 +32,28 @@ const slugIntersection = (pendingFiles, filesToRemove) =>
 const humanTranslatedProjectID = process.env.HUMAN_TRANSLATION_PROJECT_ID;
 const machineTranslatedProjectID = process.env.MACHINE_TRANSLATION_PROJECT_ID;
 
+/**
+ * Determines if a file should be included based on data from an exclusions file
+ * @param {Object[]} fileData The files to check
+ * @param {Object[]} exclusions The exclusions file
+ * @returns {Object[]} The files that should be included
+ */
+const excludeFiles = (fileData) => {
+  const exclusions = getExclusions();
+
+  return fileData.filter(({ filename, locale, contentType }) => {
+    const localeKey = Object.keys(LOCALE_IDS).find(
+      (localeKey) => LOCALE_IDS[localeKey] === locale
+    );
+    return (
+      !exclusions.excludePath[localeKey]?.some((path) =>
+        filename.startsWith(path)
+      ) &&
+      !exclusions.excludeType[localeKey]?.some((type) => contentType === type)
+    );
+  });
+};
+
 /**
  * Determines if a particular locale should be human or machine translated based on the files frontmatter.
  *
@@ -52,10 +71,11 @@ const getLocalizedFileData = (prFile) => {
   const contents = fs.readFileSync(path.join(process.cwd(), prFile.filename));
   const { data } = frontmatter(contents);
   const checkLocale = getProjectId(data.translate);
+  const contentType = data.type;
 
-  // Loops over all supported locales, determines translation project via getProjectId func.
   return Object.keys(LOCALE_IDS).map((locale) => ({
     ...prFile,
+    contentType,
     locale: LOCALE_IDS[locale],
     project_id: checkLocale(locale),
   }));
@@ -69,7 +89,9 @@ const main = async () => {
   checkArgs(3);
   const url = process.argv[2];
 
-  const queue = await getTranslations({ status: STATUS.PENDING });
+  const queue = await getTranslations({
+    status: STATUS.PENDING,
+  });
   const prFileData = await fetchPaginatedGHResults(
     url,
     process.env.GITHUB_TOKEN
@@ -80,10 +102,9 @@ const main = async () => {
     .filter((f) => f.status !== 'removed');
 
   const allLocalizedFileData = changedMdxFileData.flatMap(getLocalizedFileData);
-  const fileDataToAddToQueue = translationDifference(
-    queue,
-    allLocalizedFileData
-  );
+  const includedFiles = excludeFiles(allLocalizedFileData);
+
+  const fileDataToAddToQueue = translationDifference(queue, includedFiles);
 
   await Promise.all(
     fileDataToAddToQueue.map(({ filename, locale, project_id }) =>
@@ -106,4 +127,12 @@ const main = async () => {
   process.exit(0);
 };
 
-main();
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  getProjectId,
+  excludeFiles,
+  getLocalizedFileData,
+};

scripts/actions/translation_workflow/testing/script.sh

@@ -10,7 +10,7 @@ export GITHUB_TOKEN='' # token with no permissions
 
 # # step 1
 URL="https://api.github.com/repos/newrelic/docs-website/pulls/3271/files"
-yarn get-translated-files $URL
+yarn add-files-to-translate $URL
 
 # # step 2
 export TRANSLATION_VENDOR_API_URL=https://api.smartling.com

scripts/actions/utils/constants.js

@@ -0,0 +1,10 @@
+const LOCALE_IDS = {
+  jp: 'ja-JP',
+};
+
+const EXCLUSIONS_FILE = 'scripts/utils/docs-content-tools/i18n-exclusions.yml';
+
+module.exports = {
+  LOCALE_IDS,
+  EXCLUSIONS_FILE,
+};

scripts/actions/utils/docs-content-tools/i18n-exclusions.yml

@@ -0,0 +1,6 @@
+excludePath:
+  ja-JP:
+    - /whats-new
+excludeType:
+  ja-JP:
+    - landingPage

scripts/actions/utils/helpers.js

@@ -0,0 +1,16 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+const path = require('path');
+const { EXCLUSIONS_FILE } = require('./constants');
+
+/**
+ * Loads the Exclusions yaml file and converts it to a JSON object.
+ * @returns {Object} The Exclusions yaml file as a JSON object.
+ */
+const getExclusions = () => {
+  return yaml.load(fs.readFileSync(path.join(process.cwd(), EXCLUSIONS_FILE)));
+};
+
+module.exports = {
+  getExclusions,
+};