Merge branch 'feat/machine-translation-script' into feature/machine-t…

…ranslation-v2

.github/workflows/add-slugs-to-translate-queue.yml

@@ -42,4 +42,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${{ github.event.pull_request.number }}/files"
-          yarn add-files-to-translate $URL
+          yarn add-files-to-translate -u $URL

package.json

@@ -75,6 +75,7 @@
     "aws-sdk": "^2.827.0",
     "babel-preset-gatsby": "^1.3.0",
     "chalk": "^4.1.0",
+    "commander": "^8.3.0",
     "dotenv": "^8.2.0",
     "eslint": "^7.7.0",
     "eslint-plugin-jsx-a11y": "^6.3.1",

scripts/actions/__tests__/add-files-to-translation-queue.test.js

@@ -38,8 +38,8 @@ describe('add-files-to-translation-queue tests', () => {
 
   const setup = () => {
     const EXCLUSIONS = {
-      excludePath: { jp: ['excluded/path'], kr: ['excluded/path'] },
-      excludeType: { jp: ['excludedType'], kr: ['excludedType'] },
+      excludePath: { 'ja-JP': ['excluded/path'], 'ko-KR': ['excluded/path'] },
+      excludeType: { 'ja-JP': ['excludedType'], 'ko-KR': ['excludedType'] },
     };
     const originalAdd = jest.requireActual('../utils/constants.js');
 
@@ -55,18 +55,17 @@ describe('add-files-to-translation-queue tests', () => {
 
   describe('Queue translations', () => {
     test('Adds the Human Translation Project Id for locale under `translate` in frontmatter', async () => {
-      const file = { filename: '/content/bar.mdx' };
+      const file = '/content/bar.mdx';
       mockReadFileSync(['jp']);
       frontmatter.mockReturnValueOnce({ data: { translate: ['jp'] } });
       const toBeTranslated = getLocalizedFileData(file);
-
       expect(toBeTranslated).toEqual([
         { filename: '/content/bar.mdx', locale: 'ja-JP', project_id: 'HT_ID' },
       ]);
     });
 
     test('Adds the Machine Translation Project Id when there is no `translate` in frontmatter', async () => {
-      const file = { filename: '/content/bar.mdx' };
+      const file = '/content/bar.mdx';
       mockReadFileSync();
       frontmatter.mockReturnValueOnce({ data: {} });
       const toBeTranslated = getLocalizedFileData(file);
@@ -77,7 +76,7 @@ describe('add-files-to-translation-queue tests', () => {
     });
 
     test('Adds the relevant Project Id when there are multiple locales and only one `translate` in frontmatter', async () => {
-      const file = { filename: '/content/bar.mdx' };
+      const file = '/content/bar.mdx';
       const originalAdd = jest.requireActual('../utils/constants.js');
 
       jest.doMock('../utils/constants.js', () => {

scripts/actions/add-files-to-translation-queue.js

@@ -1,13 +1,13 @@
 const fs = require('fs');
 const path = require('path');
 const frontmatter = require('@github-docs/frontmatter');
+const { Command } = require('commander');
+const glob = require('glob');
 const {
   getTranslations,
   addTranslation,
-  deleteTranslation,
 } = require('./translation_workflow/database');
 const { fetchPaginatedGHResults } = require('./utils/github-api-helpers');
-const checkArgs = require('./utils/check-args');
 const { prop } = require('../utils/functional');
 const { LOCALE_IDS } = require('./utils/constants');
 const { getExclusions } = require('./utils/helpers');
@@ -16,6 +16,21 @@ const STATUS = {
   PENDING: 'PENDING',
 };
 
+// Sets up commander to use input arguments for this scripts from the CLI or GitHub Actions - CM
+const program = new Command();
+program
+  .option('-u, --url <url>', 'url to PR of file changes')
+  .option(
+    '-d, --directory <directory>',
+    'directory of files to be queued for translation'
+  )
+  .option(
+    '-mt, --machine-translation',
+    'Boolean to only send files needing machine translation'
+  );
+program.parse(process.argv);
+const options = program.opts();
+
 const translationDifference = (pendingFiles, prChanges) =>
   prChanges.filter(
     (file) =>
@@ -26,9 +41,6 @@ const translationDifference = (pendingFiles, prChanges) =>
       )
   );
 
-const slugIntersection = (pendingFiles, filesToRemove) =>
-  pendingFiles.filter((file) => filesToRemove.includes(file.slug));
-
 const humanTranslatedProjectID = process.env.HUMAN_TRANSLATION_PROJECT_ID;
 const machineTranslatedProjectID = process.env.MACHINE_TRANSLATION_PROJECT_ID;
 
@@ -42,14 +54,10 @@ const excludeFiles = (fileData) => {
   const exclusions = getExclusions();
 
   return fileData.filter(({ filename, locale, contentType }) => {
-    const localeKey = Object.keys(LOCALE_IDS).find(
-      (localeKey) => LOCALE_IDS[localeKey] === locale
-    );
     return (
-      !exclusions.excludePath[localeKey]?.some((path) =>
+      !exclusions.excludePath[locale]?.some((path) =>
         filename.startsWith(path)
-      ) &&
-      !exclusions.excludeType[localeKey]?.some((type) => contentType === type)
+      ) && !exclusions.excludeType[locale]?.some((type) => contentType === type)
     );
   });
 };
@@ -67,43 +75,56 @@ const getProjectId = (translateFM) => (locale) => {
     : machineTranslatedProjectID;
 };
 
-const getLocalizedFileData = (prFile) => {
-  const contents = fs.readFileSync(path.join(process.cwd(), prFile.filename));
+const getLocalizedFileData = (mdxFile) => {
+  const contents = fs.readFileSync(path.join(process.cwd(), mdxFile));
   const { data } = frontmatter(contents);
   const checkLocale = getProjectId(data.translate);
   const contentType = data.type;
 
   return Object.keys(LOCALE_IDS).map((locale) => ({
-    ...prFile,
+    filename: mdxFile,
     contentType,
     locale: LOCALE_IDS[locale],
     project_id: checkLocale(locale),
   }));
 };
 
-const removedFiles = (prFiles) =>
-  prFiles.filter((file) => file.status === 'removed').map(prop('filename'));
-
 /** Entrypoint. */
 const main = async () => {
-  checkArgs(3);
-  const url = process.argv[2];
+  // These come from the CLI input when using the script
+  const url = options.url || null;
+  const directory = options.directory || null;
+  const machineTranslation = options.machineTranslation || false;
 
-  const queue = await getTranslations({
-    status: STATUS.PENDING,
-  });
-  const prFileData = await fetchPaginatedGHResults(
-    url,
-    process.env.GITHUB_TOKEN
-  );
+  let mdxFileData;
 
-  const changedMdxFileData = prFileData
-    .filter((file) => path.extname(file.filename) === '.mdx')
-    .filter((f) => f.status !== 'removed');
+  if (url) {
+    const prFileData = await fetchPaginatedGHResults(
+      url,
+      process.env.GITHUB_TOKEN
+    );
+
+    mdxFileData = prFileData
+      .filter((file) => path.extname(file.filename) === '.mdx')
+      .filter((f) => f.status !== 'removed')
+      .map(prop('filename'));
+  } else if (directory) {
+    const directoryPath = path.join(directory, '/**/*.mdx');
+    mdxFileData = glob.sync(directoryPath);
+  }
 
-  const allLocalizedFileData = changedMdxFileData.flatMap(getLocalizedFileData);
-  const includedFiles = excludeFiles(allLocalizedFileData);
+  const allLocalizedFileData = mdxFileData.flatMap(getLocalizedFileData);
 
+  const filesToTranslate = machineTranslation
+    ? allLocalizedFileData.filter(
+        ({ project_id }) => project_id === machineTranslatedProjectID
+      )
+    : allLocalizedFileData;
+
+  const includedFiles = excludeFiles(filesToTranslate);
+  const queue = await getTranslations({
+    status: STATUS.PENDING,
+  });
   const fileDataToAddToQueue = translationDifference(queue, includedFiles);
 
   await Promise.all(
@@ -117,13 +138,6 @@ const main = async () => {
     )
   );
 
-  const translationIdsToRemove = slugIntersection(
-    queue,
-    removedFiles(prFileData)
-  ).map(prop('id'));
-
-  await Promise.all(translationIdsToRemove.map(deleteTranslation));
-
   process.exit(0);
 };
 

scripts/actions/translation_workflow/testing/connection_info.json

@@ -1,6 +1,6 @@
 {
-    "username": "",
-    "password": "",
-    "host": "",
-    "database": ""
+  "username": "",
+  "password": "",
+  "host": "",
+  "database": ""
 }

scripts/actions/translation_workflow/testing/script.sh

@@ -13,7 +13,7 @@ export GITHUB_TOKEN='' # token with no permissions
 
 # # step 1
 URL="https://api.github.com/repos/newrelic/docs-website/pulls/3271/files"
-yarn add-files-to-translate $URL
+yarn add-files-to-translate -u $URL
 
 # # step 2
 export TRANSLATION_VENDOR_API_URL=https://api.smartling.com

scripts/actions/utils/docs-content-tools/i18n-exclusions.yml

@@ -1,6 +1,13 @@
 excludePath:
   ja-JP:
-    - /whats-new
+    - src/content/whats-new
+    - src/content/docs/release-notes
+    - src/content/docs/licenses
+    - src/content/docs/style-guide
+    - src/content/docs/agile-handbook
+    - src/data-dictionary
+    - src/i18n
+    - src/content/docs/security/security-privacy/data-privacy
 excludeType:
   ja-JP:
-    - landingPage
+    - landingPage

yarn.lock

@@ -4566,6 +4566,11 @@ commander@^7.1.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
   integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
 
+commander@^8.3.0:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66"
+  integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
+
 commander@~2.9.0:
   version "2.9.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"