feat: get script working will add files to db

.github/workflows/add-slugs-to-translate-queue.yml

@@ -42,4 +42,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${{ github.event.pull_request.number }}/files"
-          yarn add-files-to-translate $URL
+          yarn add-files-to-translate -u $URL

package.json

@@ -75,6 +75,7 @@
     "aws-sdk": "^2.827.0",
     "babel-preset-gatsby": "^1.3.0",
     "chalk": "^4.1.0",
+    "commander": "^8.3.0",
     "dotenv": "^8.2.0",
     "eslint": "^7.7.0",
     "eslint-plugin-jsx-a11y": "^6.3.1",

scripts/actions/add-files-to-translation-queue.js

@@ -1,13 +1,13 @@
 const fs = require('fs');
 const path = require('path');
 const frontmatter = require('@github-docs/frontmatter');
+const { Command } = require('commander');
+const glob = require('glob');
 const {
   getTranslations,
   addTranslation,
-  deleteTranslation,
 } = require('./translation_workflow/database');
 const { fetchPaginatedGHResults } = require('./utils/github-api-helpers');
-const checkArgs = require('./utils/check-args');
 const { prop } = require('../utils/functional');
 const { LOCALE_IDS } = require('./utils/constants');
 const { getExclusions } = require('./utils/helpers');
@@ -16,6 +16,21 @@ const STATUS = {
   PENDING: 'PENDING',
 };
 
+// Sets up commander to use input arguments for this scripts from the CLI or GitHub Actions - CM
+const program = new Command();
+program
+  .option('-u, --url <url>', 'url to PR of file changes')
+  .option(
+    '-d, --directory <directory>',
+    'directory of files to be queued for translation'
+  )
+  .option(
+    '-mt, --machine-translation',
+    'Boolean to only send files needing machine translation'
+  );
+program.parse(process.argv);
+const options = program.opts();
+
 const translationDifference = (pendingFiles, prChanges) =>
   prChanges.filter(
     (file) =>
@@ -26,9 +41,6 @@ const translationDifference = (pendingFiles, prChanges) =>
       )
   );
 
-const slugIntersection = (pendingFiles, filesToRemove) =>
-  pendingFiles.filter((file) => filesToRemove.includes(file.slug));
-
 const humanTranslatedProjectID = process.env.HUMAN_TRANSLATION_PROJECT_ID;
 const machineTranslatedProjectID = process.env.MACHINE_TRANSLATION_PROJECT_ID;
 
@@ -67,43 +79,56 @@ const getProjectId = (translateFM) => (locale) => {
     : machineTranslatedProjectID;
 };
 
-const getLocalizedFileData = (prFile) => {
-  const contents = fs.readFileSync(path.join(process.cwd(), prFile.filename));
+const getLocalizedFileData = (mdxFile) => {
+  const contents = fs.readFileSync(path.join(process.cwd(), mdxFile));
   const { data } = frontmatter(contents);
   const checkLocale = getProjectId(data.translate);
   const contentType = data.type;
 
   return Object.keys(LOCALE_IDS).map((locale) => ({
-    ...prFile,
+    filename: mdxFile,
     contentType,
     locale: LOCALE_IDS[locale],
     project_id: checkLocale(locale),
   }));
 };
 
-const removedFiles = (prFiles) =>
-  prFiles.filter((file) => file.status === 'removed').map(prop('filename'));
-
 /** Entrypoint. */
 const main = async () => {
-  checkArgs(3);
-  const url = process.argv[2];
+  // These come from the CLI input when using the script
+  const url = options.url || null;
+  const directory = options.directory || null;
+  const machineTranslation = options.machineTranslation || false;
 
-  const queue = await getTranslations({
-    status: STATUS.PENDING,
-  });
-  const prFileData = await fetchPaginatedGHResults(
-    url,
-    process.env.GITHUB_TOKEN
-  );
+  let mdxFileData;
 
-  const changedMdxFileData = prFileData
-    .filter((file) => path.extname(file.filename) === '.mdx')
-    .filter((f) => f.status !== 'removed');
+  if (url) {
+    const prFileData = await fetchPaginatedGHResults(
+      url,
+      process.env.GITHUB_TOKEN
+    );
 
-  const allLocalizedFileData = changedMdxFileData.flatMap(getLocalizedFileData);
-  const includedFiles = excludeFiles(allLocalizedFileData);
+    mdxFileData = prFileData
+      .filter((file) => path.extname(file.filename) === '.mdx')
+      .filter((f) => f.status !== 'removed')
+      .map(prop('filename'));
+  } else if (directory) {
+    const directoryPath = path.join(directory, '/**/*.mdx');
+    mdxFileData = glob.sync(directoryPath);
+  }
 
+  const allLocalizedFileData = mdxFileData.flatMap(getLocalizedFileData);
+
+  const filesToTranslate = machineTranslation
+    ? allLocalizedFileData.filter(
+        ({ project_id }) => project_id === machineTranslatedProjectID
+      )
+    : allLocalizedFileData;
+
+  const includedFiles = excludeFiles(filesToTranslate);
+  const queue = await getTranslations({
+    status: STATUS.PENDING,
+  });
   const fileDataToAddToQueue = translationDifference(queue, includedFiles);
 
   await Promise.all(
@@ -117,13 +142,6 @@ const main = async () => {
     )
   );
 
-  const translationIdsToRemove = slugIntersection(
-    queue,
-    removedFiles(prFileData)
-  ).map(prop('id'));
-
-  await Promise.all(translationIdsToRemove.map(deleteTranslation));
-
   process.exit(0);
 };
 

scripts/actions/utils/docs-content-tools/i18n-exclusions.yml

@@ -1,6 +1,11 @@
 excludePath:
   ja-JP:
     - /whats-new
+    - src/content/docs/release-notes
+    - src/content/docs/licenses
+    - src/content/docs/style-guide
+    - src/content/docs/agile-handbook
+    - src/data-dictionary
 excludeType:
   ja-JP:
-    - landingPage
+    - landingPage