Merge pull request #5181 from newrelic/feature/machine-translation

machine translation feature
newrelic · Dec 15, 2021 · 2d453b4 · 2d453b4
2 parents 9b092ca + 4dd61f3
commit 2d453b4
Show file tree

Hide file tree

Showing 42 changed files with 1,035 additions and 167 deletions.
diff --git a/.github/workflows/get-slugs-to-translate.yml → ...orkflows/add-slugs-to-translate-queue.yml b/.github/workflows/get-slugs-to-translate.yml → ...orkflows/add-slugs-to-translate-queue.yml
@@ -11,9 +11,11 @@ on:
 
 env:
   DB_CONNECTION_INFO: ${{ secrets.DB_CONNECTION_INFO }}
+  HUMAN_TRANSLATION_PROJECT_ID: ${{ secrets.HUMAN_TRANSLATION_PROJECT_ID }}
+  MACHINE_TRANSLATION_PROJECT_ID: ${{ secrets.MACHINE_TRANSLATION_PROJECT_ID }}
 
 jobs:
-  get-slugs:
+  get-and-save-slugs:
     name: Get and Save Slugs
     runs-on: ubuntu-latest
     if: github.event.pull_request.merged == true
@@ -40,4 +42,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${{ github.event.pull_request.number }}/files"
-          yarn get-translated-files $URL
+          yarn add-files-to-translate $URL
diff --git a/.github/workflows/check-translations-and-deserialize.yml b/.github/workflows/check-translations-and-deserialize.yml
@@ -8,9 +8,6 @@ on:
 
 env:
   TRANSLATION_VENDOR_API_URL: ${{ secrets.TRANSLATION_VENDOR_API_URL }}
-  TRANSLATION_VENDOR_PROJECT: ${{ secrets.TRANSLATION_VENDOR_PROJECT }}
-  TRANSLATION_VENDOR_USER: ${{ secrets.TRANSLATION_VENDOR_USER }}
-  TRANSLATION_VENDOR_SECRET: ${{ secrets.TRANSLATION_VENDOR_SECRET }}
   DB_CONNECTION_INFO: ${{ secrets.DB_CONNECTION_INFO }}
   BOT_NAME: nr-opensource-bot
   BOT_EMAIL: opensource+bot@newrelic.com
@@ -20,6 +17,11 @@ jobs:
   fetch-content:
     name: Fetch translated content
     runs-on: ubuntu-latest
+    env:
+      TRANSLATION_VENDOR_PROJECT: ${{ secrets.TRANSLATION_VENDOR_PROJECT }}
+      TRANSLATION_VENDOR_USER: ${{ secrets.TRANSLATION_VENDOR_USER }}
+      TRANSLATION_VENDOR_SECRET: ${{ secrets.TRANSLATION_VENDOR_SECRET }}
+      TRANSLATION_TYPE: human
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -69,15 +71,36 @@ jobs:
           branch-suffix: short-commit-hash
           base: develop
           delete-branch: true
+          reviewers: jmiraNR
+          team-reviewers: developer-enablement
 
-      - name: Checkout repository
+      - name: Remove Completed Batches From Queue
         if: steps.fetch-deserialize.outputs.batchesToDeserialize != 0
+        id: remove-batches
+        run: |
+          node ./scripts/actions/remove-completed-batch.js
+
+  fetch-machine-translated-content:
+    name: Fetch machine translated content
+    runs-on: ubuntu-latest
+    env:
+      TRANSLATION_VENDOR_PROJECT: ${{ secrets.TRANSLATION_VENDOR_MT_PROJECT }}
+      TRANSLATION_VENDOR_USER: ${{ secrets.TRANSLATION_VENDOR_MT_USER }}
+      TRANSLATION_VENDOR_SECRET: ${{ secrets.TRANSLATION_VENDOR_MT_SECRET }}
+      TRANSLATION_TYPE: machine
+    steps:
+      - name: Checkout repository
         uses: actions/checkout@v2
 
       - name: Setup Node.js
         uses: actions/setup-node@v2
 
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
       - name: Cache dependencies
+        id: yarn-cache
         uses: actions/cache@v2
         with:
           path: '**/node_modules'
@@ -87,6 +110,36 @@ jobs:
         if: steps.yarn-cache.outputs.cache-hit != 'true'
         run: yarn install --frozen-lockfile
 
+      - name: Fetches translated content and deserializes
+        id: fetch-deserialize
+        run: |
+          node ./scripts/actions/check-job-progress.js
+
+      - name: Commit changes
+        id: commit-changes
+        if: steps.fetch-deserialize.outputs.batchesToDeserialize != 0
+        run: |
+          git config --local user.email "${{ env.BOT_EMAIL }}"
+          git config --local user.name "${{ env.BOT_NAME }}"
+          git add ./src/i18n/content
+          git commit -m 'chore: add translations'
+          echo "::set-output name=commit::true"
+
+      - name: Create Pull Request
+        if: steps.fetch-deserialize.outputs.batchesToDeserialize != 0
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.OPENSOURCE_BOT_TOKEN }}
+          title: Updated translations - ${{ steps.date.outputs.date }} (machine translation)
+          body: ''
+          labels: translations
+          branch: translations
+          branch-suffix: short-commit-hash
+          base: develop
+          delete-branch: true
+          reviewers: jmiraNR
+          team-reviewers: developer-enablement
+
       - name: Remove Completed Batches From Queue
         if: steps.fetch-deserialize.outputs.batchesToDeserialize != 0
         id: remove-batches

diff --git a/.github/workflows/send-content-to-machine-translate.yml b/.github/workflows/send-content-to-machine-translate.yml
@@ -0,0 +1,39 @@
+name: Send content to be translated
+
+on:
+  workflow_dispatch:
+  schedule:
+    # At 1 am everyday
+    - cron: '0 1 * * *'
+
+env:
+  TRANSLATION_VENDOR_API_URL: ${{ secrets.TRANSLATION_VENDOR_API_URL }}
+  TRANSLATION_VENDOR_PROJECT: ${{ secrets.TRANSLATION_VENDOR_MT_PROJECT }}
+  TRANSLATION_VENDOR_USER: ${{ secrets.TRANSLATION_VENDOR_MT_USER }}
+  TRANSLATION_VENDOR_SECRET: ${{ secrets.TRANSLATION_VENDOR_MT_SECRET }}
+  DB_CONNECTION_INFO: ${{ secrets.DB_CONNECTION_INFO }}
+
+jobs:
+  send-content:
+    name: Send content
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+
+      - name: Cache dependencies
+        id: yarn-cache
+        uses: actions/cache@v2
+        with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-node-modules-${{ hashFiles('**/yarn.lock') }}
+
+      - name: Install dependencies
+        if: steps.yarn-cache.outputs.cache-hit != 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Get content to be translated and send to vendor
+        run: node ./scripts/actions/send-and-update-translation-queue.js
diff --git a/gatsby-node.js b/gatsby-node.js
@@ -113,6 +113,7 @@ exports.createPages = async ({ actions, graphql, reporter }) => {
             frontmatter {
               type
               subject
+              translationType
             }
           }
         }
@@ -291,8 +292,12 @@ exports.createSchemaCustomization = ({ actions }) => {
   type MarkdownRemark implements Node {
     frontmatter: Frontmatter
   }
+  type Mdx implements Node {
+    frontmatter: Frontmatter
+  }
   type Frontmatter {
     isFeatured: Boolean
+    translationType: String
   }
   `;
 
@@ -321,6 +326,12 @@ exports.createResolvers = ({ createResolvers }) => {
         resolve: (source) =>
           hasOwnProperty(source, 'isFeatured') ? source.isFeatured : false,
       },
+      translationType: {
+        resolve: (source) =>
+          hasOwnProperty(source, 'translationType')
+            ? source.translationType
+            : null,
+      },
     },
   });
 };

diff --git a/loadershim.js b/loadershim.js
@@ -1,3 +1,7 @@
+process.env.HUMAN_TRANSLATION_PROJECT_ID = 'HT_ID';
+process.env.MACHINE_TRANSLATION_PROJECT_ID = 'MT_ID';
+process.env.DB_CONNECTION_INFO = '{}';
+
 global.___loader = {
   enqueue: jest.fn(),
 };
diff --git a/package.json b/package.json
@@ -9,7 +9,7 @@
     "@github-docs/frontmatter": "^1.3.1",
     "@mdx-js/mdx": "^2.0.0-next.8",
     "@mdx-js/react": "^2.0.0-next.8",
-    "@newrelic/gatsby-theme-newrelic": "^4.0.5",
+    "@newrelic/gatsby-theme-newrelic": "^4.1.6",
     "@splitsoftware/splitio-react": "^1.2.4",
     "babel-jest": "^26.3.0",
     "common-tags": "^1.8.0",
@@ -150,10 +150,12 @@
     "codemod": "node scripts/codemod.js",
     "postinstall": "patch-package",
     "extract-i18n": "i18next",
-    "get-translated-files": "node scripts/actions/add-files-to-translation-queue.js",
+    "add-files-to-translate": "node scripts/actions/add-files-to-translation-queue.js",
     "check-for-outdated-translations": "node scripts/actions/check-for-outdated-translations.js",
     "verify-mdx": "node scripts/verify_mdx.js",
-    "add-remove-redirects": "node scripts/utils/docs-content-tools/add-remove-redirects.js"
+    "add-remove-redirects": "node scripts/utils/docs-content-tools/add-remove-redirects.js",
+    "db:start": "docker-compose --file scripts/actions/translation_workflow/testing/docker-compose.yml up --build",
+    "db:clean": "./scripts/actions/translation_workflow/testing/cleanup.sh"
   },
   "husky": {
     "hooks": {

diff --git a/scripts/actions/__tests__/__snapshots__/serialize-mdx.test.js.snap b/scripts/actions/__tests__/__snapshots__/serialize-mdx.test.js.snap
@@ -2,7 +2,7 @@
 
 exports[`kitchen sink 1`] = `
 "
-<div data-type=\\"frontmatter\\" data-value=\\"eyJ0aXRsZSI6IkVuYWJsZSBzZXJ2ZXJsZXNzIG1vbml0b3JpbmcgZm9yIEFXUyBMYW1iZGEiLCJjb250ZW50VHlwZSI6InBhZ2UiLCJ0ZW1wbGF0ZSI6ImJhc2ljRG9jIiwidG9waWNzIjpbXSwid2F0ZXJtYXJrIjoiQkVUQSJ9\\">
+<div data-type=\\"frontmatter\\" data-value=\\"eyJ0aXRsZSI6IkVuYWJsZSBzZXJ2ZXJsZXNzIG1vbml0b3JpbmcgZm9yIEFXUyBMYW1iZGEiLCJjb250ZW50VHlwZSI6InBhZ2UiLCJ0ZW1wbGF0ZSI6ImJhc2ljRG9jIiwidG9waWNzIjpbXSwid2F0ZXJtYXJrIjoiQkVUQSIsInRyYW5zbGF0aW9uVHlwZSI6Imh1bWFuIn0=\\">
   <div data-key=\\"title\\">Enable serverless monitoring for AWS Lambda</div>
 </div>
 <div data-type=\\"import\\" data-value=\\"aW1wb3J0IHsgTGluayB9IGZyb20gJ0BuZXdyZWxpYy9nYXRzYnktdGhlbWUtbmV3cmVsaWMn\\"></div>
@@ -39,7 +39,7 @@ exports[`kitchen sink 1`] = `
 <table data-type=\\"component\\">
   <thead data-type=\\"component\\">
     <tr data-type=\\"component\\">
-      <th data-type=\\"component\\" data-props=\\"W3sidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJzdHlsZSIsInZhbHVlIjp7InR5cGUiOiJtZHhWYWx1ZUV4cHJlc3Npb24iLCJ2YWx1ZSI6Insgd2lkdGg6IFwiMjAwcHhcIiB9IiwicG9zaXRpb24iOnsic3RhcnQiOnsibGluZSI6NTAsImNvbHVtbiI6MTcsIm9mZnNldCI6MjE1OH0sImVuZCI6eyJsaW5lIjo1MCwiY29sdW1uIjozNywib2Zmc2V0IjoyMTc4fX19fV0=\\">
+      <th data-type=\\"component\\" data-props=\\"W3sidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJzdHlsZSIsInZhbHVlIjp7InR5cGUiOiJtZHhWYWx1ZUV4cHJlc3Npb24iLCJ2YWx1ZSI6Insgd2lkdGg6IFwiMjAwcHhcIiB9IiwicG9zaXRpb24iOnsic3RhcnQiOnsibGluZSI6NTEsImNvbHVtbiI6MTcsIm9mZnNldCI6MjE4MX0sImVuZCI6eyJsaW5lIjo1MSwiY29sdW1uIjozNywib2Zmc2V0IjoyMjAxfX19fV0=\\">
         <p><strong>If you want to...</strong></p>
       </th>
       <th data-type=\\"component\\">
@@ -74,7 +74,7 @@ exports[`kitchen sink 1`] = `
     </tr>
   </tbody>
 </table>
-<pre data-type=\\"component\\" data-component=\\"CodeBlock\\" data-props=\\"eyJsYW5nIjoicHl0aG9uIiwibWV0YSI6ImNvcHlhYmxlPWZhbHNlIiwidmFsdWUiOiJAbmV3cmVsaWMuYWdlbnQubGFtYmRhX2hhbmRsZXIoKVxuZGVmIGhhbmRsZXIoZXZlbnQsIGNvbnRleHQpOlxuICAgIG5ld3JlbGljLmFnZW50LnJlY29yZF9jdXN0b21fZXZlbnQoJ0N1c3RvbUV2ZW50Jywgeydmb28nOiAnYmFyJ30pIiwicG9zaXRpb24iOnsic3RhcnQiOnsibGluZSI6OTMsImNvbHVtbiI6MSwib2Zmc2V0IjozMTM5fSwiZW5kIjp7ImxpbmUiOjk3LCJjb2x1bW4iOjQsIm9mZnNldCI6MzI5OX0sImluZGVudCI6WzEsMSwxLDFdfX0=\\"><code></code></pre>
+<pre data-type=\\"component\\" data-component=\\"CodeBlock\\" data-props=\\"eyJsYW5nIjoicHl0aG9uIiwibWV0YSI6ImNvcHlhYmxlPWZhbHNlIiwidmFsdWUiOiJAbmV3cmVsaWMuYWdlbnQubGFtYmRhX2hhbmRsZXIoKVxuZGVmIGhhbmRsZXIoZXZlbnQsIGNvbnRleHQpOlxuICAgIG5ld3JlbGljLmFnZW50LnJlY29yZF9jdXN0b21fZXZlbnQoJ0N1c3RvbUV2ZW50Jywgeydmb28nOiAnYmFyJ30pIiwicG9zaXRpb24iOnsic3RhcnQiOnsibGluZSI6OTQsImNvbHVtbiI6MSwib2Zmc2V0IjozMTYyfSwiZW5kIjp7ImxpbmUiOjk4LCJjb2x1bW4iOjQsIm9mZnNldCI6MzMyMn0sImluZGVudCI6WzEsMSwxLDFdfX0=\\"></pre>
 <div data-type=\\"component\\" data-component=\\"TechTileGrid\\">
   <div data-type=\\"prop\\" data-prop=\\"children\\">
     <div data-type=\\"component\\" data-component=\\"TechTile\\" data-props=\\"W3sidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJuYW1lIiwidmFsdWUiOiJpT1MifSx7InR5cGUiOiJtZHhBdHRyaWJ1dGUiLCJuYW1lIjoiaWNvbiIsInZhbHVlIjoibG9nby1hcHBsZSJ9LHsidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJ0byIsInZhbHVlIjoiL2FnZW50cy9pb3MtYWdlbnQifV0=\\">
@@ -94,14 +94,14 @@ exports[`kitchen sink 1`] = `
       <div data-type=\\"prop\\" data-prop=\\"title\\">What account changes have been made using an API key?</div>
       <div data-type=\\"prop\\" data-prop=\\"children\\">
         <p>To see detailed information about changes to the account that were made using an API key during a specific time frame, include <a href=\\"#actorType\\"><code>actorType = 'api_key'</code></a> in the query. For example:</p>
-        <pre data-type=\\"component\\" data-component=\\"CodeBlock\\" data-props=\\"eyJsYW5nIjpudWxsLCJtZXRhIjpudWxsLCJ2YWx1ZSI6IlNFTEVDVCBhY3Rpb25JZGVudGlmaWVyLCBkZXNjcmlwdGlvbiwgdGFyZ2V0VHlwZSwgdGFyZ2V0SWQsIGFjdG9yQVBJS2V5LCBhY3RvcklkLCBhY3RvckVtYWlsXG5GUk9NIE5yQXVkaXRFdmVudCBXSEVSRSBhY3RvclR5cGUgPSAnYXBpX2tleScgU0lOQ0UgMSB3ZWVrIGFnbyIsInBvc2l0aW9uIjp7InN0YXJ0Ijp7ImxpbmUiOjEyNiwiY29sdW1uIjoxLCJvZmZzZXQiOjM5MTF9LCJlbmQiOnsibGluZSI6MTMwLCJjb2x1bW4iOjMsIm9mZnNldCI6NDA5M30sImluZGVudCI6WzEsMSwxLDFdfX0=\\"><code></code></pre>
+        <pre data-type=\\"component\\" data-component=\\"CodeBlock\\" data-props=\\"eyJsYW5nIjpudWxsLCJtZXRhIjpudWxsLCJ2YWx1ZSI6IlNFTEVDVCBhY3Rpb25JZGVudGlmaWVyLCBkZXNjcmlwdGlvbiwgdGFyZ2V0VHlwZSwgdGFyZ2V0SWQsIGFjdG9yQVBJS2V5LCBhY3RvcklkLCBhY3RvckVtYWlsXG5GUk9NIE5yQXVkaXRFdmVudCBXSEVSRSBhY3RvclR5cGUgPSAnYXBpX2tleScgU0lOQ0UgMSB3ZWVrIGFnbyIsInBvc2l0aW9uIjp7InN0YXJ0Ijp7ImxpbmUiOjEyNywiY29sdW1uIjoxLCJvZmZzZXQiOjM5MzR9LCJlbmQiOnsibGluZSI6MTMxLCJjb2x1bW4iOjMsIm9mZnNldCI6NDExNn0sImluZGVudCI6WzEsMSwxLDFdfX0=\\"></pre>
       </div>
     </div>
     <div data-type=\\"component\\" data-component=\\"Collapser\\" data-props=\\"W3sidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJpZCIsInZhbHVlIjoic3ludGgtdXNlciJ9LHsidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJ0aXRsZSIsInZhbHVlIjoiU3ludGhldGljczogV2hhdCBtb25pdG9ycyB3ZXJlIGNyZWF0ZWQgYnkgYSBzcGVjaWZpYyB1c2VyPyJ9XQ==\\">
       <div data-type=\\"prop\\" data-prop=\\"title\\">Synthetics: What monitors were created by a specific user?</div>
       <div data-type=\\"prop\\" data-prop=\\"children\\">
         <p>To query Synthetics monitor updates made by a specific user, include the <a href=\\"/attribute-dictionary/nrauditevent/actionidentifier\\"><code>actionIdentifier</code></a> and <a href=\\"/attribute-dictionary/nrauditevent/actoremail\\"><code>actorEmail</code></a> attribute in your query. For example:</p>
-        <pre data-type=\\"component\\" data-component=\\"CodeBlock\\" data-props=\\"eyJsYW5nIjpudWxsLCJtZXRhIjpudWxsLCJ2YWx1ZSI6IlNFTEVDVCBjb3VudCgqKSBGUk9NIE5yQXVkaXRFdmVudFxuV0hFUkUgYWN0aW9uSWRlbnRpZmllciA9ICdzeW50aGV0aWNzX21vbml0b3IudXBkYXRlX3NjcmlwdCdcbkZBQ0VUIGFjdG9yRW1haWwsIGFjdGlvbklkZW50aWZpZXIsIGRlc2NyaXB0aW9uXG5TSU5DRSAxIHdlZWsgYWdvIExJTUlUIDEwMDAiLCJwb3NpdGlvbiI6eyJzdGFydCI6eyJsaW5lIjoxMzgsImNvbHVtbiI6MSwib2Zmc2V0Ijo0NDcxfSwiZW5kIjp7ImxpbmUiOjE0NCwiY29sdW1uIjozLCJvZmZzZXQiOjQ2NzV9LCJpbmRlbnQiOlsxLDEsMSwxLDEsMV19fQ==\\"><code></code></pre>
+        <pre data-type=\\"component\\" data-component=\\"CodeBlock\\" data-props=\\"eyJsYW5nIjpudWxsLCJtZXRhIjpudWxsLCJ2YWx1ZSI6IlNFTEVDVCBjb3VudCgqKSBGUk9NIE5yQXVkaXRFdmVudFxuV0hFUkUgYWN0aW9uSWRlbnRpZmllciA9ICdzeW50aGV0aWNzX21vbml0b3IudXBkYXRlX3NjcmlwdCdcbkZBQ0VUIGFjdG9yRW1haWwsIGFjdGlvbklkZW50aWZpZXIsIGRlc2NyaXB0aW9uXG5TSU5DRSAxIHdlZWsgYWdvIExJTUlUIDEwMDAiLCJwb3NpdGlvbiI6eyJzdGFydCI6eyJsaW5lIjoxMzksImNvbHVtbiI6MSwib2Zmc2V0Ijo0NDk0fSwiZW5kIjp7ImxpbmUiOjE0NSwiY29sdW1uIjozLCJvZmZzZXQiOjQ2OTh9LCJpbmRlbnQiOlsxLDEsMSwxLDEsMV19fQ==\\"></pre>
       </div>
     </div>
   </div>
@@ -250,6 +250,34 @@ exports[`serializes CollapserGroup to html 1`] = `
 "
 `;
 
+exports[`serializes DoNotTranslate to html 1`] = `
+"
+<div data-type=\\"component\\" data-component=\\"DoNotTranslate\\" class=\\"notranslate\\">
+  <h1>Not all who wander are lost...</h1>
+  <p>Testing this line too</p>
+</div>
+"
+`;
+
+exports[`serializes DoNotTranslate to html inline 1`] = `
+"
+<p>This is an <span data-type=\\"component\\" data-component=\\"DoNotTranslate\\" class=\\"notranslate\\">MDX</span> file</p>
+"
+`;
+
+exports[`serializes DoNotTranslate wrapping a Collapser 1`] = `
+"
+<div data-type=\\"component\\" data-component=\\"DoNotTranslate\\" class=\\"notranslate\\">
+  <div data-type=\\"component\\" data-component=\\"Collapser\\" data-props=\\"W3sidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJ0aXRsZSIsInZhbHVlIjoiQ29sbGFwc2UgbWUgeW8ifV0=\\">
+    <div data-type=\\"prop\\" data-prop=\\"title\\">Collapse me yo</div>
+    <div data-type=\\"prop\\" data-prop=\\"children\\">
+      <p>These tests are hard to write docs for</p>
+    </div>
+  </div>
+</div>
+"
+`;
+
 exports[`serializes ExternalLink to html 1`] = `
 "
 <div data-type=\\"component\\" data-component=\\"ExternalLink\\" data-props=\\"W3sidHlwZSI6Im1keEF0dHJpYnV0ZSIsIm5hbWUiOiJocmVmIiwidmFsdWUiOiJodHRwczovL2RldmVsb3Blci5uZXdyZWxpYy5jb20ifV0=\\">