[DO NOT MERGE] Rewrite learning uploader

.github/workflows/deploy-tutorials.yml

@@ -25,15 +25,14 @@ jobs:
     environment: ${{ inputs.environment }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
         with:
-          python-version: "3.11"
-      - name: Install tool
-        run: pip install ./scripts/ibm-quantum-learning-uploader
+          node-version: 18
+      - name: Install Node.js dependencies
+        run: npm ci
       - name: Upload tutorials
-        run: |
-          cd tutorials
-          sync-lessons
+        run: npm run tutorial:sync
         env:
           LEARNING_API_TOKEN: ${{ secrets.LEARNING_API_TOKEN }}
-          LEARNING_API_ENVIRONMENT: ${{ vars.LEARNING_API_ENVIRONMENT }}
+          LEARNING_API_URL: ${{ vars.LEARNING_API_URL }}

package.json

@@ -16,11 +16,15 @@
     "check:qiskit-bot": "node -r esbuild-register scripts/commands/checkQiskitBotFiles.ts",
     "check:stale-images": "node -r esbuild-register scripts/commands/checkStaleImages.ts",
     "fmt": "prettier --write .",
-    "test": "jest",
+    "test": "jest --testPathIgnorePatterns='tutorial-uploader'",
+    "test:tutorial-uploader": "jest scripts/tutorial-uploader",
     "typecheck": "tsc",
     "regen-apis": "node -r esbuild-register scripts/commands/regenerateApiDocs.ts",
     "gen-api": "node -r esbuild-register scripts/commands/updateApiDocs.ts",
-    "make-historical": "node -r esbuild-register scripts/commands/convertApiDocsToHistorical.ts"
+    "make-historical": "node -r esbuild-register scripts/commands/convertApiDocsToHistorical.ts",
+    "tutorial:sync": "node -r esbuild-register scripts/tutorial-uploader/sync.ts",
+    "tutorial:validate": "node -r esbuild-register scripts/tutorial-uploader/validate-config.ts",
+    "tutorial:setup-testing": "node -r esbuild-register scripts/tutorial-uploader/setup-for-testing.ts"
   },
   "devDependencies": {
     "@swc/jest": "^0.2.29",
@@ -59,8 +63,10 @@
     "zx": "^7.2.3"
   },
   "dependencies": {
+    "@directus/sdk": "^16.0.1",
     "esbuild": "^0.19.11",
     "fast-levenshtein": "^3.0.0",
+    "js-yaml": "^4.1.0",
     "markdown-link-extractor": "^3.1.0",
     "transform-markdown-links": "^2.1.0"
   }

scripts/nb-tester/notebooks.toml

@@ -34,6 +34,7 @@ notebooks_normal_test = [
 # Don't test the following notebooks (this section can include glob patterns)
 notebooks_exclude = [
     "scripts/ibm-quantum-learning-uploader/test/template.ipynb",
+    "scripts/tutorial-uploader/lib/test-data/simple-tutorial/notebook.ipynb",
     "**/.ipynb_checkpoints/**",
     # The following notebooks are broken and need fixing
     "tutorials/submitting-transpiled-circuits/submitting-transpiled-circuits.ipynb",

scripts/tutorial-uploader/README.md

@@ -0,0 +1,40 @@
+# Tutorial uploader
+
+This script uploads tutorials to IBM Quantum Learning.
+
+## Developing
+
+To work on this script, you'll need to set up the `saiba-api` project locally.
+
+There are some extra steps you'll need to take to set up `saiba-api` for
+developing this script:
+
+- Follow the instructions in the README in the saiba-api repo. Don't forget the
+  instructions to add the `PUBLIC_URL` entry to your `docker-compose.yaml`.
+
+- Login into the local CMS (<http://0.0.0.0:8055/admin/>) using
+
+  - email: `admin@example.com`
+  - password: `password`
+
+- Create a token for local testing.
+
+  1. In the local CMS, go to "User directory" (in the leftmost navbar)
+  2. Click "Create item"
+  3. Create a new user with the "Content creator admin" role and generate a new
+     static token. Copy the token to your clipboard. Then click the tick on the
+     top-right of the page to save the user.
+  4. To test the script in this repo, export the following environment
+     variables.
+     ```
+     export LEARNING_API_URL=http://0.0.0.0:8055
+     export LEARNING_API_TOKEN=<copied-token>
+     ```
+     Consider using [direnv](https://direnv.net/) to handle this.
+
+- With the local database running, run the following command to add the topics
+  and categories that we expect to exist.
+
+  ```
+  npm run tutorial:setup-testing
+  ```

scripts/tutorial-uploader/lib/api.test.ts

@@ -0,0 +1,151 @@
+// This code is a Qiskit project.
+//
+// (C) Copyright IBM 2024.
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE file in the root directory
+// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+import { describe, expect, test } from "@jest/globals";
+import { readItem } from "@directus/sdk";
+
+import { API } from "./api";
+import { type LocalTutorialData } from "./local-tutorial-data";
+
+/* Create test data */
+const MOCK_TUTORIAL: LocalTutorialData = {
+  title: "Mock tutorial for testing",
+  short_description: "A mock tutorial for testing with Jest",
+  slug: "mock-tutorial",
+  status: "published",
+  local_path: "scripts/tutorial-uploader/lib/test-data/simple-tutorial",
+  category: "Workflow example",
+  topics: [],
+  reading_time: 50,
+  catalog_featured: false,
+};
+
+/* Just to be sure */
+if (/learning-api\.quantum\.ibm\.com/.test(process.env.LEARNING_API_URL!)) {
+  throw new Error(
+    "Tried to run tests against production! Set the env var LEARNING_API_URL to either staging or local (see tutorial-uploader/README.md)",
+  );
+}
+
+describe("Tutorial uploader API", () => {
+  const api = new API();
+
+  test("upload new tutorial", async () => {
+    expect(await api.getTutorialIdBySlug(MOCK_TUTORIAL.slug)).toBeNull();
+
+    await api.upsertTutorial(MOCK_TUTORIAL);
+    const tutorialId = await api.getTutorialIdBySlug(MOCK_TUTORIAL.slug);
+    expect(tutorialId).toBeTruthy();
+
+    const retrievedTutorial = await api.client.request(
+      // @ts-ignore
+      readItem("tutorials", tutorialId, { fields: ["*", "translations.*"] }),
+    );
+    expect(retrievedTutorial).toMatchObject({
+      slug: MOCK_TUTORIAL.slug,
+      status: MOCK_TUTORIAL.status,
+      reading_time: MOCK_TUTORIAL.reading_time,
+      catalog_featured: MOCK_TUTORIAL.catalog_featured,
+      category: await api.getId(
+        "tutorials_categories",
+        "name",
+        MOCK_TUTORIAL.category,
+      ),
+      topics: [],
+      editors: [],
+      allowed_email_domains: null,
+      required_instance_access: null,
+      sort: null,
+      translations: [
+        {
+          title: MOCK_TUTORIAL.title,
+          short_description: MOCK_TUTORIAL.short_description,
+          content: "Here's some basic content.\n",
+          languages_code: "en-US",
+        },
+      ],
+    });
+  });
+
+  test("update existing tutorial", async () => {
+    // Upload tutorial
+    await api.upsertTutorial(MOCK_TUTORIAL);
+    const tutorialId = await api.getTutorialIdBySlug(MOCK_TUTORIAL.slug);
+
+    // Mutate tutorial data and re-upload
+    const modifiedTutorial = {
+      ...MOCK_TUTORIAL,
+      title: "A new tutorial title",
+      short_description: "A modified short description",
+      status: "draft",
+      category: "How-to",
+      reading_time: 33,
+      topics: ["Scheduling", "Transpilation"],
+      catalog_featured: false,
+    };
+    await api.upsertTutorial(modifiedTutorial);
+
+    // Retrieve and check
+    const retrievedTutorial = await api.client.request(
+      // @ts-ignore
+      readItem("tutorials", tutorialId, { fields: ["*", "topics.*"] }),
+    );
+    const topicIds = (await Promise.all(
+      modifiedTutorial.topics.map((name) =>
+        api.getId("tutorials_topics", "name", name),
+      ),
+    )) as string[];
+    expect(retrievedTutorial).toMatchObject({
+      slug: modifiedTutorial.slug,
+      status: modifiedTutorial.status,
+      reading_time: modifiedTutorial.reading_time,
+      catalog_featured: modifiedTutorial.catalog_featured,
+      category: await api.getId(
+        "tutorials_categories",
+        "name",
+        modifiedTutorial.category,
+      ),
+      topics: topicIds.map((name) => {
+        return { tutorials_topics_id: name };
+      }),
+      editors: [],
+      allowed_email_domains: null,
+      required_instance_access: null,
+      sort: null,
+    });
+
+    const retrievedTranslation = (
+      await api.client.request(
+        // @ts-ignore
+        readItem("tutorials", tutorialId, { fields: ["translations.*"] }),
+      )
+    ).translations;
+    expect(retrievedTranslation).toMatchObject([
+      {
+        title: modifiedTutorial.title,
+        short_description: modifiedTutorial.short_description,
+        content: "Here's some basic content.\n",
+        languages_code: "en-US",
+      },
+    ]);
+  });
+
+  beforeAll(async () => {
+    if (await api.getTutorialIdBySlug(MOCK_TUTORIAL.slug)) {
+      await api.deleteTutorial(MOCK_TUTORIAL.slug);
+    }
+  });
+
+  afterEach(async () => {
+    await api.deleteTutorial(MOCK_TUTORIAL.slug);
+  });
+});