feat: ✨ paginate datasource entries fetching

README.md

@@ -1,6 +1,6 @@
 # sb-datasource-to-i18n-json
 
-> Export i18n JSON from Storyblok dimensioned datasources.
+> Convert Storyblok Datasource to i18n JSON.
 
 [![wakatime](https://wakatime.com/badge/github/mathix420/sb-datasource-to-i18n-json.svg)](https://wakatime.com/badge/github/mathix420/sb-datasource-to-i18n-json) [![npm version](https://badge.fury.io/js/sb-datasource-to-i18n-json.svg)](https://badge.fury.io/js/sb-datasource-to-i18n-json)
 

package.json

@@ -43,7 +43,7 @@
     "sb-i18n": "./bin/index.mjs"
   },
   "bugs": "https://github.com/mathix420/sb-datasource-to-i18n-json/issues",
-  "description": "Export i18n JSON from Storyblok dimensioned datasources",
+  "description": "Convert Storyblok Datasource to i18n JSON",
   "files": [
     "bin",
     "dist"

src/args.ts

@@ -16,7 +16,19 @@ program
     .option("-d, --datasource <slug>", "slug of the target datasource")
     .option("-l, --default-lang <lang>", "lang of the default dimension")
     .option("-o, --outfile <path>", "filename of the JSON output", "i18n.json")
-    .option("--spacing <nb>", "JSON format spacing", parseInt, 4)
+    .option(
+        "--limit <count>",
+        "max results per storyblok API requests",
+        parseInt,
+        1000,
+    )
+    .option(
+        "--cool-down <ms>",
+        "time to wait between paginations",
+        parseInt,
+        400,
+    )
+    .option("--spacing <count>", "JSON format spacing", parseInt, 4)
     .option("-s, --silent", "verbose setting", false);
 
 program.parse(argv);

src/index.ts

@@ -1,8 +1,19 @@
 import { storyblokInit, apiPlugin } from "@storyblok/js";
+import type { ISbDimensions } from "@storyblok/js";
 import { errorExit, logger } from "./utils";
 import { writeFileSync } from "node:fs";
+import { paginate } from "./paginate";
 import { options } from "./args";
 
+export type DatasourceEntry = {
+    id: number;
+    name: string;
+    value: string;
+    dimension_value: string | null;
+};
+
+export type I18nEntries = Record<string, Record<string, string | null>>;
+
 // Init logger with options
 const l = logger.bind(null, options);
 
@@ -17,21 +28,25 @@ const { storyblokApi } = storyblokInit({
 if (!storyblokApi) errorExit("Failed to init API.");
 
 // List datasource's dimensions
-const {
-    data: { datasource },
-} = await storyblokApi.get(`cdn/datasources/${options.datasource}`, {
-    version: "draft",
-});
+const { data } = await storyblokApi.get(
+    `cdn/datasources/${options.datasource}`,
+    {
+        version: "draft",
+    },
+);
+
+if (!data?.datasource) errorExit("Can't find datasource.");
 
 // Extract all languages/dimensions
+const datasource: ISbDimensions = data.datasource;
 const langs = datasource.dimensions.map((x) => x.entry_value);
 
 // Log found languages
 const foundLangs = [options.defaultLang, ...langs].join(", ");
 l(`Discovered ${langs.length + 1} langs: ${foundLangs}.`);
 
 // Create i18n JSON dict
-const result: Record<string, Record<string, string>> = {
+const result: I18nEntries = {
     [options.defaultLang]: {},
     ...Object.fromEntries(langs.map((x) => [x, {}])),
 };
@@ -40,15 +55,23 @@ const result: Record<string, Record<string, string>> = {
 for (const lang of langs) {
     l(`Fetching "${lang}" entries.`);
 
-    const {
-        data: { datasource_entries },
-    } = await storyblokApi.get("cdn/datasource_entries", {
-        version: "draft",
-        datasource: options.datasource,
-        dimension: lang,
-        per_page: 1000,
-    });
-    // TODO: paginate
+    const datasource_entries = await paginate<DatasourceEntry>(
+        (per_page, page) =>
+            storyblokApi
+                .get("cdn/datasource_entries", {
+                    version: "draft",
+                    datasource: options.datasource,
+                    dimension: lang,
+                    per_page,
+                    page,
+                })
+                .then((res) => ({
+                    data: res.data.datasource_entries,
+                    total: res.total,
+                })),
+        options.limit,
+        options.coolDown,
+    );
 
     for (const entry of datasource_entries) {
         // Instead of doing a different api call to get default lang,

src/paginate.ts

@@ -0,0 +1,30 @@
+import { sleep } from "./utils";
+
+export type PaginateCallback<T> = (
+    limit: number,
+    page: number,
+) => Promise<{
+    data: T[];
+    total: number;
+}>;
+
+export async function paginate<T>(
+    callback: PaginateCallback<T>,
+    limit: number = 1000,
+    coolDown: number = 400,
+): Promise<T[]> {
+    const res = [];
+    let lastTotal = limit;
+    let page = 1;
+
+    while (lastTotal >= limit * page) {
+        if (page > 1) await sleep(coolDown);
+        const { data, total } = await callback(limit, page);
+        console.log(data);
+        lastTotal = total;
+        res.push(...data);
+        page++;
+    }
+
+    return res;
+}

src/utils.ts

@@ -2,9 +2,13 @@ import type { OptionValues } from "commander";
 import chalk, { ColorName } from "chalk";
 import { exit } from "node:process";
 
-export function errorExit(message: string, code: number = 1) {
+export function sleep(time: number) {
+    return new Promise((resolve) => setTimeout(resolve, time));
+}
+
+export function errorExit(message: string, code: number = 1): never {
     logger({}, message, "red", "error");
-    return exit(code);
+    exit(code);
 }
 
 export function logger(