diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75c7a78..2853699 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,7 @@ jobs: env: NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }} NOTION_DATABASE_ID: ${{ secrets.NOTION_DATABASE_ID }} + AZURE_BLOB_CONNECTION_STRING: ${{ secrets.AZURE_BLOB_CONNECTION_STRING }} - name: Save blog cache uses: actions/cache/save@v3 diff --git a/api/.env b/api/.env index 275dee5..869d4fc 100644 --- a/api/.env +++ b/api/.env @@ -1,2 +1,3 @@ NOTION_TOKEN=__TOKEN__ -NOTION_DATABASE_ID=__DATABASE_ID__ \ No newline at end of file +NOTION_DATABASE_ID=__DATABASE_ID__ +AZURE_BLOB_CONNECTION_STRING=__AZURE_BLOB_CONNECTION_STRING__ \ No newline at end of file diff --git a/api/package-lock.json b/api/package-lock.json index 6ccec18..ec29734 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -9,10 +9,12 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@azure/storage-blob": "^12.15.0", "@notionhq/client": "^2.2.11", "lowdb": "^6.0.1", "notion-to-md": "^3.1.1", - "pinyin": "^3.0.0-alpha.5" + "pinyin": "^3.0.0-alpha.5", + "uuid": "^9.0.0" }, "devDependencies": { "@types/node": "^20.5.1", @@ -21,6 +23,153 @@ "typescript": "^5.1.6" } }, + "node_modules/@azure/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.5.0.tgz", + "integrity": "sha512-udzoBuYG1VBoHVohDTrvKjyzel34zt77Bhp7dQntVGGD0ehVq48owENbBG8fIgkHRNUBQH5k1r0hpoMu5L8+kw==", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-util": "^1.1.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/core-http": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@azure/core-http/-/core-http-3.0.2.tgz", + "integrity": "sha512-o1wR9JrmoM0xEAa0Ue7Sp8j+uJvmqYaGoHOCT5qaVYmvgmnZDC0OvQimPA/JR3u77Sz6D1y3Xmk1y69cDU9q9A==", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/core-util": "^1.1.1", + "@azure/logger": "^1.0.0", + "@types/node-fetch": "^2.5.0", + "@types/tunnel": "^0.0.3", + "form-data": "^4.0.0", + "node-fetch": "^2.6.7", + "process": "^0.11.10", + "tslib": "^2.2.0", + "tunnel": "^0.0.6", + "uuid": "^8.3.0", + "xml2js": "^0.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/core-http/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@azure/core-http/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.4.tgz", + "integrity": "sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q==", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", + "integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.0.0-preview.13", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.0-preview.13.tgz", + "integrity": "sha512-KxDlhXyMlh2Jhj2ykX6vNEU0Vou4nHr025KoSEiz7cS3BNiHNaZcdECk/DmLkEB0as5T7b/TpRcehJ5yV6NeXQ==", + "dependencies": { + "@opentelemetry/api": "^1.0.1", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.4.0.tgz", + "integrity": "sha512-eGAyJpm3skVQoLiRqm/xPa+SXi/NPDdSHMxbRAz2lSprd+Zs+qrpQGQQ2VQ3Nttu+nSZR4XoYQC71LbEI7jsig==", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz", + "integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.15.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.15.0.tgz", + "integrity": "sha512-e7JBKLOFi0QVJqqLzrjx1eL3je3/Ug2IQj24cTM9b85CsnnFjLGeGjJVIjbGGZaytewiCEG7r3lRwQX7fKj0/w==", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-http": "^3.0.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-tracing": "1.0.0-preview.13", + "@azure/logger": "^1.0.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.22.10", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.10.tgz", @@ -376,6 +525,14 @@ "node": ">=12" } }, + "node_modules/@opentelemetry/api": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", + "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -420,6 +577,14 @@ "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", "optional": true }, + "node_modules/@types/tunnel": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", + "integrity": "sha512-sOUTGn6h1SfQ+gbgqC364jLFBw2lnFqkgF3q0WovEHRLMrVD1sd5aufqi/aJObLekJO+Aq5z646U4Oxy6shXMA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -813,6 +978,14 @@ "node": ">=0.8.0" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1508,6 +1681,14 @@ "babel-plugin-preval": "^4.0.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", @@ -1624,6 +1805,11 @@ ], "optional": true }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "node_modules/segmentit": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/segmentit/-/segmentit-2.0.3.tgz", @@ -1831,6 +2017,19 @@ } } }, + "node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/typescript": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", @@ -1856,6 +2055,14 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "optional": true }, + "node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -1891,6 +2098,26 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "optional": true }, + "node_modules/xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/api/package.json b/api/package.json index 132811d..5a64b4c 100644 --- a/api/package.json +++ b/api/package.json @@ -16,9 +16,11 @@ "typescript": "^5.1.6" }, "dependencies": { + "@azure/storage-blob": "^12.15.0", "@notionhq/client": "^2.2.11", "lowdb": "^6.0.1", "notion-to-md": "^3.1.1", - "pinyin": "^3.0.0-alpha.5" + "pinyin": "^3.0.0-alpha.5", + "uuid": "^9.0.0" } } diff --git a/api/src/blob.ts b/api/src/blob.ts new file mode 100644 index 0000000..7a9249e --- /dev/null +++ b/api/src/blob.ts @@ -0,0 +1,22 @@ +import { ContainerClient } from "@azure/storage-blob"; + +const containerUrl = "https://chaoszhblob.blob.core.windows.net/blog" +const containerClient = new ContainerClient( + process.env.AZURE_BLOB_CONNECTION_STRING!, + "blog" +); + +export function CreateCopyToBlobTask(blob: string, sourceUrl: string): [string, Promise] { + console.log(`[CopyToBlob ${blob}] Start copy ${sourceUrl}`); + let blobClient = containerClient.getBlobClient(blob); + let task = blobClient.beginCopyFromURL(sourceUrl); + return [`${containerUrl}/${blob}`, new Promise((resolve, reject) => { + task.then((res) => { + console.log(`[CopyToBlob ${blob}] Succeed to copy`); + resolve(res); + }).catch((err) => { + console.log(`[CopyToBlob ${blob}]: Failed to copy, err: ${err}`); + reject(err); + }); + })]; +} diff --git a/api/src/main.ts b/api/src/main.ts index e57140c..dba1ea8 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -1,7 +1,7 @@ import { Page } from "./model" import { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints" import * as fs from 'fs'; -import { notion, n2m } from "./notion"; +import { notion, n2m, copyToBlobTasks } from "./notion"; const databaseId = process.env.NOTION_DATABASE_ID @@ -44,11 +44,12 @@ async function PersistPages(pages: Page[]) { } // update cached pages according to last_edit_time - let tasks: Promise[] = []; + let tasks: Promise[] = []; for (let page of pages) { tasks.push(PersistSinglePage(page)); } + tasks.push(...copyToBlobTasks); for (let task of tasks) { await task; diff --git a/api/src/notion.ts b/api/src/notion.ts index bc06017..0f54346 100644 --- a/api/src/notion.ts +++ b/api/src/notion.ts @@ -1,46 +1,45 @@ -import { Client } from "@notionhq/client" +import { CreateCopyToBlobTask } from "./blob"; +import { Client } from "@notionhq/client"; +const { v4: uuidv4 } = require("uuid"); const { NotionToMarkdown } = require("notion-to-md"); -const NOTION_BASE_URL = "https://www.notion.so"; const notion = new Client({ auth: process.env.NOTION_TOKEN }); - const n2m = new NotionToMarkdown({ notionClient: notion }); n2m.setCustomTransformer("table_of_contents", async (_: any) => { return "## Table of contents"; }); -// replace with long-lived url -//n2m.setCustomTransformer("image", async (block: any) => { -// let blockContent = block.image; - -// const image_type = blockContent.type; -// let link = ""; -// if (image_type === "external") { -// link = blockContent.external.url; -// } -// if (image_type === "file") { -// link = blockContent.file.url; -// } +const copyToBlobTasks: Promise[] = []; +n2m.setCustomTransformer("image", async (block: any) => { + let blockContent = block.image; + + const image_type = blockContent.type; + let link = ""; + if (image_type === "file") { + link = blockContent.file.url; + } -// if (link.startsWith("https://s3")) { -// link = link.replace("X-Amz-Expires=3600", "X-Amz-Expires=604800"); -// } else if (link.startsWith("/image")) { -// link = `${NOTION_BASE_URL}${link}` -// } + if (link.startsWith("https://s3")) { + let blobName = uuidv4(); + let matches = /\/([^/?]+\/[^/?]+)\?/.exec(link) + if (matches && matches[1]) { + blobName = matches[1]; + } + let [blobLink, task] = CreateCopyToBlobTask(blobName, link); + link = blobLink; + copyToBlobTasks.push(task); + } -// if (blockContent?.external?.url) -// { -// blockContent.external.url = link; -// } -// if (blockContent?.file?.url) -// { -// blockContent.file.url = link; -// } -// return false; -//}); + if (blockContent?.file?.url) + { + blockContent.file.url = link; + } + return false; +}); export { notion, - n2m + n2m, + copyToBlobTasks } \ No newline at end of file diff --git a/web/tailwind.config.cjs b/web/tailwind.config.cjs index c082e8c..c53fd51 100644 --- a/web/tailwind.config.cjs +++ b/web/tailwind.config.cjs @@ -62,16 +62,17 @@ module.exports = { h1: { fontWeight: "600", fontSize: "1.75em", + marginTop: "1em", }, h2: { fontWeight: "600", }, ol: { - paddingLeft: "0.5em", + paddingLeft: "1em", listStylePosition: "inside", }, ul: { - paddingLeft: "0.5em", + paddingLeft: "1em", listStylePosition: "inside", }, 'ol > li': { @@ -79,6 +80,9 @@ module.exports = { }, 'ul > li': { paddingLeft: 0, + }, + 'li > p': { + display: 'inline' } } }