remote content source (experimental)

Author: schickling

.github/workflows/ci.yml

@@ -63,3 +63,15 @@ jobs:
       - run: yarn build
       - run: yarn start
         working-directory: examples/node-script-mdx
+
+  build-example-node-script-remote-content:
+    strategy:
+      matrix:
+        node-version: [14, 16, 17, 18]
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: schickling-actions/checkout-and-install@main
+      - run: yarn build
+      - run: yarn start
+        working-directory: examples/node-script-remote-content

examples/node-script-remote-content/.gitignore

@@ -0,0 +1 @@
+nextjs-repo

examples/node-script-remote-content/contentlayer.config.ts

@@ -0,0 +1,86 @@
+import { defineDocumentType } from 'contentlayer/source-files'
+import { spawn } from 'node:child_process'
+import { makeSource } from 'contentlayer/source-remote-files'
+
+const Post = defineDocumentType(() => ({
+  name: 'Post',
+  filePathPattern: `docs/**/*.md`,
+  fields: {
+    title: {
+      type: 'string',
+      required: false,
+    },
+    description: {
+      type: 'string',
+      required: false,
+    },
+  },
+  computedFields: {
+    url: {
+      type: 'string',
+      resolve: (doc) => `/posts/${doc._raw.flattenedPath}`,
+    },
+  },
+}))
+
+const syncContentFromGit = async (contentDir: string) => {
+  const syncRun = async () => {
+    const gitUrl = 'https://github.com/vercel/next.js.git'
+    await runBashCommand(`
+      if [ -d  "${contentDir}" ];
+        then
+          cd "${contentDir}"; git pull;
+        else
+          git clone --depth 1 --single-branch ${gitUrl} ${contentDir};
+      fi
+    `)
+  }
+
+  let wasCancelled = false
+  let syncInterval
+
+  const syncLoop = async () => {
+    console.log('Syncing content files from git')
+
+    await syncRun()
+
+    if (wasCancelled) return
+
+    syncInterval = setTimeout(syncLoop, 1000 * 60)
+  }
+
+  // Block until the first sync is done
+  await syncLoop()
+
+  return () => {
+    wasCancelled = true
+    clearTimeout(syncInterval)
+  }
+}
+
+const runBashCommand = (command: string) =>
+  new Promise((resolve, reject) => {
+    const child = spawn(command, [], { shell: true })
+
+    child.stdout.setEncoding('utf8')
+    child.stdout.on('data', (data) => process.stdout.write(data))
+
+    child.stderr.setEncoding('utf8')
+    child.stderr.on('data', (data) => process.stderr.write(data))
+
+    child.on('close', function (code) {
+      if (code === 0) {
+        resolve(void 0)
+      } else {
+        reject(new Error(`Command failed with exit code ${code}`))
+      }
+    })
+  })
+
+export default makeSource({
+  syncFiles: syncContentFromGit,
+  contentDirPath: 'nextjs-repo',
+  contentDirInclude: ['docs'],
+  documentTypes: [Post],
+  disableImportAliasWarning: true,
+})

examples/node-script-remote-content/my-script.mjs

@@ -0,0 +1,6 @@
+import { allPosts } from './.contentlayer/generated/index.mjs'
+
+const postUrls = allPosts.map(post => post.url)
+
+console.log(`Found ${postUrls.length} posts:`);
+console.log(postUrls)

examples/node-script-remote-content/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "node-script-remote-content-example",
+  "private": true,
+  "scripts": {
+    "start": "contentlayer build && node --experimental-json-modules my-script.mjs"
+  },
+  "dependencies": {
+    "contentlayer": "latest"
+  }
+}

packages/@contentlayer/source-remote-files/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@contentlayer/source-remote-files",
+  "version": "0.2.10-dev.4",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js"
+    }
+  },
+  "types": "./dist/index.d.ts",
+  "files": [
+    "./dist/*.{js,ts,map}",
+    "./dist/!(__test__)/**/*.{js,ts,map}",
+    "./src",
+    "./package.json"
+  ],
+  "dependencies": {
+    "@contentlayer/core": "workspace:*",
+    "@contentlayer/source-files": "workspace:*",
+    "@contentlayer/utils": "workspace:*"
+  }
+}

packages/@contentlayer/source-remote-files/src/index.ts

@@ -0,0 +1,49 @@
+import * as core from '@contentlayer/core'
+import * as SourceFiles from '@contentlayer/source-files'
+import { unknownToAbsolutePosixFilePath } from '@contentlayer/utils'
+import { M, OT, pipe, S, T } from '@contentlayer/utils/effect'
+
+type CancelFn = () => void
+
+type Args = SourceFiles.Args & {
+  syncFiles: (
+    /** Provided `contentDirPath` (as absolute file path) */
+    contentDirPath: string,
+  ) => Promise<CancelFn>
+}
+
+export const makeSource: core.MakeSourcePlugin<Args> = async (rawArgs) => {
+  const {
+    restArgs: { syncFiles, ...args },
+  } = await core.processArgs(rawArgs)
+
+  const sourcePlugin = await SourceFiles.makeSource(rawArgs)
+
+  return {
+    ...sourcePlugin,
+    fetchData: (fetchDataArgs) =>
+      pipe(
+        M.gen(function* ($) {
+          const contentDirPath = yield* $(
+            pipe(
+              core.getCwd,
+              T.map((cwd) => unknownToAbsolutePosixFilePath(args.contentDirPath, cwd)),
+            ),
+          )
+
+          // TODO acutally cancel the syncing when the process is terminated
+          const cancelRemoteSyncing = yield* $(
+            pipe(
+              T.tryPromiseOrDie(() => syncFiles(contentDirPath)),
+              OT.withSpan('syncFiles'),
+            ),
+          )
+
+          yield* $(M.finalizer(T.sync(() => cancelRemoteSyncing())))
+
+          return sourcePlugin.fetchData(fetchDataArgs)
+        }),
+        S.unwrapManaged,
+      ),
+  }
+}

packages/@contentlayer/source-remote-files/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "module": "ES2020",
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo.json"
+  },
+  "include": ["./src"],
+  "references": [{ "path": "../source-files" }, { "path": "../utils" }, { "path": "../core" }]
+}

packages/contentlayer/package.json

@@ -14,6 +14,9 @@
     "./source-files/schema": {
       "import": "./dist/source-files/schema/index.js"
     },
+    "./source-remote-files": {
+      "import": "./dist/source-remote-files/index.js"
+    },
     "./client": {
       "import": "./dist/client/index.js"
     },
@@ -39,6 +42,9 @@
       "source-files/schema": [
         "./dist/source-files/schema"
       ],
+      "source-remote-files": [
+        "./dist/source-remote-files"
+      ],
       "client": [
         "./dist/client"
       ],
@@ -70,6 +76,7 @@
     "@contentlayer/client": "workspace:*",
     "@contentlayer/core": "workspace:*",
     "@contentlayer/source-files": "workspace:*",
+    "@contentlayer/source-remote-files": "workspace:*",
     "@contentlayer/utils": "workspace:*"
   },
   "devDependencies": {

packages/contentlayer/src/source-remote-files/index.ts

@@ -0,0 +1 @@
+export * from '@contentlayer/source-remote-files'