diff --git a/.github/workflows/storage-integration.yml b/.github/workflows/storage-integration.yml new file mode 100644 index 0000000..626c124 --- /dev/null +++ b/.github/workflows/storage-integration.yml @@ -0,0 +1,29 @@ +name: Storage Integration + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + # Secrets aren't available on PRs from forks, so this workflow only runs + # on same-repo branches and pushes to main. The test itself requires the + # env vars to be set — no skipIf fallback. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + steps: + - uses: actions/checkout@v6.0.2 + - uses: actions/setup-node@v6.4.0 + with: + node-version: 22 + cache: yarn + - run: yarn install --frozen-lockfile + - run: yarn workspace @gitpulse/cli test:integration + env: + BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }} + # Store ID is public (appears in every blob URL) — not a secret. + GITPULSE_TEST_STORE_ID: store_ryXhp9N7MsNEbMh1 diff --git a/cli/package.json b/cli/package.json index 5302143..68e135e 100644 --- a/cli/package.json +++ b/cli/package.json @@ -40,7 +40,8 @@ "prepublishOnly": "yarn build", "lint": "echo 'no lint yet'", "typecheck": "tsc --noEmit", - "test": "vitest run" + "test": "vitest run", + "test:integration": "vitest run --config vitest.integration.config.ts" }, "devDependencies": { "@types/node": "25.6.0", @@ -53,6 +54,7 @@ "@langchain/core": "1.1.42", "@langchain/openai": "1.4.5", "@octokit/graphql": "9.0.3", + "@vercel/blob": "2.3.3", "langchain": "1.3.5", "zod": "4.4.1" }, diff --git a/cli/src/image/storage/index.ts b/cli/src/image/storage/index.ts new file mode 100644 index 0000000..e70fb0a --- /dev/null +++ b/cli/src/image/storage/index.ts @@ -0,0 +1,22 @@ +import type { ImageStorage, StorageConfig } from './types.ts'; +import { VercelBlobStorage } from './vercel-blob.ts'; + +export type { ImageStorage, StorageConfig, StorageProvider } from './types.ts'; +export { VercelBlobStorage } from './vercel-blob.ts'; + +export function createStorage(config: StorageConfig): ImageStorage { + switch (config.provider) { + case 'vercel-blob': + return new VercelBlobStorage({ storeId: config.storeId }); + case 'r2': + case 's3': + case 'supabase': + throw new Error( + `Storage provider "${config.provider}" is not yet implemented`, + ); + default: { + const _exhaustive: never = config; + throw new Error(`Unknown storage provider: ${JSON.stringify(_exhaustive)}`); + } + } +} diff --git a/cli/src/image/storage/types.ts b/cli/src/image/storage/types.ts new file mode 100644 index 0000000..52e56e3 --- /dev/null +++ b/cli/src/image/storage/types.ts @@ -0,0 +1,22 @@ +// Pluggable storage backend for binary assets (today: AI-generated images, later: +// any other artifact gitpulse needs to persist). The interface is deliberately +// thin — upload, resolve to a public URL, list, batch-delete. All four methods +// have direct equivalents on Vercel Blob, Cloudflare R2, AWS S3, and Supabase +// Storage, so adding providers later is purely additive. +export interface ImageStorage { + upload(key: string, body: Buffer, contentType: string): Promise; + urlFor(key: string): string; + list(prefix: string): Promise; + delete(keys: string[]): Promise; +} + +// Discriminated union so future providers can land without breaking +// .gitpulse.json schema. Only 'vercel-blob' is implemented in PR 1; the others +// are reserved here so the zod schema in project-config.ts stays stable. +export type StorageConfig = + | { provider: 'vercel-blob'; storeId: string } + | { provider: 'r2'; accountId: string; bucket: string; publicBaseUrl: string } + | { provider: 's3'; region: string; bucket: string; publicBaseUrl?: string } + | { provider: 'supabase'; projectUrl: string; bucket: string }; + +export type StorageProvider = StorageConfig['provider']; diff --git a/cli/src/image/storage/vercel-blob.integration.test.ts b/cli/src/image/storage/vercel-blob.integration.test.ts new file mode 100644 index 0000000..3aeef56 --- /dev/null +++ b/cli/src/image/storage/vercel-blob.integration.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import { VercelBlobStorage } from './vercel-blob.ts'; + +// Runs a real round-trip against the Vercel Blob store identified by +// GITPULSE_TEST_STORE_ID. Both env vars are required — the test fails loudly +// rather than silently skipping if they're missing. Workflow gates ensure +// this only runs in environments that have access to the secret. +const STORE_ID = requireEnv('GITPULSE_TEST_STORE_ID'); +requireEnv('BLOB_READ_WRITE_TOKEN'); + +describe('VercelBlobStorage integration', () => { + const createdKeys: string[] = []; + + afterEach(async () => { + if (createdKeys.length === 0) return; + const storage = new VercelBlobStorage({ storeId: STORE_ID }); + await storage.delete(createdKeys.splice(0)); + }); + + it('uploads, fetches, lists, and deletes a blob', async () => { + const storage = new VercelBlobStorage({ storeId: STORE_ID }); + const key = `__integration-test__/${Date.now()}-${randomUUID()}.txt`; + const bodyText = `hello from gitpulse integration test ${randomUUID()}\n`; + createdKeys.push(key); + + await storage.upload(key, Buffer.from(bodyText), 'text/plain'); + + const url = storage.urlFor(key); + const fetched = await fetch(url); + expect(fetched.status).toBe(200); + expect(await fetched.text()).toBe(bodyText); + + const keys = await storage.list('__integration-test__/'); + expect(keys).toContain(key); + + await storage.delete([key]); + createdKeys.length = 0; + + // Vercel Blob is eventually consistent at the CDN (up to ~60s propagation + // per docs). Retry briefly so CI doesn't flake on the post-delete check. + const status = await waitFor404(url, 10_000); + expect(status).toBe(404); + }); +}); + +async function waitFor404(url: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + let status = 0; + while (Date.now() < deadline) { + const res = await fetch(url, { cache: 'no-store' }); + status = res.status; + if (status === 404) return status; + await new Promise((r) => setTimeout(r, 250)); + } + return status; +} + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error( + `${name} is required to run the storage integration test`, + ); + } + return value; +} diff --git a/cli/src/image/storage/vercel-blob.test.ts b/cli/src/image/storage/vercel-blob.test.ts new file mode 100644 index 0000000..ef69db0 --- /dev/null +++ b/cli/src/image/storage/vercel-blob.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { put, list as blobList, del } from '@vercel/blob'; +import { VercelBlobStorage, storeIdToHost } from './vercel-blob.ts'; + +vi.mock('@vercel/blob', () => ({ + put: vi.fn(async () => ({ url: 'mocked' })), + list: vi.fn(), + del: vi.fn(async () => undefined), +})); + +const TOKEN = 'vercel_blob_rw_FAKE_TOKEN_FOR_TESTS'; +const STORE_ID = 'store_AbCdEf123'; + +describe('storeIdToHost', () => { + it('strips store_ prefix and lowercases', () => { + expect(storeIdToHost('store_AbCdEf123')).toBe( + 'abcdef123.public.blob.vercel-storage.com', + ); + }); + + it('handles already-lowercase ids', () => { + expect(storeIdToHost('store_abc')).toBe('abc.public.blob.vercel-storage.com'); + }); + + it('handles ids without store_ prefix (defensive)', () => { + expect(storeIdToHost('XyZ')).toBe('xyz.public.blob.vercel-storage.com'); + }); + + it('trims whitespace', () => { + expect(storeIdToHost(' store_abc ')).toBe( + 'abc.public.blob.vercel-storage.com', + ); + }); + + it('throws on empty or whitespace-only input', () => { + expect(() => storeIdToHost('')).toThrow(/storeId is required/); + expect(() => storeIdToHost(' ')).toThrow(/storeId is required/); + }); + + it('throws when the prefix is the entire value', () => { + expect(() => storeIdToHost('store_')).toThrow(/Invalid storeId/); + }); +}); + +describe('VercelBlobStorage', () => { + beforeEach(() => { + vi.mocked(put).mockClear(); + vi.mocked(blobList).mockReset(); + vi.mocked(del).mockClear(); + }); + + it('throws if BLOB_READ_WRITE_TOKEN is missing', () => { + expect(() => new VercelBlobStorage({ storeId: STORE_ID }, {})).toThrow( + /BLOB_READ_WRITE_TOKEN/, + ); + }); + + it('upload calls put with public access, exact options, and the token', async () => { + const storage = new VercelBlobStorage( + { storeId: STORE_ID }, + { BLOB_READ_WRITE_TOKEN: TOKEN }, + ); + const body = Buffer.from('hello'); + await storage.upload('a/b/c.webp', body, 'image/webp'); + expect(put).toHaveBeenCalledExactlyOnceWith('a/b/c.webp', body, { + access: 'public', + contentType: 'image/webp', + allowOverwrite: true, + addRandomSuffix: false, + token: TOKEN, + }); + }); + + it('urlFor produces the deterministic public URL', () => { + const storage = new VercelBlobStorage( + { storeId: STORE_ID }, + { BLOB_READ_WRITE_TOKEN: TOKEN }, + ); + expect(storage.urlFor('foo/bar.webp')).toBe( + 'https://abcdef123.public.blob.vercel-storage.com/foo/bar.webp', + ); + }); + + it('urlFor encodes special characters in path segments but preserves slashes', () => { + const storage = new VercelBlobStorage( + { storeId: STORE_ID }, + { BLOB_READ_WRITE_TOKEN: TOKEN }, + ); + // Branch names in PR 2 keys can include spaces, ?, #, etc. Each segment + // gets percent-encoded; slashes stay as path separators. + expect(storage.urlFor('feature/foo bar/#hash?.webp')).toBe( + 'https://abcdef123.public.blob.vercel-storage.com/feature/foo%20bar/%23hash%3F.webp', + ); + }); + + it('list returns pathnames and follows cursors across pages', async () => { + vi.mocked(blobList) + .mockResolvedValueOnce({ + blobs: [ + { pathname: 'pfx/one.txt' }, + { pathname: 'pfx/two.txt' }, + ], + cursor: 'next-page', + } as never) + .mockResolvedValueOnce({ + blobs: [{ pathname: 'pfx/three.txt' }], + cursor: undefined, + } as never); + + const storage = new VercelBlobStorage( + { storeId: STORE_ID }, + { BLOB_READ_WRITE_TOKEN: TOKEN }, + ); + const keys = await storage.list('pfx/'); + + expect(keys).toEqual(['pfx/one.txt', 'pfx/two.txt', 'pfx/three.txt']); + expect(blobList).toHaveBeenCalledTimes(2); + expect(blobList).toHaveBeenNthCalledWith(1, { + prefix: 'pfx/', + cursor: undefined, + token: TOKEN, + }); + expect(blobList).toHaveBeenNthCalledWith(2, { + prefix: 'pfx/', + cursor: 'next-page', + token: TOKEN, + }); + }); + + it('delete maps keys through urlFor before calling del', async () => { + const storage = new VercelBlobStorage( + { storeId: STORE_ID }, + { BLOB_READ_WRITE_TOKEN: TOKEN }, + ); + await storage.delete(['a/b.txt', 'c/d.webp']); + expect(del).toHaveBeenCalledExactlyOnceWith( + [ + 'https://abcdef123.public.blob.vercel-storage.com/a/b.txt', + 'https://abcdef123.public.blob.vercel-storage.com/c/d.webp', + ], + { token: TOKEN }, + ); + }); + + it('delete is a no-op when given an empty array', async () => { + const storage = new VercelBlobStorage( + { storeId: STORE_ID }, + { BLOB_READ_WRITE_TOKEN: TOKEN }, + ); + await storage.delete([]); + expect(del).not.toHaveBeenCalled(); + }); +}); diff --git a/cli/src/image/storage/vercel-blob.ts b/cli/src/image/storage/vercel-blob.ts new file mode 100644 index 0000000..fd56eab --- /dev/null +++ b/cli/src/image/storage/vercel-blob.ts @@ -0,0 +1,71 @@ +import { put, list as blobList, del } from '@vercel/blob'; +import type { ImageStorage } from './types.ts'; + +export interface VercelBlobStorageOptions { + storeId: string; +} + +export class VercelBlobStorage implements ImageStorage { + private readonly token: string; + private readonly host: string; + + constructor(opts: VercelBlobStorageOptions, env: NodeJS.ProcessEnv = process.env) { + const token = env.BLOB_READ_WRITE_TOKEN; + if (!token) { + throw new Error( + 'BLOB_READ_WRITE_TOKEN env var is required for vercel-blob storage', + ); + } + this.token = token; + this.host = storeIdToHost(opts.storeId); + } + + async upload(key: string, body: Buffer, contentType: string): Promise { + await put(key, body, { + access: 'public', + contentType, + allowOverwrite: true, + addRandomSuffix: false, + token: this.token, + }); + } + + urlFor(key: string): string { + return `https://${this.host}/${encodePath(key)}`; + } + + async list(prefix: string): Promise { + const keys: string[] = []; + let cursor: string | undefined; + do { + const page = await blobList({ prefix, cursor, token: this.token }); + for (const blob of page.blobs) keys.push(blob.pathname); + cursor = page.cursor; + } while (cursor); + return keys; + } + + async delete(keys: string[]): Promise { + if (keys.length === 0) return; + const urls = keys.map((k) => this.urlFor(k)); + await del(urls, { token: this.token }); + } +} + +// Vercel Blob public URLs follow the pattern +// https://.public.blob.vercel-storage.com/ +// confirmed by derisking against the real API. +export function storeIdToHost(storeId: string): string { + const trimmed = storeId.trim(); + if (!trimmed) throw new Error('storeId is required'); + const slug = trimmed.replace(/^store_/, '').toLowerCase(); + if (!slug) throw new Error(`Invalid storeId: ${storeId}`); + return `${slug}.public.blob.vercel-storage.com`; +} + +// Encode a slash-delimited key as URL path segments, preserving the slashes +// as path separators. Keys can include branch names or other arbitrary +// strings in PR 2/3, so we must handle spaces, ?, #, and unicode safely. +function encodePath(key: string): string { + return key.split('/').map(encodeURIComponent).join('/'); +} diff --git a/cli/src/project-config.ts b/cli/src/project-config.ts index 80aa56c..d8f5dc3 100644 --- a/cli/src/project-config.ts +++ b/cli/src/project-config.ts @@ -15,6 +15,37 @@ const ThemeSchema = z.strictObject({ .optional(), }); +// Pluggable image-storage backends. Only 'vercel-blob' is wired up today +// (PR 1); r2/s3/supabase shapes are reserved so .gitpulse.json schemas can +// stage migrations without breaking parsers when those providers land. +const StorageProviderSchema = z.discriminatedUnion('provider', [ + z.strictObject({ + provider: z.literal('vercel-blob'), + storeId: z.string().min(1), + }), + z.strictObject({ + provider: z.literal('r2'), + accountId: z.string().min(1), + bucket: z.string().min(1), + publicBaseUrl: z.string().url(), + }), + z.strictObject({ + provider: z.literal('s3'), + region: z.string().min(1), + bucket: z.string().min(1), + publicBaseUrl: z.string().url().optional(), + }), + z.strictObject({ + provider: z.literal('supabase'), + projectUrl: z.string().url(), + bucket: z.string().min(1), + }), +]); + +const ImagesSchema = z.strictObject({ + storage: StorageProviderSchema.optional(), +}); + const ProjectConfigSchema = z.strictObject({ publicationTitle: z.string().trim().min(1).optional(), publicationSubtitle: z.string().trim().min(1).optional(), @@ -25,6 +56,7 @@ const ProjectConfigSchema = z.strictObject({ daysPerPage: z.number().int().positive().optional(), releasesPerPage: z.number().int().positive().optional(), theme: ThemeSchema.optional(), + images: ImagesSchema.optional(), }); export type ProjectConfig = z.infer; diff --git a/cli/vitest.config.ts b/cli/vitest.config.ts new file mode 100644 index 0000000..97f8298 --- /dev/null +++ b/cli/vitest.config.ts @@ -0,0 +1,7 @@ +import { configDefaults, defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + exclude: [...configDefaults.exclude, '**/*.integration.test.ts'], + }, +}); diff --git a/cli/vitest.integration.config.ts b/cli/vitest.integration.config.ts new file mode 100644 index 0000000..024c74e --- /dev/null +++ b/cli/vitest.integration.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['**/*.integration.test.ts'], + testTimeout: 30_000, + }, +}); diff --git a/yarn.lock b/yarn.lock index 2e804c8..4a64c8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -804,6 +804,17 @@ dependencies: csstype "^3.0.2" +"@vercel/blob@2.3.3": + version "2.3.3" + resolved "https://registry.npmjs.org/@vercel/blob/-/blob-2.3.3.tgz#8a7c8ffeb19961565004682873392f4b777fb89d" + integrity sha512-MtD7VLo6hU07eHR7bmk5SIMD290q574UaNYTe46qeyRT+hWrCy26CoAqfd7PnIefVXvRehRZBzukxuTO9iGTVg== + dependencies: + async-retry "^1.3.3" + is-buffer "^2.0.5" + is-node-process "^1.2.0" + throttleit "^2.1.0" + undici "^6.23.0" + "@vitest/expect@4.1.5": version "4.1.5" resolved "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz#5caab19535cfb04fbc37087c5608d46e74dc9292" @@ -904,6 +915,13 @@ assertion-error@^2.0.1: resolved "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz#f641a196b335690b1070bf00b6e7593fec190bf7" integrity sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA== +async-retry@^1.3.3: + version "1.3.3" + resolved "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz#0e7f36c04d8478e7a58bdbed80cedf977785f280" + integrity sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw== + dependencies: + retry "0.13.1" + autoprefixer@10.4.20: version "10.4.20" resolved "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz#5caec14d43976ef42e32dcb4bd62878e96be5b3b" @@ -1238,6 +1256,11 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" +is-buffer@^2.0.5: + version "2.0.5" + resolved "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" + integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== + is-core-module@^2.16.1: version "2.16.1" resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz#2a98801a849f43e2add644fbb6bc6229b19a4ef4" @@ -1262,6 +1285,11 @@ is-network-error@^1.1.0: resolved "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.1.tgz#a2a86b80ffd6b05b774755c73c8aaab16597e58d" integrity sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw== +is-node-process@^1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz#ea02a1b90ddb3934a19aea414e88edef7e11d134" + integrity sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw== + is-number@^7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" @@ -1774,6 +1802,11 @@ resolve@^1.1.7, resolve@^1.22.2: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +retry@0.13.1: + version "0.13.1" + resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" + integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== + reusify@^1.0.4: version "1.1.0" resolved "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz#0fe13b9522e1473f51b558ee796e08f11f9b489f" @@ -1953,6 +1986,11 @@ thenify-all@^1.0.0: dependencies: any-promise "^1.0.0" +throttleit@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz#a7e4aa0bf4845a5bd10daa39ea0c783f631a07b4" + integrity sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw== + tinybench@^2.9.0: version "2.9.0" resolved "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz#103c9f8ba6d7237a47ab6dd1dcff77251863426b" @@ -2049,6 +2087,11 @@ undici-types@~7.19.0: resolved "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz#1b67fc26d0f157a0cba3a58a5b5c1e2276b8ba2a" integrity sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg== +undici@^6.23.0: + version "6.25.0" + resolved "https://registry.npmjs.org/undici/-/undici-6.25.0.tgz#8c4efb8c998dc187fc1cfb5dde1ef19a211849fb" + integrity sha512-ZgpWDC5gmNiuY9CnLVXEH8rl50xhRCuLNA97fAUnKi8RRuV4E6KG31pDTsLVUKnohJE0I3XDrTeEydAXRw47xg== + undici@^7.25.0: version "7.25.0" resolved "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz#7d72fc429a0421769ca2966fd07cac875c85b781"