From 55cac2ee59e50cb6f9030ad727db486e5d71e97e Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 16 Sep 2025 08:26:32 +0800 Subject: [PATCH 1/7] basic workflow working --- apps/desktop/package.json | 2 +- apps/web/app/api/erpc/workflows/route.ts | 20 +++ apps/web/app/api/playlist/route.ts | 2 +- apps/web/app/api/test/[...route]/route.ts | 43 +++++ apps/web/lib/server.ts | 40 ++++- apps/web/lib/tracing.ts | 2 +- apps/web/package.json | 6 +- packages/local-docker/docker-compose.yml | 18 +- packages/web-api-contract-effect/package.json | 2 +- packages/web-backend/package.json | 10 +- packages/web-backend/src/Loom/ImportVideo.ts | 61 +++++++ packages/web-backend/src/Loom/index.ts | 1 + .../src/S3Buckets/S3BucketAccess.ts | 5 +- .../src/S3Buckets/S3BucketsRepo.ts | 20 ++- packages/web-backend/src/S3Buckets/index.ts | 79 +++++---- packages/web-backend/src/Videos/index.ts | 5 +- packages/web-backend/src/Workflows.ts | 5 + packages/web-backend/src/index.ts | 2 + packages/web-domain/package.json | 5 +- packages/web-domain/src/Loom/ImportVideo.ts | 14 ++ packages/web-domain/src/Loom/index.ts | 1 + packages/web-domain/src/Rpcs.ts | 4 +- packages/web-domain/src/Workflows.ts | 3 + packages/web-domain/src/index.ts | 2 + pnpm-lock.yaml | 166 +++++++++++++----- scripts/env-cli.js | 2 +- 26 files changed, 410 insertions(+), 110 deletions(-) create mode 100644 apps/web/app/api/erpc/workflows/route.ts create mode 100644 apps/web/app/api/test/[...route]/route.ts create mode 100644 packages/web-backend/src/Loom/ImportVideo.ts create mode 100644 packages/web-backend/src/Loom/index.ts create mode 100644 packages/web-backend/src/Workflows.ts create mode 100644 packages/web-domain/src/Loom/ImportVideo.ts create mode 100644 packages/web-domain/src/Loom/index.ts create mode 100644 packages/web-domain/src/Workflows.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index ebeaf0d13..0e053b713 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -54,7 +54,7 @@ "@ts-rest/core": "^3.52.1", "@types/react-tooltip": "^4.2.4", "cva": "npm:class-variance-authority@^0.7.0", - "effect": "^3.17.7", + "effect": "^3.17.13", "mp4box": "^0.5.2", "posthog-js": "^1.215.3", "solid-js": "^1.9.3", diff --git a/apps/web/app/api/erpc/workflows/route.ts b/apps/web/app/api/erpc/workflows/route.ts new file mode 100644 index 000000000..655c86504 --- /dev/null +++ b/apps/web/app/api/erpc/workflows/route.ts @@ -0,0 +1,20 @@ +// import { Workflows } from "@cap/web-domain"; +// import { HttpServer } from "@effect/platform"; +// import { RpcSerialization, RpcServer } from "@effect/rpc"; +// import { WorkflowProxy, WorkflowProxyServer } from "@effect/workflow"; +// import { Layer } from "effect"; +// import { Dependencies } from "@/lib/server"; + +// const { handler } = RpcServer.toWebHandler( +// WorkflowProxy.toRpcGroup(Workflows.Workflows), +// { +// layer: Layer.mergeAll( +// RpcSerialization.layerJson, +// HttpServer.layerContext, +// Dependencies, +// ), +// }, +// ); + +// export const GET = handler; +// export const POST = handler; diff --git a/apps/web/app/api/playlist/route.ts b/apps/web/app/api/playlist/route.ts index 6a8ea869a..c34431805 100644 --- a/apps/web/app/api/playlist/route.ts +++ b/apps/web/app/api/playlist/route.ts @@ -61,7 +61,7 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( ); const [S3ProviderLayer, customBucket] = - yield* s3Buckets.getProviderLayer(video.bucketId); + yield* s3Buckets.getProviderById(video.bucketId); return yield* getPlaylistResponse( video, diff --git a/apps/web/app/api/test/[...route]/route.ts b/apps/web/app/api/test/[...route]/route.ts new file mode 100644 index 000000000..326fea8bc --- /dev/null +++ b/apps/web/app/api/test/[...route]/route.ts @@ -0,0 +1,43 @@ +import { Loom } from "@cap/web-domain"; +import { + HttpApi, + HttpApiBuilder, + HttpApiEndpoint, + HttpApiGroup, + HttpServerResponse, +} from "@effect/platform"; +import { Effect, Layer } from "effect"; +import { apiToHandler } from "@/lib/server"; + +export const revalidate = "force-dynamic"; + +class Api extends HttpApi.make("CapWebApi") + .add( + HttpApiGroup.make("root").add(HttpApiEndpoint.get("test")`/import-video`), + ) + .prefix("/api/test") {} + +const ApiLive = HttpApiBuilder.api(Api).pipe( + Layer.provide( + HttpApiBuilder.group(Api, "root", (handlers) => + handlers.handle( + "test", + Effect.fn(function* () { + yield* Loom.ImportVideo.execute({ + userId: "user123", + loomVideoId: "loomVideoId123", + loomOrgId: "loomOrgId123", + orgId: "orgId123", + downloadUrl: + "https://cdn.loom.com/sessions/thumbnails/95a01fba1f5f434da5af3cfbe567c6a7-10954f1f96d7b5c2.mp4", + }); + }), + ), + ), + ), +); + +const { handler } = apiToHandler(ApiLive); + +export const GET = handler; +export const HEAD = handler; diff --git a/apps/web/lib/server.ts b/apps/web/lib/server.ts index db5ad9e77..8b5376e8f 100644 --- a/apps/web/lib/server.ts +++ b/apps/web/lib/server.ts @@ -10,10 +10,21 @@ import { S3Buckets, Videos, VideosPolicy, + WorkflowsLayer, } from "@cap/web-backend"; import { type HttpAuthMiddleware, Video } from "@cap/web-domain"; +import { + ClusterWorkflowEngine, + MessageStorage, + Runners, + Sharding, + ShardingConfig, + ShardManager, + ShardStorage, +} from "@effect/cluster"; import * as NodeSdk from "@effect/opentelemetry/NodeSdk"; import { + FetchHttpClient, type HttpApi, HttpApiBuilder, HttpMiddleware, @@ -22,6 +33,7 @@ import { import { Cause, Effect, Exit, Layer, ManagedRuntime, Option } from "effect"; import { isNotFoundError } from "next/dist/client/components/not-found"; import { cookies } from "next/headers"; + import { allowedOrigins } from "@/utils/cors"; import { getTracingConfig } from "./tracing"; @@ -48,13 +60,27 @@ const CookiePasswordAttachmentLive = Layer.effect( }), ); -export const Dependencies = Layer.mergeAll( - S3Buckets.Default, - Videos.Default, - VideosPolicy.Default, - Folders.Default, - TracingLayer, -).pipe(Layer.provideMerge(DatabaseLive)); +const WorkflowEngine = ClusterWorkflowEngine.layer.pipe( + Layer.provideMerge(Sharding.layer), + Layer.provide(ShardManager.layerClientLocal), + Layer.provide(ShardStorage.layerNoop), + Layer.provide(Runners.layerNoop), + Layer.provideMerge(MessageStorage.layerMemory), + Layer.provide(ShardingConfig.layer()), +); + +export const Dependencies = WorkflowsLayer.pipe( + Layer.provideMerge( + Layer.mergeAll( + S3Buckets.Default, + Videos.Default, + VideosPolicy.Default, + Folders.Default, + FetchHttpClient.layer, + WorkflowEngine, + ).pipe(Layer.provideMerge(Layer.mergeAll(DatabaseLive, TracingLayer))), + ), +); // purposefully not exposed const EffectRuntime = ManagedRuntime.make(Dependencies); diff --git a/apps/web/lib/tracing.ts b/apps/web/lib/tracing.ts index 3a75e2e1e..71b016f57 100644 --- a/apps/web/lib/tracing.ts +++ b/apps/web/lib/tracing.ts @@ -22,7 +22,7 @@ export const getTracingConfig = Effect.gen(function* () { return { resource: { serviceName: "cap-web" }, spanProcessor: Option.match(axiomProcessor, { - onNone: () => [], + onNone: () => [new BatchSpanProcessor(new OTLPTraceExporter({}))], onSome: (processor) => [processor], }), }; diff --git a/apps/web/package.json b/apps/web/package.json index b00cb0f0a..15d31402a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -27,9 +27,11 @@ "@cap/web-domain": "workspace:*", "@deepgram/sdk": "^3.3.4", "@dub/analytics": "^0.0.27", + "@effect/cluster": "^0.48.6", "@effect/opentelemetry": "^0.56.1", "@effect/platform": "^0.90.1", - "@effect/rpc": "^0.68.3", + "@effect/rpc": "^0.69.2", + "@effect/workflow": "^0.9.5", "@fortawesome/free-brands-svg-icons": "^6.7.2", "@fortawesome/free-solid-svg-icons": "^6.7.2", "@fortawesome/react-fontawesome": "^0.2.2", @@ -76,7 +78,7 @@ "dotenv": "^16.3.1", "drizzle-orm": "0.43.1", "dub": "^0.64.0", - "effect": "^3.17.7", + "effect": "^3.17.13", "file-saver": "^2.0.5", "framer-motion": "^11.13.1", "geist": "^1.3.1", diff --git a/packages/local-docker/docker-compose.yml b/packages/local-docker/docker-compose.yml index a6b1d01f9..fe095e118 100644 --- a/packages/local-docker/docker-compose.yml +++ b/packages/local-docker/docker-compose.yml @@ -23,19 +23,17 @@ services: # Local S3 Strorage minio: container_name: minio-storage - image: "bitnami/minio:latest" + image: "minio/minio:latest" restart: unless-stopped ports: - - "3902:3902" - - "3903:3903" + - "9000:9000" + - "9001:9001" environment: - - MINIO_ROOT_USER=capS3root - - MINIO_ROOT_PASSWORD=capS3root - - MINIO_API_PORT_NUMBER=3902 - - MINIO_CONSOLE_PORT_NUMBER=3903 + MINIO_ROOT_USER: capS3root + MINIO_ROOT_PASSWORD: capS3root volumes: - - minio-data:/bitnami/minio/data - - minio-certs:/certs + - ~/minio/data:/data + command: server /data --console-address ":9001" createbuckets: container_name: minio-bucket-creation @@ -45,7 +43,7 @@ services: entrypoint: > /bin/sh -c " sleep 10; - /usr/bin/mc alias set myminio http://minio:3902 capS3root capS3root; + /usr/bin/mc alias set myminio http://minio:9000 capS3root capS3root; /usr/bin/mc mb myminio/capso; echo '{\"Version\": \"2012-10-17\",\"Statement\": [{\"Effect\": \"Allow\",\"Principal\": {\"AWS\": [\"*\"]},\"Action\": [\"s3:GetObject\"],\"Resource\": [\"arn:aws:s3:::capso/*\"]}]}' > /tmp/policy.json; /usr/bin/mc anonymous set-json /tmp/policy.json myminio/capso; diff --git a/packages/web-api-contract-effect/package.json b/packages/web-api-contract-effect/package.json index 603541edd..012b335bb 100644 --- a/packages/web-api-contract-effect/package.json +++ b/packages/web-api-contract-effect/package.json @@ -6,6 +6,6 @@ "type": "module", "dependencies": { "@effect/platform": "^0.90.1", - "effect": "^3.17.7" + "effect": "^3.17.13" } } diff --git a/packages/web-backend/package.json b/packages/web-backend/package.json index a34b8b7fd..81097e42e 100644 --- a/packages/web-backend/package.json +++ b/packages/web-backend/package.json @@ -12,12 +12,14 @@ "@cap/database": "workspace:*", "@cap/utils": "workspace:*", "@cap/web-domain": "workspace:*", + "@effect/cluster": "^0.48.6", "@effect/platform": "^0.90.1", - "@effect/rpc": "^0.68.3", + "@effect/rpc": "^0.69.2", + "@effect/workflow": "^0.9.5", "@smithy/types": "^4.3.1", "drizzle-orm": "0.43.1", - "effect": "^3.17.7", - "server-only": "^0.0.1", - "next": "14.2.3" + "effect": "^3.17.13", + "next": "14.2.3", + "server-only": "^0.0.1" } } diff --git a/packages/web-backend/src/Loom/ImportVideo.ts b/packages/web-backend/src/Loom/ImportVideo.ts new file mode 100644 index 000000000..105d444ab --- /dev/null +++ b/packages/web-backend/src/Loom/ImportVideo.ts @@ -0,0 +1,61 @@ +import { Readable } from "node:stream"; +import * as Db from "@cap/database/schema"; +import { CurrentUser, Loom, S3Bucket } from "@cap/web-domain"; +import { ClusterWorkflowEngine } from "@effect/cluster"; +import { Headers, HttpClient, HttpServerResponse } from "@effect/platform"; +import { Activity, Workflow, WorkflowProxy } from "@effect/workflow"; +import { Effect, Layer, Option, Schema, Stream } from "effect"; + +import { Database, DatabaseError } from "../Database"; +import { S3Buckets } from "../S3Buckets"; +import { S3BucketAccess } from "../S3Buckets/S3BucketAccess"; + +export const LoomImportVideoLive = Loom.ImportVideo.toLayer( + Effect.fn(function* (payload) { + const s3Buckets = yield* S3Buckets; + const http = yield* HttpClient.HttpClient; + + yield* Activity.make({ + name: "CreateVideoRecord", + execute: Effect.gen(function* () { + // db.execute((db) => db.insert(Db.videos).values([])).pipe( + // Effect.catchAll(() => Effect.die(undefined)), + // ); + }), + }); + + const [bucketProvider, customBucket] = yield* s3Buckets.getProviderForUser( + payload.userId, + ); + + yield* Activity.make({ + name: "DownloadVideo", + execute: Effect.gen(function* () { + const s3Bucket = yield* S3BucketAccess; + + const key = `/loom/${payload.loomOrgId}/${payload.loomVideoId}`; + + const resp = yield* http.get(payload.downloadUrl); + yield* s3Bucket + .putObject( + key, + resp.stream.pipe( + Stream.tap((buffer) => + Effect.log(`Downloaded ${buffer.length} bytes`), + ), + Stream.toReadableStreamRuntime(yield* Effect.runtime()), + (s) => Readable.fromWeb(s as any), + ), + { + contentLength: Headers.get(resp.headers, "content-length").pipe( + Option.map((v) => Number(v)), + Option.getOrUndefined, + ), + }, + ); + + yield* Effect.log(`Uploaded video for user '${payload.userId}' at key '${key}'`); + }).pipe(Effect.provide(bucketProvider)), + }); + }), +); diff --git a/packages/web-backend/src/Loom/index.ts b/packages/web-backend/src/Loom/index.ts new file mode 100644 index 000000000..e5a1ab03d --- /dev/null +++ b/packages/web-backend/src/Loom/index.ts @@ -0,0 +1 @@ +export * from "./ImportVideo"; diff --git a/packages/web-backend/src/S3Buckets/S3BucketAccess.ts b/packages/web-backend/src/S3Buckets/S3BucketAccess.ts index 02573f105..49af73f10 100644 --- a/packages/web-backend/src/S3Buckets/S3BucketAccess.ts +++ b/packages/web-backend/src/S3Buckets/S3BucketAccess.ts @@ -108,7 +108,7 @@ export class S3BucketAccess extends Effect.Service()( putObject: ( key: string, body: StreamingBlobPayloadInputTypes, - fields?: { contentType?: string }, + fields?: { contentType?: string; contentLength?: number }, ) => wrapS3Promise((provider) => provider.getInternal.pipe( @@ -119,10 +119,13 @@ export class S3BucketAccess extends Effect.Service()( Key: key, Body: body, ContentType: fields?.contentType, + ContentLength: fields?.contentLength, }), ), ), ), + ).pipe( + Effect.withSpan("S3BucketAccess.putObject", { attributes: { key } }), ), /** Copy an object within the same bucket */ copyObject: ( diff --git a/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts b/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts index 4c483ae51..eb551f7cc 100644 --- a/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts +++ b/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts @@ -48,7 +48,25 @@ export class S3BucketsRepo extends Effect.Service()( }), ); - return { getForVideo, getById }; + const getForUser = Effect.fn("S3BucketsRepo.getForUser")( + (userId: string) => + Effect.gen(function* () { + const [res] = yield* db.execute((db) => + db + .select({ bucket: Db.s3Buckets }) + .from(Db.s3Buckets) + .where(Dz.eq(Db.s3Buckets.ownerId, userId)), + ); + + return Option.fromNullable(res).pipe( + Option.map((v) => + S3Bucket.decodeSync({ ...v.bucket, name: v.bucket.bucketName }), + ), + ); + }), + ); + + return { getForVideo, getById, getForUser }; }), }, ) {} diff --git a/packages/web-backend/src/S3Buckets/index.ts b/packages/web-backend/src/S3Buckets/index.ts index 0d4507d8e..24d4b2d29 100644 --- a/packages/web-backend/src/S3Buckets/index.ts +++ b/packages/web-backend/src/S3Buckets/index.ts @@ -133,8 +133,45 @@ export class S3Buckets extends Effect.Service()("S3Buckets", { ), ); + const getProvider = Effect.fn("S3Buckets.getProviderLayer")(function* ( + customBucket: Option.Option, + ) { + const layer = yield* Option.match(customBucket, { + onNone: () => { + const provider = Layer.succeed(S3BucketClientProvider, { + getInternal: Effect.succeed(createDefaultClient(true)), + getPublic: Effect.succeed(createDefaultClient(false)), + bucket: defaultConfigs.bucket, + }); + + return Option.match(cloudfrontBucketAccess, { + onSome: (access) => access, + onNone: () => defaultBucketAccess, + }).pipe(Layer.merge(provider), Effect.succeed); + }, + onSome: (customBucket) => + Effect.gen(function* () { + const bucket = yield* Effect.promise(() => + decrypt(customBucket.name), + ); + + const provider = Layer.succeed(S3BucketClientProvider, { + getInternal: Effect.promise(() => + createBucketClient(customBucket), + ), + getPublic: Effect.promise(() => createBucketClient(customBucket)), + bucket, + }); + + return Layer.merge(defaultBucketAccess, provider); + }), + }); + + return [layer, customBucket] as const; + }); + return { - getProviderLayer: Effect.fn("S3Buckets.getProviderLayer")(function* ( + getProviderById: Effect.fn("S3Buckets.getProviderById")(function* ( bucketId: Option.Option, ) { const customBucket = yield* bucketId.pipe( @@ -143,38 +180,16 @@ export class S3Buckets extends Effect.Service()("S3Buckets", { Effect.map(Option.flatten), ); - let layer; - - if (Option.isNone(customBucket)) { - const provider = Layer.succeed(S3BucketClientProvider, { - getInternal: Effect.succeed(createDefaultClient(true)), - getPublic: Effect.succeed(createDefaultClient(false)), - bucket: defaultConfigs.bucket, - }); + return yield* getProvider(customBucket); + }), + getProviderForUser: Effect.fn("S3Buckets.getProviderForUser")(function* ( + userId: string, + ) { + const customBucket = yield* repo + .getForUser(userId) + .pipe(Effect.option, Effect.map(Option.flatten)); - layer = Option.match(cloudfrontBucketAccess, { - onSome: (access) => access, - onNone: () => defaultBucketAccess, - }).pipe(Layer.merge(provider)); - } else { - layer = defaultBucketAccess.pipe( - Layer.merge( - Layer.succeed(S3BucketClientProvider, { - getInternal: Effect.promise(() => - createBucketClient(customBucket.value), - ), - getPublic: Effect.promise(() => - createBucketClient(customBucket.value), - ), - bucket: yield* Effect.promise(() => - decrypt(customBucket.value.name), - ), - }), - ), - ); - } - - return [layer, customBucket] as const; + return yield* getProvider(customBucket); }), }; }), diff --git a/packages/web-backend/src/Videos/index.ts b/packages/web-backend/src/Videos/index.ts index 720e77df5..c9d598909 100644 --- a/packages/web-backend/src/Videos/index.ts +++ b/packages/web-backend/src/Videos/index.ts @@ -2,6 +2,7 @@ import * as Db from "@cap/database/schema"; import { CurrentUser, Policy, Video } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Array, Effect, Option, pipe } from "effect"; + import { Database } from "../Database"; import { S3Buckets } from "../S3Buckets"; import { S3BucketAccess } from "../S3Buckets/S3BucketAccess"; @@ -38,7 +39,7 @@ export class Videos extends Effect.Service()("Videos", { Effect.flatMap(Effect.catchAll(() => new Video.NotFoundError())), ); - const [S3ProviderLayer] = yield* s3Buckets.getProviderLayer( + const [S3ProviderLayer] = yield* s3Buckets.getProviderById( video.bucketId, ); @@ -78,7 +79,7 @@ export class Videos extends Effect.Service()("Videos", { Policy.withPolicy(policy.isOwner(videoId)), ); - const [S3ProviderLayer] = yield* s3Buckets.getProviderLayer( + const [S3ProviderLayer] = yield* s3Buckets.getProviderById( video.bucketId, ); diff --git a/packages/web-backend/src/Workflows.ts b/packages/web-backend/src/Workflows.ts new file mode 100644 index 000000000..602903474 --- /dev/null +++ b/packages/web-backend/src/Workflows.ts @@ -0,0 +1,5 @@ +import { Layer } from "effect"; + +import { LoomImportVideoLive } from "./Loom"; + +export const WorkflowsLayer = Layer.mergeAll(LoomImportVideoLive); diff --git a/packages/web-backend/src/index.ts b/packages/web-backend/src/index.ts index dde8f5b34..fb0714052 100644 --- a/packages/web-backend/src/index.ts +++ b/packages/web-backend/src/index.ts @@ -3,8 +3,10 @@ import "server-only"; export * from "./Auth"; export * from "./Database"; export { Folders } from "./Folders"; +export * from "./Loom"; export * from "./Rpcs"; export { S3Buckets } from "./S3Buckets"; export { S3BucketAccess } from "./S3Buckets/S3BucketAccess"; export { Videos } from "./Videos"; export { VideosPolicy } from "./Videos/VideosPolicy"; +export * from "./Workflows"; diff --git a/packages/web-domain/package.json b/packages/web-domain/package.json index 49ba61d0e..6c768bc6b 100644 --- a/packages/web-domain/package.json +++ b/packages/web-domain/package.json @@ -6,7 +6,8 @@ "type": "module", "dependencies": { "@effect/platform": "^0.90.1", - "@effect/rpc": "^0.68.3", - "effect": "^3.17.7" + "@effect/rpc": "^0.69.2", + "@effect/workflow": "^0.9.5", + "effect": "^3.17.13" } } diff --git a/packages/web-domain/src/Loom/ImportVideo.ts b/packages/web-domain/src/Loom/ImportVideo.ts new file mode 100644 index 000000000..8dc14a46c --- /dev/null +++ b/packages/web-domain/src/Loom/ImportVideo.ts @@ -0,0 +1,14 @@ +import { Workflow } from "@effect/workflow"; +import { Schema } from "effect"; + +export const ImportVideo = Workflow.make({ + name: "LoomImportVideo", + payload: { + userId: Schema.String, + loomVideoId: Schema.String, + loomOrgId: Schema.String, + orgId: Schema.String, + downloadUrl: Schema.String, + }, + idempotencyKey: (p) => `${p.loomOrgId}-${p.loomVideoId}`, +}); diff --git a/packages/web-domain/src/Loom/index.ts b/packages/web-domain/src/Loom/index.ts new file mode 100644 index 000000000..e5a1ab03d --- /dev/null +++ b/packages/web-domain/src/Loom/index.ts @@ -0,0 +1 @@ +export * from "./ImportVideo"; diff --git a/packages/web-domain/src/Rpcs.ts b/packages/web-domain/src/Rpcs.ts index 747359980..64bcb65b6 100644 --- a/packages/web-domain/src/Rpcs.ts +++ b/packages/web-domain/src/Rpcs.ts @@ -1,4 +1,6 @@ +import { RpcGroup } from "@effect/rpc"; + import { FolderRpcs } from "./Folder"; import { VideoRpcs } from "./Video"; -export const Rpcs = VideoRpcs.merge(FolderRpcs); +export const Rpcs = RpcGroup.make().merge(VideoRpcs, FolderRpcs); diff --git a/packages/web-domain/src/Workflows.ts b/packages/web-domain/src/Workflows.ts new file mode 100644 index 000000000..396d0ca49 --- /dev/null +++ b/packages/web-domain/src/Workflows.ts @@ -0,0 +1,3 @@ +import * as Loom from "./Loom"; + +export const Workflows = [Loom.ImportVideo] as const; diff --git a/packages/web-domain/src/index.ts b/packages/web-domain/src/index.ts index 0295b50cc..94bea9227 100644 --- a/packages/web-domain/src/index.ts +++ b/packages/web-domain/src/index.ts @@ -1,8 +1,10 @@ export * from "./Authentication"; export * from "./Errors"; export * as Folder from "./Folder"; +export * as Loom from "./Loom"; export * from "./Organisation"; export * as Policy from "./Policy"; export { Rpcs } from "./Rpcs"; export * as S3Bucket from "./S3Bucket"; export * as Video from "./Video"; +export * as Workflows from "./Workflows"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db3514f84..29a3215cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,8 +162,8 @@ importers: specifier: npm:class-variance-authority@^0.7.0 version: class-variance-authority@0.7.1 effect: - specifier: ^3.17.7 - version: 3.17.7 + specifier: ^3.17.13 + version: 3.17.13 mp4box: specifier: ^0.5.2 version: 0.5.4 @@ -468,15 +468,21 @@ importers: '@dub/analytics': specifier: ^0.0.27 version: 0.0.27 + '@effect/cluster': + specifier: ^0.48.6 + version: 0.48.6(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(@effect/sql@0.44.2(@effect/experimental@0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1))(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(@effect/workflow@0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13))(effect@3.17.13) '@effect/opentelemetry': specifier: ^0.56.1 - version: 0.56.1(@effect/platform@0.90.2(effect@3.17.7))(@opentelemetry/api@1.9.0)(@opentelemetry/resources@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-logs@0.203.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-node@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-web@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.36.0)(effect@3.17.7) + version: 0.56.1(@effect/platform@0.90.2(effect@3.17.13))(@opentelemetry/api@1.9.0)(@opentelemetry/resources@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-logs@0.203.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-node@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-web@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.36.0)(effect@3.17.13) '@effect/platform': specifier: ^0.90.1 - version: 0.90.2(effect@3.17.7) + version: 0.90.2(effect@3.17.13) '@effect/rpc': - specifier: ^0.68.3 - version: 0.68.4(@effect/platform@0.90.2(effect@3.17.7))(effect@3.17.7) + specifier: ^0.69.2 + version: 0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13) + '@effect/workflow': + specifier: ^0.9.5 + version: 0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13) '@fortawesome/free-brands-svg-icons': specifier: ^6.7.2 version: 6.7.2 @@ -616,8 +622,8 @@ importers: specifier: ^0.64.0 version: 0.64.0(@modelcontextprotocol/sdk@1.6.1)(zod@3.25.76) effect: - specifier: ^3.17.7 - version: 3.17.7 + specifier: ^3.17.13 + version: 3.17.13 file-saver: specifier: ^2.0.5 version: 2.0.5 @@ -1190,10 +1196,10 @@ importers: dependencies: '@effect/platform': specifier: ^0.90.1 - version: 0.90.2(effect@3.17.7) + version: 0.90.2(effect@3.17.13) effect: - specifier: ^3.17.7 - version: 3.17.7 + specifier: ^3.17.13 + version: 3.17.13 packages/web-backend: dependencies: @@ -1218,12 +1224,18 @@ importers: '@cap/web-domain': specifier: workspace:* version: link:../web-domain + '@effect/cluster': + specifier: ^0.48.6 + version: 0.48.6(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(@effect/sql@0.44.2(@effect/experimental@0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1))(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(@effect/workflow@0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13))(effect@3.17.13) '@effect/platform': specifier: ^0.90.1 - version: 0.90.2(effect@3.17.7) + version: 0.90.2(effect@3.17.13) '@effect/rpc': - specifier: ^0.68.3 - version: 0.68.4(@effect/platform@0.90.2(effect@3.17.7))(effect@3.17.7) + specifier: ^0.69.2 + version: 0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13) + '@effect/workflow': + specifier: ^0.9.5 + version: 0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13) '@smithy/types': specifier: ^4.3.1 version: 4.3.1 @@ -1231,8 +1243,8 @@ importers: specifier: 0.43.1 version: 0.43.1(@cloudflare/workers-types@4.20250507.0)(@opentelemetry/api@1.9.0)(@planetscale/database@1.19.0)(mysql2@3.14.1) effect: - specifier: ^3.17.7 - version: 3.17.7 + specifier: ^3.17.13 + version: 3.17.13 next: specifier: 14.2.3 version: 14.2.3(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) @@ -1244,13 +1256,16 @@ importers: dependencies: '@effect/platform': specifier: ^0.90.1 - version: 0.90.2(effect@3.17.7) + version: 0.90.2(effect@3.17.13) '@effect/rpc': - specifier: ^0.68.3 - version: 0.68.4(@effect/platform@0.90.2(effect@3.17.7))(effect@3.17.7) + specifier: ^0.69.2 + version: 0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13) + '@effect/workflow': + specifier: ^0.9.5 + version: 0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13) effect: - specifier: ^3.17.7 - version: 3.17.7 + specifier: ^3.17.13 + version: 3.17.13 packages: @@ -1948,6 +1963,28 @@ packages: '@effect-ts/system@0.57.5': resolution: {integrity: sha512-/crHGujo0xnuHIYNc1VgP0HGJGFSoSqq88JFXe6FmFyXPpWt8Xu39LyLg7rchsxfXFeEdA9CrIZvLV5eswXV5g==} + '@effect/cluster@0.48.6': + resolution: {integrity: sha512-ZrFOdAZhEiNRr0R6H+yP17aoQ5kWqU6MR8awV074GLGKd6Hyocn+9GNPGU0qvbP48Iad1j707J7b/nfvhVF8TA==} + peerDependencies: + '@effect/platform': ^0.90.9 + '@effect/rpc': ^0.69.2 + '@effect/sql': ^0.44.2 + '@effect/workflow': ^0.9.5 + effect: ^3.17.13 + + '@effect/experimental@0.54.6': + resolution: {integrity: sha512-UqHMvCQmrZT6kUVoUC0lqyno4Yad+j9hBGCdUjW84zkLwAq08tPqySiZUKRwY+Ae5B2Ab8rISYJH7nQvct9DMQ==} + peerDependencies: + '@effect/platform': ^0.90.2 + effect: ^3.17.7 + ioredis: ^5 + lmdb: ^3 + peerDependenciesMeta: + ioredis: + optional: true + lmdb: + optional: true + '@effect/language-service@0.34.0': resolution: {integrity: sha512-gweg1jchswd0zN6FmLR08e5IkSTdW1jNZn2VMdyxuSPhMGNHgM14KvFcVm0N4c6a0GxyKZ00KE/9EkkzZZn7eQ==} hasBin: true @@ -1986,12 +2023,26 @@ packages: peerDependencies: effect: ^3.17.7 - '@effect/rpc@0.68.4': - resolution: {integrity: sha512-iFGqBtZjjatNWwgDCCajYZvQSHc55XyZEdmHGqrKS6UfdRsV7qUSPgnplMf++tIIyGme7IbAjD/3VJdbHI5Gzg==} + '@effect/rpc@0.69.2': + resolution: {integrity: sha512-h6+e3JsIz5rmEZfldxNiNoXyQgMTB7VjDuoF8LPsOxobQZDKPgGE9BMnEQYqiVWRA2bTORkXK14rFZXzU1yyPg==} peerDependencies: - '@effect/platform': ^0.90.2 + '@effect/platform': ^0.90.6 + effect: ^3.17.11 + + '@effect/sql@0.44.2': + resolution: {integrity: sha512-DEcvriHvj88zu7keruH9NcHQzam7yQzLNLJO6ucDXMCAwWzYZSJOsmkxBznRFv8ylFtccSclKH2fuj+wRKPjCQ==} + peerDependencies: + '@effect/experimental': ^0.54.6 + '@effect/platform': ^0.90.4 effect: ^3.17.7 + '@effect/workflow@0.9.5': + resolution: {integrity: sha512-4039fgsa+kI7NePZ4v0qT8lzN5cxXXK+8lgHMjScngabQYdSKlk0I3YpqzktVrHAYS3bKUSeU/W7p4Ww/ub4hg==} + peerDependencies: + '@effect/platform': ^0.90.8 + '@effect/rpc': ^0.69.2 + effect: ^3.17.13 + '@emnapi/core@1.4.3': resolution: {integrity: sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g==} @@ -8297,8 +8348,8 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - effect@3.17.7: - resolution: {integrity: sha512-dpt0ONUn3zzAuul6k4nC/coTTw27AL5nhkORXgTi6NfMPzqWYa1M05oKmOMTxpVSTKepqXVcW9vIwkuaaqx9zA==} + effect@3.17.13: + resolution: {integrity: sha512-JMz5oBxs/6mu4FP9Csjub4jYMUwMLrp+IzUmSDVIzn2NoeoyOXMl7x1lghfr3dLKWffWrdnv/d8nFFdgrHXPqw==} ejs@3.1.10: resolution: {integrity: sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==} @@ -12647,7 +12698,7 @@ packages: superagent@8.1.2: resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==} engines: {node: '>=6.4.0 <13 || >=14'} - deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net supertest@6.3.4: resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==} @@ -14610,8 +14661,8 @@ snapshots: dependencies: '@babel/parser': 7.27.5 '@babel/types': 7.27.6 - '@jridgewell/gen-mapping': 0.3.8 - '@jridgewell/trace-mapping': 0.3.25 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 jsesc: 3.1.0 '@babel/helper-compilation-targets@7.27.2': @@ -15132,13 +15183,29 @@ snapshots: '@effect-ts/system@0.57.5': {} + '@effect/cluster@0.48.6(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(@effect/sql@0.44.2(@effect/experimental@0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1))(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(@effect/workflow@0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13))(effect@3.17.13)': + dependencies: + '@effect/platform': 0.90.2(effect@3.17.13) + '@effect/rpc': 0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13) + '@effect/sql': 0.44.2(@effect/experimental@0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1))(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13) + '@effect/workflow': 0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13) + effect: 3.17.13 + + '@effect/experimental@0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1)': + dependencies: + '@effect/platform': 0.90.2(effect@3.17.13) + effect: 3.17.13 + uuid: 11.1.0 + optionalDependencies: + ioredis: 5.6.1 + '@effect/language-service@0.34.0': {} - '@effect/opentelemetry@0.56.1(@effect/platform@0.90.2(effect@3.17.7))(@opentelemetry/api@1.9.0)(@opentelemetry/resources@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-logs@0.203.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-node@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-web@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.36.0)(effect@3.17.7)': + '@effect/opentelemetry@0.56.1(@effect/platform@0.90.2(effect@3.17.13))(@opentelemetry/api@1.9.0)(@opentelemetry/resources@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-logs@0.203.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-node@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-web@2.0.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.36.0)(effect@3.17.13)': dependencies: - '@effect/platform': 0.90.2(effect@3.17.7) + '@effect/platform': 0.90.2(effect@3.17.13) '@opentelemetry/semantic-conventions': 1.36.0 - effect: 3.17.7 + effect: 3.17.13 optionalDependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/resources': 2.0.1(@opentelemetry/api@1.9.0) @@ -15148,18 +15215,31 @@ snapshots: '@opentelemetry/sdk-trace-node': 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-web': 2.0.1(@opentelemetry/api@1.9.0) - '@effect/platform@0.90.2(effect@3.17.7)': + '@effect/platform@0.90.2(effect@3.17.13)': dependencies: '@opentelemetry/semantic-conventions': 1.36.0 - effect: 3.17.7 + effect: 3.17.13 find-my-way-ts: 0.1.6 msgpackr: 1.11.4 multipasta: 0.2.7 - '@effect/rpc@0.68.4(@effect/platform@0.90.2(effect@3.17.7))(effect@3.17.7)': + '@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)': + dependencies: + '@effect/platform': 0.90.2(effect@3.17.13) + effect: 3.17.13 + + '@effect/sql@0.44.2(@effect/experimental@0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1))(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)': + dependencies: + '@effect/experimental': 0.54.6(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13)(ioredis@5.6.1) + '@effect/platform': 0.90.2(effect@3.17.13) + effect: 3.17.13 + uuid: 11.1.0 + + '@effect/workflow@0.9.5(@effect/platform@0.90.2(effect@3.17.13))(@effect/rpc@0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13))(effect@3.17.13)': dependencies: - '@effect/platform': 0.90.2(effect@3.17.7) - effect: 3.17.7 + '@effect/platform': 0.90.2(effect@3.17.13) + '@effect/rpc': 0.69.2(@effect/platform@0.90.2(effect@3.17.13))(effect@3.17.13) + effect: 3.17.13 '@emnapi/core@1.4.3': dependencies: @@ -16161,7 +16241,7 @@ snapshots: '@jest/source-map@29.6.3': dependencies: - '@jridgewell/trace-mapping': 0.3.25 + '@jridgewell/trace-mapping': 0.3.31 callsites: 3.1.0 graceful-fs: 4.2.11 @@ -16230,8 +16310,8 @@ snapshots: '@jridgewell/source-map@0.3.6': dependencies: - '@jridgewell/gen-mapping': 0.3.8 - '@jridgewell/trace-mapping': 0.3.25 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 '@jridgewell/sourcemap-codec@1.5.0': {} @@ -16240,7 +16320,7 @@ snapshots: '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@jridgewell/trace-mapping@0.3.31': dependencies: @@ -21976,7 +22056,7 @@ snapshots: ee-first@1.1.1: {} - effect@3.17.7: + effect@3.17.13: dependencies: '@standard-schema/spec': 1.0.0 fast-check: 3.23.2 @@ -28669,7 +28749,7 @@ snapshots: v8-to-istanbul@9.3.0: dependencies: - '@jridgewell/trace-mapping': 0.3.25 + '@jridgewell/trace-mapping': 0.3.31 '@types/istanbul-lib-coverage': 2.0.6 convert-source-map: 2.0.0 diff --git a/scripts/env-cli.js b/scripts/env-cli.js index 57c577d45..d54af8813 100644 --- a/scripts/env-cli.js +++ b/scripts/env-cli.js @@ -18,7 +18,7 @@ const DOCKER_S3_ENVS = { secretKey: "capS3root", bucket: "capso", region: "us-east-1", - endpoint: "http://localhost:3902", + endpoint: "http://localhost:9000", }; const DOCKER_DB_ENVS = { From cddc2ce5645d0b5ea98e22d89f2f16cf326bc0af Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Fri, 19 Sep 2025 15:07:21 +0800 Subject: [PATCH 2/7] import videos --- apps/desktop/package.json | 2 +- apps/desktop/src-tauri/tauri.conf.json | 2 +- apps/discord-bot/package.json | 2 +- apps/tasks/package.json | 1 - apps/web/app/(org)/dashboard/admin/page.tsx | 5 +- apps/web/app/(org)/verify-otp/form.tsx | 1 + .../web/app/api/desktop/[...route]/session.ts | 2 - apps/web/app/api/erpc/workflows/route.ts | 20 - apps/web/app/api/loom/[...route]/route.ts | 84 ++ apps/web/app/api/playlist/route.ts | 2 +- apps/web/app/api/test/[...route]/route.ts | 43 - apps/web/app/api/upload/[...route]/signed.ts | 4 +- apps/web/lib/server.ts | 97 +- apps/web/lib/tracing-server.ts | 5 + apps/web/next.config.mjs | 21 +- apps/web/package.json | 5 + apps/workflow-manager/package.json | 15 + apps/workflow-manager/src/index.ts | 23 + apps/workflow-runner/package.json | 23 + apps/workflow-runner/src/index.ts | 67 ++ biome.json | 5 +- package.json | 6 + .../{auth-options.tsx => auth-options.ts} | 19 +- packages/database/auth/drizzle-adapter.ts | 5 +- packages/database/index.ts | 3 +- packages/database/package.json | 13 + packages/database/schema.ts | 21 +- packages/env/index.ts | 4 +- packages/env/package.json | 1 + packages/env/server.ts | 2 + packages/utils/package.json | 8 +- packages/utils/src/index.ts | 5 + packages/utils/src/index.tsx | 5 - packages/web-backend/src/Auth.ts | 45 +- packages/web-backend/src/Database.ts | 30 +- .../web-backend/src/Folders/FoldersPolicy.ts | 4 +- .../web-backend/src/Folders/FoldersRpcs.ts | 2 +- packages/web-backend/src/Folders/index.ts | 7 +- packages/web-backend/src/Loom/ImportVideo.ts | 181 +++- packages/web-backend/src/Loom/index.ts | 2 +- .../src/Organisations/OrganisationsRepo.ts | 3 +- packages/web-backend/src/Rpcs.ts | 8 +- .../src/S3Buckets/S3BucketAccess.ts | 64 +- .../src/S3Buckets/S3BucketsRepo.ts | 3 +- packages/web-backend/src/S3Buckets/index.ts | 12 +- packages/web-backend/src/Spaces/SpacesRepo.ts | 3 +- .../web-backend/src/Videos/VideosPolicy.ts | 8 +- packages/web-backend/src/Videos/VideosRepo.ts | 144 +-- packages/web-backend/src/Videos/VideosRpcs.ts | 6 +- packages/web-backend/src/Videos/index.ts | 25 +- packages/web-backend/src/Workflows.ts | 8 +- packages/web-backend/src/index.ts | 22 +- packages/web-domain/src/Authentication.ts | 3 +- packages/web-domain/src/Folder.ts | 7 +- packages/web-domain/src/Loom/ImportVideo.ts | 14 - packages/web-domain/src/Loom/index.ts | 1 - packages/web-domain/src/Policy.ts | 3 +- packages/web-domain/src/Rpcs.ts | 4 +- packages/web-domain/src/S3Bucket.ts | 2 +- packages/web-domain/src/Video.ts | 68 +- packages/web-domain/src/Workflows.ts | 25 +- packages/web-domain/src/index.ts | 19 +- packages/web-domain/src/utils.ts | 5 + pnpm-lock.yaml | 869 ++++++++++++++++-- 64 files changed, 1639 insertions(+), 479 deletions(-) delete mode 100644 apps/web/app/api/erpc/workflows/route.ts create mode 100644 apps/web/app/api/loom/[...route]/route.ts delete mode 100644 apps/web/app/api/test/[...route]/route.ts create mode 100644 apps/web/lib/tracing-server.ts create mode 100644 apps/workflow-manager/package.json create mode 100644 apps/workflow-manager/src/index.ts create mode 100644 apps/workflow-runner/package.json create mode 100644 apps/workflow-runner/src/index.ts rename packages/database/auth/{auth-options.tsx => auth-options.ts} (95%) create mode 100644 packages/utils/src/index.ts delete mode 100644 packages/utils/src/index.tsx delete mode 100644 packages/web-domain/src/Loom/ImportVideo.ts delete mode 100644 packages/web-domain/src/Loom/index.ts create mode 100644 packages/web-domain/src/utils.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 0e053b713..cfc69739b 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -5,7 +5,7 @@ "dev": "pnpm -w cap-setup && dotenv -e ../../.env -- pnpm run preparescript && dotenv -e ../../.env -- pnpm tauri dev", "build:tauri": "dotenv -e ../../.env -- pnpm run preparescript && dotenv -e ../../.env -- pnpm tauri build", "preparescript": "node scripts/prepare.js", - "localdev": "dotenv -e ../../.env -- vinxi dev --port 3001", + "localdev": "dotenv -e ../../.env -- vinxi dev --port 3002", "build": "vinxi build", "tauri": "tauri" }, diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index 214c68a74..5f2a676eb 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -5,7 +5,7 @@ "mainBinaryName": "Cap - Development", "build": { "beforeDevCommand": "pnpm localdev", - "devUrl": "http://localhost:3001", + "devUrl": "http://localhost:3002", "beforeBuildCommand": "pnpm turbo build --filter @cap/desktop", "frontendDist": "../.output/public" }, diff --git a/apps/discord-bot/package.json b/apps/discord-bot/package.json index 96ddf0e7b..f2b68587c 100644 --- a/apps/discord-bot/package.json +++ b/apps/discord-bot/package.json @@ -3,7 +3,7 @@ "private": true, "scripts": { "deploy": "wrangler deploy", - "dev": "wrangler dev", + "bot-dev": "wrangler dev", "start": "wrangler dev", "test": "vitest", "cf-typegen": "wrangler types" diff --git a/apps/tasks/package.json b/apps/tasks/package.json index 30ee20f4d..2010d3489 100644 --- a/apps/tasks/package.json +++ b/apps/tasks/package.json @@ -5,7 +5,6 @@ "main": "src/index.ts", "scripts": { "start": "node dist/src/index.js", - "dev": "ts-node src/index.ts", "build": "tsc", "start:dist": "node dist/src/index.js", "test": "jest", diff --git a/apps/web/app/(org)/dashboard/admin/page.tsx b/apps/web/app/(org)/dashboard/admin/page.tsx index 1ba406285..137fbaef4 100644 --- a/apps/web/app/(org)/dashboard/admin/page.tsx +++ b/apps/web/app/(org)/dashboard/admin/page.tsx @@ -5,7 +5,10 @@ import AdminDashboardClient from "./AdminDashboardClient"; export default async function AdminDashboard() { const currentUser = await getCurrentUser(); - if (currentUser?.email !== "richie@mcilroy.co") { + if ( + currentUser?.email !== "richie@mcilroy.co" || + currentUser.email.endsWith("@cap.so") + ) { redirect("/dashboard"); } diff --git a/apps/web/app/(org)/verify-otp/form.tsx b/apps/web/app/(org)/verify-otp/form.tsx index f2effe76c..e40cdf902 100644 --- a/apps/web/app/(org)/verify-otp/form.tsx +++ b/apps/web/app/(org)/verify-otp/form.tsx @@ -78,6 +78,7 @@ export function VerifyOTPForm({ // shoutout https://github.com/buoyad/Tally/pull/14 const res = await fetch( `/api/auth/callback/email?email=${encodeURIComponent(email)}&token=${encodeURIComponent(otpCode)}&callbackUrl=${encodeURIComponent("/login-success")}`, + { redirect: "manual" }, ); if (!res.url.includes("/login-success")) { diff --git a/apps/web/app/api/desktop/[...route]/session.ts b/apps/web/app/api/desktop/[...route]/session.ts index 72b99f8ff..9082d8b66 100644 --- a/apps/web/app/api/desktop/[...route]/session.ts +++ b/apps/web/app/api/desktop/[...route]/session.ts @@ -1,12 +1,10 @@ import { db } from "@cap/database"; -import { authOptions } from "@cap/database/auth/auth-options"; import { getCurrentUser } from "@cap/database/auth/session"; import { authApiKeys } from "@cap/database/schema"; import { serverEnv } from "@cap/env"; import { zValidator } from "@hono/zod-validator"; import { Hono } from "hono"; import { getCookie } from "hono/cookie"; -import { getServerSession } from "next-auth"; import { decode } from "next-auth/jwt"; import { z } from "zod"; diff --git a/apps/web/app/api/erpc/workflows/route.ts b/apps/web/app/api/erpc/workflows/route.ts deleted file mode 100644 index 655c86504..000000000 --- a/apps/web/app/api/erpc/workflows/route.ts +++ /dev/null @@ -1,20 +0,0 @@ -// import { Workflows } from "@cap/web-domain"; -// import { HttpServer } from "@effect/platform"; -// import { RpcSerialization, RpcServer } from "@effect/rpc"; -// import { WorkflowProxy, WorkflowProxyServer } from "@effect/workflow"; -// import { Layer } from "effect"; -// import { Dependencies } from "@/lib/server"; - -// const { handler } = RpcServer.toWebHandler( -// WorkflowProxy.toRpcGroup(Workflows.Workflows), -// { -// layer: Layer.mergeAll( -// RpcSerialization.layerJson, -// HttpServer.layerContext, -// Dependencies, -// ), -// }, -// ); - -// export const GET = handler; -// export const POST = handler; diff --git a/apps/web/app/api/loom/[...route]/route.ts b/apps/web/app/api/loom/[...route]/route.ts new file mode 100644 index 000000000..a5ba75594 --- /dev/null +++ b/apps/web/app/api/loom/[...route]/route.ts @@ -0,0 +1,84 @@ +import { + CurrentUser, + HttpAuthMiddleware, + Loom, + Workflows, +} from "@cap/web-domain"; +import { + HttpApi, + HttpApiBuilder, + HttpApiEndpoint, + HttpApiError, + HttpApiGroup, +} from "@effect/platform"; +import { Effect, Layer, Option, Schema } from "effect"; +import { apiToHandler } from "@/lib/server"; + +export const revalidate = "force-dynamic"; + +class Api extends HttpApi.make("CapWebApi") + .add( + HttpApiGroup.make("root").add( + HttpApiEndpoint.post("loom")`/import-video` + .setPayload( + Schema.Struct({ + loom: Schema.Struct({ + downloadUrl: Schema.String, + videoId: Schema.String, + }), + }), + ) + .middleware(HttpAuthMiddleware) + .addError(HttpApiError.InternalServerError), + ), + ) + .prefix("/api/loom") {} + +const ApiLive = HttpApiBuilder.api(Api).pipe( + Layer.provide( + HttpApiBuilder.group(Api, "root", (handlers) => + handlers.handle( + "loom", + Effect.fn( + function* ({ payload }) { + const { workflows } = yield* Workflows.HttpClient; + const user = yield* CurrentUser; + + yield* workflows.LoomImportVideo({ + payload: { + cap: { + userId: user.id, + orgId: user.activeOrgId, + }, + loom: { + userId: "loomVideoId123", + orgId: "loomOrgId123", + video: { + id: payload.loom.videoId, + name: "loom video name", + downloadUrl: payload.loom.downloadUrl, + width: Option.none(), + height: Option.none(), + durationSecs: Option.none(), + fps: Option.none(), + }, + }, + }, + }); + }, + (e) => + e.pipe( + Effect.tapDefect(Effect.log), + Effect.catchAll(() => new HttpApiError.InternalServerError()), + ), + ), + ), + ), + ), +); + +const { handler } = apiToHandler(ApiLive); + +export const GET = handler; +export const HEAD = handler; +export const POST = handler; diff --git a/apps/web/app/api/playlist/route.ts b/apps/web/app/api/playlist/route.ts index c34431805..3d706f1c4 100644 --- a/apps/web/app/api/playlist/route.ts +++ b/apps/web/app/api/playlist/route.ts @@ -61,7 +61,7 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( ); const [S3ProviderLayer, customBucket] = - yield* s3Buckets.getProviderById(video.bucketId); + yield* s3Buckets.getProviderForBucket(video.bucketId); return yield* getPlaylistResponse( video, diff --git a/apps/web/app/api/test/[...route]/route.ts b/apps/web/app/api/test/[...route]/route.ts deleted file mode 100644 index 326fea8bc..000000000 --- a/apps/web/app/api/test/[...route]/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Loom } from "@cap/web-domain"; -import { - HttpApi, - HttpApiBuilder, - HttpApiEndpoint, - HttpApiGroup, - HttpServerResponse, -} from "@effect/platform"; -import { Effect, Layer } from "effect"; -import { apiToHandler } from "@/lib/server"; - -export const revalidate = "force-dynamic"; - -class Api extends HttpApi.make("CapWebApi") - .add( - HttpApiGroup.make("root").add(HttpApiEndpoint.get("test")`/import-video`), - ) - .prefix("/api/test") {} - -const ApiLive = HttpApiBuilder.api(Api).pipe( - Layer.provide( - HttpApiBuilder.group(Api, "root", (handlers) => - handlers.handle( - "test", - Effect.fn(function* () { - yield* Loom.ImportVideo.execute({ - userId: "user123", - loomVideoId: "loomVideoId123", - loomOrgId: "loomOrgId123", - orgId: "orgId123", - downloadUrl: - "https://cdn.loom.com/sessions/thumbnails/95a01fba1f5f434da5af3cfbe567c6a7-10954f1f96d7b5c2.mp4", - }); - }), - ), - ), - ), -); - -const { handler } = apiToHandler(ApiLive); - -export const GET = handler; -export const HEAD = handler; diff --git a/apps/web/app/api/upload/[...route]/signed.ts b/apps/web/app/api/upload/[...route]/signed.ts index 06e8b132e..b4255a8ff 100644 --- a/apps/web/app/api/upload/[...route]/signed.ts +++ b/apps/web/app/api/upload/[...route]/signed.ts @@ -4,12 +4,12 @@ import { } from "@aws-sdk/client-cloudfront"; import { db, updateIfDefined } from "@cap/database"; import { s3Buckets, videos } from "@cap/database/schema"; -import type { VideoMetadata } from "@cap/database/types"; import { serverEnv } from "@cap/env"; import { zValidator } from "@hono/zod-validator"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { Hono } from "hono"; import { z } from "zod"; + import { createBucketProvider } from "@/utils/s3"; import { stringOrNumberOptional } from "@/utils/zod"; import { withAuth } from "../../utils"; diff --git a/apps/web/lib/server.ts b/apps/web/lib/server.ts index 8b5376e8f..3bb141b4d 100644 --- a/apps/web/lib/server.ts +++ b/apps/web/lib/server.ts @@ -1,51 +1,35 @@ import "server-only"; -import { db } from "@cap/database"; import { decrypt } from "@cap/database/crypto"; +import { serverEnv } from "@cap/env"; import { Database, - DatabaseError, Folders, HttpAuthMiddlewareLive, S3Buckets, Videos, VideosPolicy, - WorkflowsLayer, } from "@cap/web-backend"; -import { type HttpAuthMiddleware, Video } from "@cap/web-domain"; -import { - ClusterWorkflowEngine, - MessageStorage, - Runners, - Sharding, - ShardingConfig, - ShardManager, - ShardStorage, -} from "@effect/cluster"; +import { type HttpAuthMiddleware, Video, Workflows } from "@cap/web-domain"; import * as NodeSdk from "@effect/opentelemetry/NodeSdk"; import { FetchHttpClient, type HttpApi, HttpApiBuilder, + HttpApiClient, + HttpClient, + HttpClientRequest, HttpMiddleware, HttpServer, } from "@effect/platform"; +import { RpcClient } from "@effect/rpc"; import { Cause, Effect, Exit, Layer, ManagedRuntime, Option } from "effect"; import { isNotFoundError } from "next/dist/client/components/not-found"; import { cookies } from "next/headers"; - import { allowedOrigins } from "@/utils/cors"; import { getTracingConfig } from "./tracing"; -const DatabaseLive = Layer.sync(Database, () => ({ - execute: (cb) => - Effect.tryPromise({ - try: () => cb(db()), - catch: (error) => new DatabaseError({ message: String(error) }), - }), -})); - -const TracingLayer = NodeSdk.layer(getTracingConfig); +export const TracingLayer = NodeSdk.layer(getTracingConfig); const CookiePasswordAttachmentLive = Layer.effect( Video.VideoPasswordAttachment, @@ -60,28 +44,59 @@ const CookiePasswordAttachmentLive = Layer.effect( }), ); -const WorkflowEngine = ClusterWorkflowEngine.layer.pipe( - Layer.provideMerge(Sharding.layer), - Layer.provide(ShardManager.layerClientLocal), - Layer.provide(ShardStorage.layerNoop), - Layer.provide(Runners.layerNoop), - Layer.provideMerge(MessageStorage.layerMemory), - Layer.provide(ShardingConfig.layer()), +const WorkflowRpcClient = Layer.scoped( + Workflows.RpcClient, + Effect.gen(function* () { + const envs = Option.zipWith( + Option.fromNullable(serverEnv().REMOTE_WORKFLOW_URL), + Option.fromNullable(serverEnv().REMOTE_WORKFLOW_SECRET), + (l, r) => [l, r] as const, + ); + + return yield* Option.match(envs, { + onNone: () => + RpcClient.make(Workflows.RpcGroup).pipe( + Effect.provide( + RpcClient.layerProtocolHttp({ + url: "http://localhost:42169/rpc", + }).pipe(Layer.provide(Workflows.RpcSerialization)), + ), + ), + onSome: ([url, secret]) => + RpcClient.make(Workflows.RpcGroup).pipe( + Effect.provide( + RpcClient.layerProtocolHttp({ + url, + transformClient: HttpClient.mapRequest( + HttpClientRequest.setHeader("Authorization", `Token ${secret}`), + ), + }).pipe(Layer.provide(Workflows.RpcSerialization)), + ), + ), + }); + }), ); -export const Dependencies = WorkflowsLayer.pipe( - Layer.provideMerge( - Layer.mergeAll( - S3Buckets.Default, - Videos.Default, - VideosPolicy.Default, - Folders.Default, - FetchHttpClient.layer, - WorkflowEngine, - ).pipe(Layer.provideMerge(Layer.mergeAll(DatabaseLive, TracingLayer))), - ), +const WorkflowHttpClient = Layer.scoped( + Workflows.HttpClient, + Effect.gen(function* () { + const a = yield* HttpApiClient.make(Workflows.Api, { + baseUrl: "http://localhost:42169", + }); + return a; + }), ); +export const Dependencies = Layer.mergeAll( + S3Buckets.Default, + Videos.Default, + VideosPolicy.Default, + Folders.Default, + Database.Default, + WorkflowRpcClient, + WorkflowHttpClient, +).pipe(Layer.provideMerge(Layer.mergeAll(TracingLayer, FetchHttpClient.layer))); + // purposefully not exposed const EffectRuntime = ManagedRuntime.make(Dependencies); diff --git a/apps/web/lib/tracing-server.ts b/apps/web/lib/tracing-server.ts new file mode 100644 index 000000000..53b062d27 --- /dev/null +++ b/apps/web/lib/tracing-server.ts @@ -0,0 +1,5 @@ +import * as NodeSdk from "@effect/opentelemetry/NodeSdk"; + +import { getTracingConfig } from "./tracing"; + +export const TracingLayer = NodeSdk.layer(getTracingConfig); diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 0a30c0e48..0c3133bb7 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -52,7 +52,7 @@ const nextConfig = { process.env.NODE_ENV === "development" && { protocol: "http", hostname: "localhost", - port: "3902", + port: "9000", pathname: "**", }, ].filter(Boolean), @@ -110,6 +110,25 @@ const nextConfig = { // If the DOCKER_BUILD environment variable is set to true, we are output nextjs to standalone ready for docker deployment output: process.env.NEXT_PUBLIC_DOCKER_BUILD === "true" ? "standalone" : undefined, + // webpack: (config) => { + // config.module.rules.push({ + // test: /\.(?:js|ts)$/, + // use: [ + // { + // loader: "babel-loader", + // options: { + // presets: ["next/babel"], + // plugins: [ + // "@babel/plugin-transform-private-property-in-object", + // "@babel/plugin-transform-private-methods", + // ], + // }, + // }, + // ], + // }); + + // return config; + // }, }; export default nextConfig; diff --git a/apps/web/package.json b/apps/web/package.json index 15d31402a..21f25a1b4 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -30,7 +30,9 @@ "@effect/cluster": "^0.48.6", "@effect/opentelemetry": "^0.56.1", "@effect/platform": "^0.90.1", + "@effect/platform-node": "^0.96.1", "@effect/rpc": "^0.69.2", + "@effect/sql-mysql2": "^0.45.1", "@effect/workflow": "^0.9.5", "@fortawesome/free-brands-svg-icons": "^6.7.2", "@fortawesome/free-solid-svg-icons": "^6.7.2", @@ -121,6 +123,8 @@ "zod": "^3.25.76" }, "devDependencies": { + "@babel/plugin-transform-private-methods": "^7.27.1", + "@babel/plugin-transform-private-property-in-object": "^7.27.1", "@smithy/types": "^4.3.1", "@types/canvas-confetti": "^1.9.0", "@types/file-saver": "^2.0.7", @@ -133,6 +137,7 @@ "@types/react-responsive-masonry": "^2.6.0", "@types/uuid": "^9.0.8", "autoprefixer": "^10.4.14", + "babel-loader": "^10.0.0", "eslint": "^9.30.1", "eslint-config-next": "14.1.0", "postcss": "^8.4.23", diff --git a/apps/workflow-manager/package.json b/apps/workflow-manager/package.json new file mode 100644 index 000000000..9671eb4c4 --- /dev/null +++ b/apps/workflow-manager/package.json @@ -0,0 +1,15 @@ +{ + "name": "@cap/workflow-manager", + "type": "module", + "scripts": { + "dev": "dotenv -e ../../.env -- deno --allow-all --watch src/index.ts" + }, + "dependencies": { + "@effect/platform-node": "^0.96.1", + "@effect/sql-mysql2": "^0.45.1", + "effect": "^3.17.13" + }, + "devDependencies": { + "dotenv-cli": "^10.0.0" + } +} diff --git a/apps/workflow-manager/src/index.ts b/apps/workflow-manager/src/index.ts new file mode 100644 index 000000000..1d337c744 --- /dev/null +++ b/apps/workflow-manager/src/index.ts @@ -0,0 +1,23 @@ +import { + NodeClusterShardManagerSocket, + NodeRuntime, +} from "@effect/platform-node"; +import { MysqlClient } from "@effect/sql-mysql2"; +import { Config, Effect, Layer, Logger, Redacted } from "effect"; + +const DatabaseLive = Layer.unwrapEffect( + Effect.gen(function* () { + const url = Redacted.make(yield* Config.string("DATABASE_URL")); + + return MysqlClient.layer({ url }); + }), +); + +NodeClusterShardManagerSocket.layer({ + storage: "sql", +}).pipe( + Layer.provide(DatabaseLive), + Layer.provide(Logger.pretty), + Layer.launch, + NodeRuntime.runMain, +); diff --git a/apps/workflow-runner/package.json b/apps/workflow-runner/package.json new file mode 100644 index 000000000..7583d910b --- /dev/null +++ b/apps/workflow-runner/package.json @@ -0,0 +1,23 @@ +{ + "name": "@cap/workflow-runner", + "type": "module", + "scripts": { + "dev": "pnpm dotenv -e ../../.env -- deno run --allow-all --watch ./src/index.ts" + }, + "dependencies": { + "@cap/web-backend": "workspace:*", + "@cap/web-domain": "workspace:*", + "@effect/cluster": "^0.48.6", + "@effect/opentelemetry": "^0.56.1", + "@effect/platform": "^0.90.1", + "@effect/platform-node": "^0.96.1", + "@effect/sql-mysql2": "^0.45.1", + "@effect/workflow": "^0.9.5", + "@opentelemetry/exporter-trace-otlp-http": "^0.203.0", + "@opentelemetry/sdk-trace-base": "^2.0.1", + "effect": "^3.17.13" + }, + "devDependencies": { + "dotenv-cli": "^10.0.0" + } +} diff --git a/apps/workflow-runner/src/index.ts b/apps/workflow-runner/src/index.ts new file mode 100644 index 000000000..23317d780 --- /dev/null +++ b/apps/workflow-runner/src/index.ts @@ -0,0 +1,67 @@ +import { createServer } from "node:http"; +import { Database, S3Buckets, Videos, WorkflowsLayer } from "@cap/web-backend"; +import { Workflows } from "@cap/web-domain"; +import { ClusterWorkflowEngine, RunnerAddress } from "@effect/cluster"; +import * as NodeSdk from "@effect/opentelemetry/NodeSdk"; +import { FetchHttpClient, HttpApiBuilder, HttpServer } from "@effect/platform"; +import { + NodeClusterRunnerSocket, + NodeHttpServer, + NodeRuntime, +} from "@effect/platform-node"; +import { MysqlClient } from "@effect/sql-mysql2"; +import { WorkflowProxyServer } from "@effect/workflow"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { Config, Effect, Layer, Option } from "effect"; + +const SqlLayer = Layer.unwrapEffect( + Effect.gen(function* () { + const url = yield* Config.string("DATABASE_URL").pipe((v) => + Config.redacted(v), + ); + return MysqlClient.layer({ url }); + }), +); + +const ClusterWorkflowLive = ClusterWorkflowEngine.layer.pipe( + Layer.provide( + NodeClusterRunnerSocket.layer({ + storage: "sql", + shardingConfig: { + runnerAddress: Option.some(RunnerAddress.make("localhost", 42069)), + }, + }), + ), + Layer.provide(SqlLayer), +); + +const WorkflowApiLive = HttpApiBuilder.api(Workflows.Api).pipe( + Layer.provide( + WorkflowProxyServer.layerHttpApi( + Workflows.Api, + "workflows", + Workflows.Workflows, + ), + ), + Layer.provide(WorkflowsLayer), + Layer.provide(ClusterWorkflowLive), + HttpServer.withLogAddress, +); + +const TracingLayer = NodeSdk.layer(() => ({ + resource: { serviceName: "cap-workflow-runner" }, + spanProcessor: [new BatchSpanProcessor(new OTLPTraceExporter({}))], +})); + +HttpApiBuilder.serve().pipe( + Layer.provide(WorkflowApiLive), + Layer.provide(NodeHttpServer.layer(createServer, { port: 42169 })), + Layer.provide(Videos.Default), + Layer.provide(S3Buckets.Default), + Layer.provide(Database.Default), + Layer.provide(FetchHttpClient.layer), + Layer.provide(TracingLayer), + Layer.launch, + NodeRuntime.runMain, +); diff --git a/biome.json b/biome.json index f2a353d51..621aba760 100644 --- a/biome.json +++ b/biome.json @@ -20,7 +20,10 @@ "linter": { "enabled": true, "rules": { - "recommended": true + "recommended": true, + "suspicious": { + "noShadowRestrictedNames": "off" + } } }, "javascript": { diff --git a/package.json b/package.json index bee43acc5..ff4fce11d 100644 --- a/package.json +++ b/package.json @@ -37,5 +37,11 @@ "name": "cap", "engines": { "node": "20" + }, +"pnpm": { + "overrides": { + "undici": "5.28.4" + } } + } diff --git a/packages/database/auth/auth-options.tsx b/packages/database/auth/auth-options.ts similarity index 95% rename from packages/database/auth/auth-options.tsx rename to packages/database/auth/auth-options.ts index 31f797e93..92b4fb817 100644 --- a/packages/database/auth/auth-options.tsx +++ b/packages/database/auth/auth-options.ts @@ -1,7 +1,6 @@ +import crypto from "node:crypto"; import { serverEnv } from "@cap/env"; -import crypto from "crypto"; import { eq } from "drizzle-orm"; -import { cookies } from "next/headers"; import type { NextAuthOptions } from "next-auth"; import { getServerSession as _getServerSession } from "next-auth"; import type { Adapter } from "next-auth/adapters"; @@ -9,13 +8,14 @@ import EmailProvider from "next-auth/providers/email"; import GoogleProvider from "next-auth/providers/google"; import type { Provider } from "next-auth/providers/index"; import WorkOSProvider from "next-auth/providers/workos"; -import { db } from "../"; -import { dub } from "../dub"; -import { sendEmail } from "../emails/config"; -import { nanoId } from "../helpers"; -import { organizationMembers, organizations, users } from "../schema"; -import { isEmailAllowedForSignup } from "./domain-utils"; -import { DrizzleAdapter } from "./drizzle-adapter"; + +import { dub } from "../dub.ts"; +import { sendEmail } from "../emails/config.ts"; +import { nanoId } from "../helpers.ts"; +import { db } from "../index.ts"; +import { organizationMembers, organizations, users } from "../schema.ts"; +import { isEmailAllowedForSignup } from "./domain-utils.ts"; +import { DrizzleAdapter } from "./drizzle-adapter.ts"; export const config = { maxDuration: 120, @@ -136,6 +136,7 @@ export const authOptions = (): NextAuthOptions => { dbUser.activeOrganizationId === ""; if (needsOrganizationSetup) { + const { cookies } = await import("next/headers"); const dubId = cookies().get("dub_id")?.value; const dubPartnerData = cookies().get("dub_partner_data")?.value; diff --git a/packages/database/auth/drizzle-adapter.ts b/packages/database/auth/drizzle-adapter.ts index 14d61270a..94a196a02 100644 --- a/packages/database/auth/drizzle-adapter.ts +++ b/packages/database/auth/drizzle-adapter.ts @@ -3,8 +3,9 @@ import { and, eq } from "drizzle-orm"; import type { PlanetScaleDatabase } from "drizzle-orm/planetscale-serverless"; import type { Adapter } from "next-auth/adapters"; import type Stripe from "stripe"; -import { nanoId } from "../helpers"; -import { accounts, sessions, users, verificationTokens } from "../schema"; + +import { nanoId } from "../helpers.ts"; +import { accounts, sessions, users, verificationTokens } from "../schema.ts"; export function DrizzleAdapter(db: PlanetScaleDatabase): Adapter { return { diff --git a/packages/database/index.ts b/packages/database/index.ts index b29c14dfd..921de496f 100644 --- a/packages/database/index.ts +++ b/packages/database/index.ts @@ -1,11 +1,10 @@ -import { serverEnv } from "@cap/env"; import { Client, type Config } from "@planetscale/database"; import { sql } from "drizzle-orm"; import type { AnyMySqlColumn } from "drizzle-orm/mysql-core"; import { drizzle } from "drizzle-orm/planetscale-serverless"; function createDrizzle() { - const URL = serverEnv().DATABASE_URL; + const URL = process.env.DATABASE_URL!; let fetchHandler: Promise | undefined; diff --git a/packages/database/package.json b/packages/database/package.json index 5484a796d..e68264083 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -1,6 +1,7 @@ { "name": "@cap/database", "private": true, + "type": "module", "types": "./index.ts", "main": "index.js", "scripts": { @@ -16,6 +17,7 @@ "dependencies": { "@cap/env": "workspace:*", "@cap/web-domain": "workspace:*", + "@effect/sql-mysql2": "^0.45.1", "@mattrax/mysql-planetscale": "^0.0.3", "@paralleldrive/cuid2": "^2.2.2", "@planetscale/database": "^1.13.0", @@ -24,6 +26,7 @@ "@react-email/tailwind": "^1.0.5", "drizzle-orm": "0.43.1", "dub": "^0.64.0", + "effect": "^3.17.13", "nanoid": "^5.0.4", "next": "14.2.9", "next-auth": "^4.24.5", @@ -48,5 +51,15 @@ }, "engines": { "node": "20" + }, + "exports": { + ".": "./index.ts", + "./auth/auth-options": "./auth/auth-options.ts", + "./auth/session": "./auth/session.ts", + "./emails/config": "./emails/config.ts", + "./emails/*": "./emails/*.tsx", + "./schema": "./schema.ts", + "./crypto": "./crypto.ts", + "./helpers": "./helpers.ts" } } diff --git a/packages/database/schema.ts b/packages/database/schema.ts index fdb8beb8b..e92b318c2 100644 --- a/packages/database/schema.ts +++ b/packages/database/schema.ts @@ -8,6 +8,7 @@ import { int, json, mysqlTable, + primaryKey, text, timestamp, tinyint, @@ -15,8 +16,9 @@ import { varchar, } from "drizzle-orm/mysql-core"; import { relations } from "drizzle-orm/relations"; -import { nanoIdLength } from "./helpers"; -import type { VideoMetadata } from "./types"; + +import { nanoIdLength } from "./helpers.ts"; +import type { VideoMetadata } from "./types/index.ts"; const nanoId = customType<{ data: string; notNull: true }>({ dataType() { @@ -240,6 +242,8 @@ export const videos = mysqlTable( { id: nanoId("id").notNull().primaryKey().unique().$type(), ownerId: nanoId("ownerId").notNull(), + // TODO: make this non-null + orgId: nanoIdNullable("orgId"), name: varchar("name", { length: 255 }).notNull().default("My Video"), bucket: nanoIdNullable("bucket"), // in seconds @@ -648,3 +652,16 @@ export const videoUploads = mysqlTable("video_uploads", { startedAt: timestamp("started_at").notNull().defaultNow(), updatedAt: timestamp("updated_at").notNull().defaultNow(), }); + +export const importedVideos = mysqlTable( + "imported_videos", + { + id: nanoId("id").notNull(), + orgId: nanoIdNullable("orgId").notNull(), + source: varchar("source", { length: 255, enum: ["loom"] }).notNull(), + sourceId: varchar("source_id", { length: 255 }).notNull(), + }, + (table) => [ + primaryKey({ columns: [table.orgId, table.source, table.sourceId] }), + ], +); diff --git a/packages/env/index.ts b/packages/env/index.ts index 51282f4fb..815f62a2e 100644 --- a/packages/env/index.ts +++ b/packages/env/index.ts @@ -1,2 +1,2 @@ -export { buildEnv, NODE_ENV } from "./build"; -export { serverEnv } from "./server"; +export { buildEnv, NODE_ENV } from "./build.ts"; +export { serverEnv } from "./server.ts"; diff --git a/packages/env/package.json b/packages/env/package.json index 7fe3a31fa..67a2d7089 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -1,6 +1,7 @@ { "name": "@cap/env", "private": true, + "type": "module", "main": "./index.ts", "types": "./index.ts", "dependencies": { diff --git a/packages/env/server.ts b/packages/env/server.ts index 678825bc5..32f671735 100644 --- a/packages/env/server.ts +++ b/packages/env/server.ts @@ -65,6 +65,8 @@ function createServerEnv() { CLOUDFRONT_KEYPAIR_PRIVATE_KEY: z.string().optional(), S3_PUBLIC_ENDPOINT: z.string().optional(), S3_INTERNAL_ENDPOINT: z.string().optional(), + REMOTE_WORKFLOW_URL: z.string().optional(), + REMOTE_WORKFLOW_SECRET: z.string().optional(), }, experimental__runtimeEnv: { ...process.env, diff --git a/packages/utils/package.json b/packages/utils/package.json index fbd9dcb12..082b364a0 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,10 +1,10 @@ { "name": "@cap/utils", - "version": "0.0.0", - "main": "./src/index.tsx", - "types": "./src/index.tsx", + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", "exports": { - ".": "./src/index.tsx" + ".": "./src/index.ts" }, "scripts": { "typecheck": "tsc -b" diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts new file mode 100644 index 000000000..72c80b23d --- /dev/null +++ b/packages/utils/src/index.ts @@ -0,0 +1,5 @@ +export * from "./constants/plans.ts"; +export * from "./constants/s3.ts"; +export * from "./helpers.ts"; +export * from "./lib/stripe/stripe.ts"; +export * from "./types/database.ts"; diff --git a/packages/utils/src/index.tsx b/packages/utils/src/index.tsx deleted file mode 100644 index 5e82387dc..000000000 --- a/packages/utils/src/index.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export * from "./constants/plans"; -export * from "./constants/s3"; -export * from "./helpers"; -export * from "./lib/stripe/stripe"; -export * from "./types/database"; diff --git a/packages/web-backend/src/Auth.ts b/packages/web-backend/src/Auth.ts index 757776c45..46e0b39ba 100644 --- a/packages/web-backend/src/Auth.ts +++ b/packages/web-backend/src/Auth.ts @@ -1,11 +1,11 @@ import { getServerSession } from "@cap/database/auth/auth-options"; import * as Db from "@cap/database/schema"; import { CurrentUser, HttpAuthMiddleware } from "@cap/web-domain"; -import { HttpApiError, type HttpApp } from "@effect/platform"; +import { HttpApiError, HttpServerRequest } from "@effect/platform"; import * as Dz from "drizzle-orm"; -import { type Cause, Effect, Layer, Option } from "effect"; +import { type Cause, Effect, Layer, Option, Schema } from "effect"; -import { Database, type DatabaseError } from "./Database"; +import { Database, type DatabaseError } from "./Database.ts"; export const getCurrentUser = Effect.gen(function* () { const db = yield* Database; @@ -37,24 +37,47 @@ export const HttpAuthMiddlewareLive = Layer.effect( return HttpAuthMiddleware.of( Effect.gen(function* () { - const user = yield* getCurrentUser.pipe( - Effect.flatten, + const headers = yield* HttpServerRequest.schemaHeaders( + Schema.Struct({ authorization: Schema.optional(Schema.String) }), + ); + const authHeader = headers.authorization?.split(" ")[1]; + + let user; + + if (authHeader?.length === 36) { + user = yield* database + .execute((db) => + db + .select() + .from(Db.users) + .leftJoin( + Db.authApiKeys, + Dz.eq(Db.users.id, Db.authApiKeys.userId), + ) + .where(Dz.eq(Db.authApiKeys.id, authHeader)), + ) + .pipe(Effect.map(([entry]) => Option.fromNullable(entry?.users))); + } else { + user = yield* getCurrentUser; + } + + return yield* user.pipe( + Option.map((user) => ({ + id: user.id, + email: user.email, + activeOrgId: user.activeOrganizationId, + })), Effect.catchTag( "NoSuchElementException", () => new HttpApiError.Unauthorized(), ), ); - - return { - id: user.id, - email: user.email, - activeOrgId: user.activeOrganizationId, - }; }).pipe( Effect.provideService(Database, database), Effect.catchTags({ UnknownException: () => new HttpApiError.InternalServerError(), DatabaseError: () => new HttpApiError.InternalServerError(), + ParseError: () => new HttpApiError.BadRequest(), }), ), ); diff --git a/packages/web-backend/src/Database.ts b/packages/web-backend/src/Database.ts index 1bc6a516b..16e6eb01c 100644 --- a/packages/web-backend/src/Database.ts +++ b/packages/web-backend/src/Database.ts @@ -1,15 +1,19 @@ -import type { db } from "@cap/database"; -import { Context, Data, type Effect } from "effect"; +import { db } from "@cap/database"; +import { Effect, Schema } from "effect"; -export class Database extends Context.Tag("Database")< - Database, - { - execute( - callback: (_: ReturnType) => Promise, - ): Effect.Effect; - } ->() {} +export class DatabaseError extends Schema.TaggedError()( + "DatabaseError", + { cause: Schema.Unknown }, +) {} -export class DatabaseError extends Data.TaggedError("DatabaseError")<{ - message: string; -}> {} +export class Database extends Effect.Service()("Database", { + effect: Effect.gen(function* () { + return { + execute: (cb: (_: ReturnType) => Promise) => + Effect.tryPromise({ + try: () => cb(db()), + catch: (cause) => new DatabaseError({ cause }), + }), + }; + }), +}) {} diff --git a/packages/web-backend/src/Folders/FoldersPolicy.ts b/packages/web-backend/src/Folders/FoldersPolicy.ts index 7b0f0e47b..42b677ec6 100644 --- a/packages/web-backend/src/Folders/FoldersPolicy.ts +++ b/packages/web-backend/src/Folders/FoldersPolicy.ts @@ -2,7 +2,8 @@ import * as Db from "@cap/database/schema"; import { type Folder, Policy } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Effect } from "effect"; -import { Database } from "../Database"; + +import { Database } from "../Database.ts"; export class FoldersPolicy extends Effect.Service()( "FoldersPolicy", @@ -41,5 +42,6 @@ export class FoldersPolicy extends Effect.Service()( return { canEdit }; }), + dependencies: [Database.Default], }, ) {} diff --git a/packages/web-backend/src/Folders/FoldersRpcs.ts b/packages/web-backend/src/Folders/FoldersRpcs.ts index afc1a72be..078d6f10f 100644 --- a/packages/web-backend/src/Folders/FoldersRpcs.ts +++ b/packages/web-backend/src/Folders/FoldersRpcs.ts @@ -1,7 +1,7 @@ import { Folder, InternalError } from "@cap/web-domain"; import { Effect } from "effect"; -import { Folders } from "."; +import { Folders } from "./index.ts"; export const FolderRpcsLive = Folder.FolderRpcs.toLayer( Effect.gen(function* () { diff --git a/packages/web-backend/src/Folders/index.ts b/packages/web-backend/src/Folders/index.ts index b620072a5..bf4b2c10f 100644 --- a/packages/web-backend/src/Folders/index.ts +++ b/packages/web-backend/src/Folders/index.ts @@ -3,9 +3,10 @@ import * as Db from "@cap/database/schema"; import { CurrentUser, Folder, Policy } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Effect, Option } from "effect"; -import { Database, type DatabaseError } from "../Database"; -import { FoldersPolicy } from "./FoldersPolicy"; +import { Database, type DatabaseError } from "../Database.ts"; +import { FoldersPolicy } from "./FoldersPolicy.ts"; +// @effect-diagnostics-next-line leakingRequirements:off export class Folders extends Effect.Service()("Folders", { effect: Effect.gen(function* () { const db = yield* Database; @@ -123,5 +124,5 @@ export class Folders extends Effect.Service()("Folders", { }), }; }), - dependencies: [FoldersPolicy.Default], + dependencies: [FoldersPolicy.Default, Database.Default], }) {} diff --git a/packages/web-backend/src/Loom/ImportVideo.ts b/packages/web-backend/src/Loom/ImportVideo.ts index 105d444ab..0d76cc6b3 100644 --- a/packages/web-backend/src/Loom/ImportVideo.ts +++ b/packages/web-backend/src/Loom/ImportVideo.ts @@ -1,61 +1,162 @@ -import { Readable } from "node:stream"; -import * as Db from "@cap/database/schema"; -import { CurrentUser, Loom, S3Bucket } from "@cap/web-domain"; -import { ClusterWorkflowEngine } from "@effect/cluster"; -import { Headers, HttpClient, HttpServerResponse } from "@effect/platform"; -import { Activity, Workflow, WorkflowProxy } from "@effect/workflow"; -import { Effect, Layer, Option, Schema, Stream } from "effect"; - -import { Database, DatabaseError } from "../Database"; -import { S3Buckets } from "../S3Buckets"; -import { S3BucketAccess } from "../S3Buckets/S3BucketAccess"; - -export const LoomImportVideoLive = Loom.ImportVideo.toLayer( +import { S3Bucket, Video } from "@cap/web-domain"; +import { Headers, HttpClient } from "@effect/platform"; +import { Activity, Workflow } from "@effect/workflow"; +import { Effect, Option, Schedule, Schema, Stream } from "effect"; + +import { DatabaseError } from "../Database.ts"; +import { S3Buckets } from "../S3Buckets/index.ts"; +import { S3BucketAccess, S3Error } from "../S3Buckets/S3BucketAccess.ts"; +import { Videos } from "../Videos/index.ts"; + +class LoomApiError extends Schema.TaggedError("LoomApiError")( + "LoomApiError", + { cause: Schema.Unknown }, +) {} + +const LoomImportVideoError = Schema.Union( + DatabaseError, + Video.NotFoundError, + S3Error, + LoomApiError, +); + +export const LoomImportVideo = Workflow.make({ + name: "LoomImportVideo", + payload: { + cap: Schema.Struct({ + userId: Schema.String, + orgId: Schema.String, + }), + loom: Schema.Struct({ + userId: Schema.String, + orgId: Schema.String, + video: Schema.Struct({ + id: Schema.String, + name: Schema.String, + downloadUrl: Schema.String, + width: Schema.OptionFromNullOr(Schema.Number), + height: Schema.OptionFromNullOr(Schema.Number), + fps: Schema.OptionFromNullOr(Schema.Number), + durationSecs: Schema.OptionFromNullOr(Schema.Number), + }), + }), + attempt: Schema.optional(Schema.Number), + }, + error: LoomImportVideoError, + idempotencyKey: (p) => + `${p.cap.userId}-${p.loom.orgId}-${p.loom.video.id}-${p.attempt ?? 0}`, +}); + +export const LoomImportVideoLive = LoomImportVideo.toLayer( Effect.fn(function* (payload) { + const videos = yield* Videos; const s3Buckets = yield* S3Buckets; const http = yield* HttpClient.HttpClient; - yield* Activity.make({ + const { videoId, customBucketId } = yield* Activity.make({ name: "CreateVideoRecord", + error: LoomImportVideoError, + success: Schema.Struct({ + videoId: Video.VideoId, + customBucketId: Schema.Option(S3Bucket.S3BucketId), + }), execute: Effect.gen(function* () { - // db.execute((db) => db.insert(Db.videos).values([])).pipe( - // Effect.catchAll(() => Effect.die(undefined)), - // ); + const loomVideo = payload.loom.video; + + const [_, customBucket] = yield* s3Buckets + .getProviderForUser(payload.cap.userId) + .pipe(Effect.catchAll(() => Effect.die(null))); + + const customBucketId = Option.map(customBucket, (b) => b.id); + + const videoId = yield* videos.create({ + ownerId: payload.cap.userId, + orgId: payload.cap.orgId, + bucketId: customBucketId, + source: { type: "desktopMP4" as const }, + name: payload.loom.video.name, + duration: loomVideo.durationSecs, + width: loomVideo.width, + height: loomVideo.height, + public: true, + metadata: Option.none(), + folderId: Option.none(), + transcriptionStatus: Option.none(), + importSource: new Video.ImportSource({ + source: "loom", + id: loomVideo.id, + }), + }); + + return { videoId, customBucketId }; }), }); - const [bucketProvider, customBucket] = yield* s3Buckets.getProviderForUser( - payload.userId, - ); + const source = new Video.Mp4Source({ + videoId: videoId, + ownerId: payload.cap.userId, + }); - yield* Activity.make({ + const { fileKey } = yield* Activity.make({ name: "DownloadVideo", + error: LoomImportVideoError, + success: Schema.Struct({ fileKey: Schema.String }), execute: Effect.gen(function* () { - const s3Bucket = yield* S3BucketAccess; + const [bucketProvider] = + yield* s3Buckets.getProviderForBucket(customBucketId); + + return yield* Effect.gen(function* () { + const s3Bucket = yield* S3BucketAccess; + + const resp = yield* http + .get(payload.loom.video.downloadUrl) + .pipe(Effect.catchAll((cause) => new LoomApiError({ cause }))); + const contentLength = Headers.get( + resp.headers, + "content-length", + ).pipe( + Option.map((v) => Number(v)), + Option.getOrUndefined, + ); + yield* Effect.log(`Downloading ${contentLength} bytes`); + + let downloadedBytes = 0; - const key = `/loom/${payload.loomOrgId}/${payload.loomVideoId}`; + const key = source.getFileKey(); - const resp = yield* http.get(payload.downloadUrl); - yield* s3Bucket - .putObject( - key, - resp.stream.pipe( - Stream.tap((buffer) => - Effect.log(`Downloaded ${buffer.length} bytes`), + yield* s3Bucket + .putObject( + key, + resp.stream.pipe( + Stream.tap((bytes) => { + downloadedBytes += bytes.length; + return Effect.void; + }), ), - Stream.toReadableStreamRuntime(yield* Effect.runtime()), - (s) => Readable.fromWeb(s as any), - ), - { - contentLength: Headers.get(resp.headers, "content-length").pipe( - Option.map((v) => Number(v)), - Option.getOrUndefined, + { contentLength }, + ) + .pipe( + Effect.race( + // TODO: Connect this with upload progress + Effect.repeat( + Effect.gen(function* () { + const bytes = yield* Effect.succeed(downloadedBytes); + yield* Effect.log(`Downloaded ${bytes} bytes`); + }), + Schedule.forever.pipe(Schedule.delayed(() => "2 seconds")), + ).pipe(Effect.delay("100 millis")), ), - }, + ); + + yield* Effect.log( + `Uploaded video for user '${payload.cap.userId}' at key '${key}'`, ); - yield* Effect.log(`Uploaded video for user '${payload.userId}' at key '${key}'`); - }).pipe(Effect.provide(bucketProvider)), + return { fileKey: key }; + }).pipe(Effect.provide(bucketProvider)); + }), }); + + return { fileKey, videoId }; }), ); diff --git a/packages/web-backend/src/Loom/index.ts b/packages/web-backend/src/Loom/index.ts index e5a1ab03d..e9362c9c9 100644 --- a/packages/web-backend/src/Loom/index.ts +++ b/packages/web-backend/src/Loom/index.ts @@ -1 +1 @@ -export * from "./ImportVideo"; +export * from "./ImportVideo.ts"; diff --git a/packages/web-backend/src/Organisations/OrganisationsRepo.ts b/packages/web-backend/src/Organisations/OrganisationsRepo.ts index 314b46d3a..73a8c02fe 100644 --- a/packages/web-backend/src/Organisations/OrganisationsRepo.ts +++ b/packages/web-backend/src/Organisations/OrganisationsRepo.ts @@ -3,7 +3,7 @@ import type { Video } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Effect } from "effect"; -import { Database } from "../Database"; +import { Database } from "../Database.ts"; export class OrganisationsRepo extends Effect.Service()( "OrganisationsRepo", @@ -33,5 +33,6 @@ export class OrganisationsRepo extends Effect.Service()( ), }; }), + dependencies: [Database.Default], }, ) {} diff --git a/packages/web-backend/src/Rpcs.ts b/packages/web-backend/src/Rpcs.ts index 54ded2158..0cced04ca 100644 --- a/packages/web-backend/src/Rpcs.ts +++ b/packages/web-backend/src/Rpcs.ts @@ -5,10 +5,10 @@ import { } from "@cap/web-domain"; import { Effect, Layer, Option } from "effect"; -import { getCurrentUser } from "./Auth"; -import { Database } from "./Database"; -import { FolderRpcsLive } from "./Folders/FoldersRpcs"; -import { VideosRpcsLive } from "./Videos/VideosRpcs"; +import { getCurrentUser } from "./Auth.ts"; +import { Database } from "./Database.ts"; +import { FolderRpcsLive } from "./Folders/FoldersRpcs.ts"; +import { VideosRpcsLive } from "./Videos/VideosRpcs.ts"; export const RpcsLive = Layer.mergeAll(VideosRpcsLive, FolderRpcsLive); diff --git a/packages/web-backend/src/S3Buckets/S3BucketAccess.ts b/packages/web-backend/src/S3Buckets/S3BucketAccess.ts index 49af73f10..4254a7b2e 100644 --- a/packages/web-backend/src/S3Buckets/S3BucketAccess.ts +++ b/packages/web-backend/src/S3Buckets/S3BucketAccess.ts @@ -1,23 +1,24 @@ +import { Readable } from "node:stream"; import * as S3 from "@aws-sdk/client-s3"; import { createPresignedPost, type PresignedPostOptions, } from "@aws-sdk/s3-presigned-post"; import * as S3Presigner from "@aws-sdk/s3-request-presigner"; -import type { - RequestPresigningArguments, - StreamingBlobPayloadInputTypes, -} from "@smithy/types"; -import { type Cause, Data, Effect, Option } from "effect"; -import { S3BucketClientProvider } from "./S3BucketClientProvider"; +import type { RequestPresigningArguments } from "@smithy/types"; +import { type Cause, Effect, Option, Schema, Stream } from "effect"; -export class S3Error extends Data.TaggedError("S3Error")<{ message: string }> {} +import { S3BucketClientProvider } from "./S3BucketClientProvider.ts"; + +export class S3Error extends Schema.TaggedError()("S3Error", { + cause: Schema.Unknown, +}) {} const wrapS3Promise = ( callback: ( provider: S3BucketClientProvider["Type"], ) => Promise | Effect.Effect, Cause.UnknownException>, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const provider = yield* S3BucketClientProvider; @@ -26,7 +27,7 @@ const wrapS3Promise = ( if (cbResult instanceof Promise) { return yield* Effect.tryPromise({ try: () => cbResult, - catch: (e) => new S3Error({ message: String(e) }), + catch: (cause) => new S3Error({ cause }), }); } @@ -34,11 +35,13 @@ const wrapS3Promise = ( Effect.flatMap((cbResult) => Effect.tryPromise({ try: () => cbResult, - catch: (e) => new S3Error({ message: String(e) }), + catch: (cause) => new S3Error({ cause }), }), ), ); - }); + }).pipe( + Effect.catchTag("UnknownException", (cause) => new S3Error({ cause })), + ); // @effect-diagnostics-next-line leakingRequirements:off export class S3BucketAccess extends Effect.Service()( @@ -105,23 +108,38 @@ export class S3BucketAccess extends Effect.Service()( ), ), ), - putObject: ( + putObject: ( key: string, - body: StreamingBlobPayloadInputTypes, + body: string | Uint8Array | ArrayBuffer | Stream.Stream, fields?: { contentType?: string; contentLength?: number }, ) => wrapS3Promise((provider) => provider.getInternal.pipe( - Effect.map((client) => - client.send( - new S3.PutObjectCommand({ - Bucket: provider.bucket, - Key: key, - Body: body, - ContentType: fields?.contentType, - ContentLength: fields?.contentLength, - }), - ), + Effect.flatMap((client) => + Effect.gen(function* () { + let _body; + + if (typeof body === "string" || body instanceof Uint8Array) { + _body = body; + } else if (body instanceof ArrayBuffer) { + _body = new Uint8Array(body); + } else { + _body = body.pipe( + Stream.toReadableStreamRuntime(yield* Effect.runtime()), + (s) => Readable.fromWeb(s as any), + ); + } + + return client.send( + new S3.PutObjectCommand({ + Bucket: provider.bucket, + Key: key, + Body: _body, + ContentType: fields?.contentType, + ContentLength: fields?.contentLength, + }), + ); + }), ), ), ).pipe( diff --git a/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts b/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts index eb551f7cc..3a6fb3365 100644 --- a/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts +++ b/packages/web-backend/src/S3Buckets/S3BucketsRepo.ts @@ -3,7 +3,7 @@ import { S3Bucket, type Video } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Effect, Option } from "effect"; -import { Database } from "../Database"; +import { Database } from "../Database.ts"; export class S3BucketsRepo extends Effect.Service()( "S3BucketsRepo", @@ -68,5 +68,6 @@ export class S3BucketsRepo extends Effect.Service()( return { getForVideo, getById, getForUser }; }), + dependencies: [Database.Default], }, ) {} diff --git a/packages/web-backend/src/S3Buckets/index.ts b/packages/web-backend/src/S3Buckets/index.ts index 24d4b2d29..1cbf14085 100644 --- a/packages/web-backend/src/S3Buckets/index.ts +++ b/packages/web-backend/src/S3Buckets/index.ts @@ -4,9 +4,11 @@ import { decrypt } from "@cap/database/crypto"; import { S3_BUCKET_URL } from "@cap/utils"; import type { S3Bucket } from "@cap/web-domain"; import { Config, Context, Effect, Layer, Option } from "effect"; -import { S3BucketAccess } from "./S3BucketAccess"; -import { S3BucketClientProvider } from "./S3BucketClientProvider"; -import { S3BucketsRepo } from "./S3BucketsRepo"; + +import { Database } from "../Database.ts"; +import { S3BucketAccess } from "./S3BucketAccess.ts"; +import { S3BucketClientProvider } from "./S3BucketClientProvider.ts"; +import { S3BucketsRepo } from "./S3BucketsRepo.ts"; export class S3Buckets extends Effect.Service()("S3Buckets", { effect: Effect.gen(function* () { @@ -171,7 +173,7 @@ export class S3Buckets extends Effect.Service()("S3Buckets", { }); return { - getProviderById: Effect.fn("S3Buckets.getProviderById")(function* ( + getProviderForBucket: Effect.fn("S3Buckets.getProviderById")(function* ( bucketId: Option.Option, ) { const customBucket = yield* bucketId.pipe( @@ -193,5 +195,5 @@ export class S3Buckets extends Effect.Service()("S3Buckets", { }), }; }), - dependencies: [S3BucketsRepo.Default], + dependencies: [S3BucketsRepo.Default, Database.Default], }) {} diff --git a/packages/web-backend/src/Spaces/SpacesRepo.ts b/packages/web-backend/src/Spaces/SpacesRepo.ts index 0797026ac..bb77e42f5 100644 --- a/packages/web-backend/src/Spaces/SpacesRepo.ts +++ b/packages/web-backend/src/Spaces/SpacesRepo.ts @@ -3,7 +3,7 @@ import type { Video } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Effect } from "effect"; -import { Database } from "../Database"; +import { Database } from "../Database.ts"; export class SpacesRepo extends Effect.Service()("SpacesRepo", { effect: Effect.gen(function* () { @@ -28,4 +28,5 @@ export class SpacesRepo extends Effect.Service()("SpacesRepo", { ), }; }), + dependencies: [Database.Default], }) {} diff --git a/packages/web-backend/src/Videos/VideosPolicy.ts b/packages/web-backend/src/Videos/VideosPolicy.ts index c6cb0ea30..9e3aa6f6f 100644 --- a/packages/web-backend/src/Videos/VideosPolicy.ts +++ b/packages/web-backend/src/Videos/VideosPolicy.ts @@ -1,9 +1,10 @@ import { Policy, Video } from "@cap/web-domain"; import { Array, Effect, Option } from "effect"; -import { OrganisationsRepo } from "../Organisations/OrganisationsRepo"; -import { SpacesRepo } from "../Spaces/SpacesRepo"; -import { VideosRepo } from "./VideosRepo"; +import { Database } from "../Database.ts"; +import { OrganisationsRepo } from "../Organisations/OrganisationsRepo.ts"; +import { SpacesRepo } from "../Spaces/SpacesRepo.ts"; +import { VideosRepo } from "./VideosRepo.ts"; export class VideosPolicy extends Effect.Service()( "VideosPolicy", @@ -91,6 +92,7 @@ export class VideosPolicy extends Effect.Service()( VideosRepo.Default, OrganisationsRepo.Default, SpacesRepo.Default, + Database.Default, ], }, ) {} diff --git a/packages/web-backend/src/Videos/VideosRepo.ts b/packages/web-backend/src/Videos/VideosRepo.ts index 2f3b66c56..c7ed57077 100644 --- a/packages/web-backend/src/Videos/VideosRepo.ts +++ b/packages/web-backend/src/Videos/VideosRepo.ts @@ -2,76 +2,96 @@ import { nanoId } from "@cap/database/helpers"; import * as Db from "@cap/database/schema"; import { Video } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; +import type { MySqlInsertBase } from "drizzle-orm/mysql-core"; import { Effect, Option } from "effect"; -import { Database } from "../Database"; +import type { Schema } from "effect/Schema"; +import { Database } from "../Database.ts"; + +export type CreateVideoInput = Omit< + Schema.Type, + "id" | "createdAt" | "updatedAt" | "orgId" +> & { password?: string; importSource?: Video.ImportSource; orgId: string }; export class VideosRepo extends Effect.Service()("VideosRepo", { effect: Effect.gen(function* () { const db = yield* Database; - return { - /** - * Gets a `Video` and its accompanying password if available. - * - * The password is returned separately as the `Video` class is client-safe - */ - getById: (id: Video.VideoId) => - Effect.gen(function* () { - const [video] = yield* db.execute((db) => - db.select().from(Db.videos).where(Dz.eq(Db.videos.id, id)), - ); + /** + * Gets a `Video` and its accompanying password if available. + * + * The password is returned separately as the `Video` class is client-safe + */ + const getById = (id: Video.VideoId) => + Effect.gen(function* () { + const [video] = yield* db.execute((db) => + db.select().from(Db.videos).where(Dz.eq(Db.videos.id, id)), + ); + + return Option.fromNullable(video).pipe( + Option.map( + (v) => + [ + Video.Video.decodeSync({ + ...v, + bucketId: v.bucket, + createdAt: v.createdAt.toISOString(), + updatedAt: v.updatedAt.toISOString(), + metadata: v.metadata as any, + }), + Option.fromNullable(video?.password), + ] as const, + ), + ); + }); + + const delete_ = (id: Video.VideoId) => + db.execute((db) => db.delete(Db.videos).where(Dz.eq(Db.videos.id, id))); - return Option.fromNullable(video).pipe( - Option.map( - (v) => - [ - Video.Video.decodeSync({ - ...v, - bucketId: v.bucket, - createdAt: v.createdAt.toISOString(), - updatedAt: v.updatedAt.toISOString(), - metadata: v.metadata as any, - }), - Option.fromNullable(video?.password), - ] as const, - ), - ); - }), - delete: (id: Video.VideoId) => - db.execute((db) => db.delete(Db.videos).where(Dz.eq(Db.videos.id, id))), - create: ( - data: Pick< - (typeof Video.Video)["Encoded"], - | "ownerId" - | "name" - | "bucketId" - | "metadata" - | "public" - | "transcriptionStatus" - | "source" - | "folderId" - > & { password?: string }, - ) => { - const id = nanoId(); + const create = (data: CreateVideoInput) => + Effect.gen(function* () { + const id = Video.VideoId.make(nanoId()); - return db.execute((db) => - db - .insert(Db.videos) - .values({ - id, - ownerId: data.ownerId, - name: data.name, - bucket: data.bucketId, - metadata: data.metadata, - public: data.public, - transcriptionStatus: data.transcriptionStatus, - source: data.source, - folderId: data.folderId, - password: data.password, - }) - .then(() => Video.VideoId.make(id)), + yield* db.execute((db) => + db.transaction(async (db) => { + const promises: MySqlInsertBase[] = [ + db.insert(Db.videos).values([ + { + ...data, + id, + orgId: Option.getOrNull(data.orgId ?? Option.none()), + bucket: Option.getOrNull(data.bucketId ?? Option.none()), + metadata: Option.getOrNull(data.metadata ?? Option.none()), + transcriptionStatus: Option.getOrNull( + data.transcriptionStatus ?? Option.none(), + ), + folderId: Option.getOrNull(data.folderId ?? Option.none()), + width: Option.getOrNull(data.width ?? Option.none()), + height: Option.getOrNull(data.height ?? Option.none()), + duration: Option.getOrNull(data.duration ?? Option.none()), + }, + ]), + ]; + + if (data.importSource) + promises.push( + db.insert(Db.importedVideos).values([ + { + id, + orgId: data.orgId, + source: data.importSource.source, + sourceId: data.importSource.id, + }, + ]), + ); + + await Promise.all(promises); + }), ); - }, - }; + + return id; + }); + + return { getById, delete: delete_, create }; }), + dependencies: [Database.Default], }) {} diff --git a/packages/web-backend/src/Videos/VideosRpcs.ts b/packages/web-backend/src/Videos/VideosRpcs.ts index c04de7f1d..ba6690c39 100644 --- a/packages/web-backend/src/Videos/VideosRpcs.ts +++ b/packages/web-backend/src/Videos/VideosRpcs.ts @@ -1,7 +1,8 @@ import { InternalError, Video } from "@cap/web-domain"; import { Effect } from "effect"; -import { provideOptionalAuth } from "../Auth"; -import { Videos } from "."; + +import { provideOptionalAuth } from "../Auth.ts"; +import { Videos } from "./index.ts"; export const VideosRpcsLive = Video.VideoRpcs.toLayer( Effect.gen(function* () { @@ -27,7 +28,6 @@ export const VideosRpcsLive = Video.VideoRpcs.toLayer( GetUploadProgress: (videoId) => videos.getUploadProgress(videoId).pipe( provideOptionalAuth, - (v) => v, Effect.catchTags({ DatabaseError: () => new InternalError({ type: "database" }), UnknownException: () => new InternalError({ type: "unknown" }), diff --git a/packages/web-backend/src/Videos/index.ts b/packages/web-backend/src/Videos/index.ts index c9d598909..f26fb5bbf 100644 --- a/packages/web-backend/src/Videos/index.ts +++ b/packages/web-backend/src/Videos/index.ts @@ -3,11 +3,11 @@ import { CurrentUser, Policy, Video } from "@cap/web-domain"; import * as Dz from "drizzle-orm"; import { Array, Effect, Option, pipe } from "effect"; -import { Database } from "../Database"; -import { S3Buckets } from "../S3Buckets"; -import { S3BucketAccess } from "../S3Buckets/S3BucketAccess"; -import { VideosPolicy } from "./VideosPolicy"; -import { VideosRepo } from "./VideosRepo"; +import { Database } from "../Database.ts"; +import { S3Buckets } from "../S3Buckets/index.ts"; +import { S3BucketAccess } from "../S3Buckets/S3BucketAccess.ts"; +import { VideosPolicy } from "./VideosPolicy.ts"; +import { type CreateVideoInput, VideosRepo } from "./VideosRepo.ts"; export class Videos extends Effect.Service()("Videos", { effect: Effect.gen(function* () { @@ -39,7 +39,7 @@ export class Videos extends Effect.Service()("Videos", { Effect.flatMap(Effect.catchAll(() => new Video.NotFoundError())), ); - const [S3ProviderLayer] = yield* s3Buckets.getProviderById( + const [S3ProviderLayer] = yield* s3Buckets.getProviderForBucket( video.bucketId, ); @@ -79,12 +79,12 @@ export class Videos extends Effect.Service()("Videos", { Policy.withPolicy(policy.isOwner(videoId)), ); - const [S3ProviderLayer] = yield* s3Buckets.getProviderById( + const [S3ProviderLayer] = yield* s3Buckets.getProviderForBucket( video.bucketId, ); // Don't duplicate password or sharing data - const newVideoId = yield* repo.create(yield* video.toJS()); + const newVideoId = yield* repo.create(video); yield* Effect.gen(function* () { const s3 = yield* S3BucketAccess; @@ -136,7 +136,14 @@ export class Videos extends Effect.Service()("Videos", { Option.map((r) => new Video.UploadProgress(r)), ); }), + + create: Effect.fn("Videos.create")(repo.create), }; }), - dependencies: [VideosPolicy.Default, VideosRepo.Default, S3Buckets.Default], + dependencies: [ + VideosPolicy.Default, + VideosRepo.Default, + Database.Default, + S3Buckets.Default, + ], }) {} diff --git a/packages/web-backend/src/Workflows.ts b/packages/web-backend/src/Workflows.ts index 602903474..f06b8d9af 100644 --- a/packages/web-backend/src/Workflows.ts +++ b/packages/web-backend/src/Workflows.ts @@ -1,5 +1,11 @@ +import { Workflows } from "@cap/web-domain"; +import { WorkflowProxyServer } from "@effect/workflow"; import { Layer } from "effect"; -import { LoomImportVideoLive } from "./Loom"; +import { LoomImportVideoLive } from "./Loom/index.ts"; export const WorkflowsLayer = Layer.mergeAll(LoomImportVideoLive); + +export const WorkflowsRpcLayer = WorkflowProxyServer.layerRpcHandlers( + Workflows.Workflows, +); diff --git a/packages/web-backend/src/index.ts b/packages/web-backend/src/index.ts index fb0714052..f48728378 100644 --- a/packages/web-backend/src/index.ts +++ b/packages/web-backend/src/index.ts @@ -1,12 +1,10 @@ -import "server-only"; - -export * from "./Auth"; -export * from "./Database"; -export { Folders } from "./Folders"; -export * from "./Loom"; -export * from "./Rpcs"; -export { S3Buckets } from "./S3Buckets"; -export { S3BucketAccess } from "./S3Buckets/S3BucketAccess"; -export { Videos } from "./Videos"; -export { VideosPolicy } from "./Videos/VideosPolicy"; -export * from "./Workflows"; +export * from "./Auth.ts"; +export * from "./Database.ts"; +export { Folders } from "./Folders/index.ts"; +export * from "./Loom/index.ts"; +export * from "./Rpcs.ts"; +export { S3Buckets } from "./S3Buckets/index.ts"; +export { S3BucketAccess } from "./S3Buckets/S3BucketAccess.ts"; +export { Videos } from "./Videos/index.ts"; +export { VideosPolicy } from "./Videos/VideosPolicy.ts"; +export * from "./Workflows.ts"; diff --git a/packages/web-domain/src/Authentication.ts b/packages/web-domain/src/Authentication.ts index db92b874a..b4ef466d3 100644 --- a/packages/web-domain/src/Authentication.ts +++ b/packages/web-domain/src/Authentication.ts @@ -2,7 +2,7 @@ import { HttpApiError, HttpApiMiddleware } from "@effect/platform"; import { RpcMiddleware } from "@effect/rpc"; import { Context, Schema } from "effect"; -import { InternalError } from "./Errors"; +import { InternalError } from "./Errors.ts"; export class CurrentUser extends Context.Tag("CurrentUser")< CurrentUser, @@ -16,6 +16,7 @@ export class HttpAuthMiddleware extends HttpApiMiddleware.Tag `${p.loomOrgId}-${p.loomVideoId}`, -}); diff --git a/packages/web-domain/src/Loom/index.ts b/packages/web-domain/src/Loom/index.ts deleted file mode 100644 index e5a1ab03d..000000000 --- a/packages/web-domain/src/Loom/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./ImportVideo"; diff --git a/packages/web-domain/src/Policy.ts b/packages/web-domain/src/Policy.ts index 750e0d751..d159eafd1 100644 --- a/packages/web-domain/src/Policy.ts +++ b/packages/web-domain/src/Policy.ts @@ -1,7 +1,8 @@ // shoutout https://lucas-barake.github.io/building-a-composable-policy-system/ import { Context, Data, Effect, type Option, Schema } from "effect"; -import { CurrentUser } from "./Authentication"; + +import { CurrentUser } from "./Authentication.ts"; export type Policy = Effect.Effect< void, diff --git a/packages/web-domain/src/Rpcs.ts b/packages/web-domain/src/Rpcs.ts index 64bcb65b6..6cb50ece2 100644 --- a/packages/web-domain/src/Rpcs.ts +++ b/packages/web-domain/src/Rpcs.ts @@ -1,6 +1,6 @@ import { RpcGroup } from "@effect/rpc"; -import { FolderRpcs } from "./Folder"; -import { VideoRpcs } from "./Video"; +import { FolderRpcs } from "./Folder.ts"; +import { VideoRpcs } from "./Video.ts"; export const Rpcs = RpcGroup.make().merge(VideoRpcs, FolderRpcs); diff --git a/packages/web-domain/src/S3Bucket.ts b/packages/web-domain/src/S3Bucket.ts index e332f349b..8f6c0e1d6 100644 --- a/packages/web-domain/src/S3Bucket.ts +++ b/packages/web-domain/src/S3Bucket.ts @@ -4,7 +4,7 @@ export const S3BucketId = Schema.String.pipe(Schema.brand("S3BucketId")); export type S3BucketId = typeof S3BucketId.Type; export class S3Bucket extends Schema.Class("S3Bucket")({ - id: Schema.String, + id: S3BucketId, ownerId: Schema.String, region: Schema.String, endpoint: Schema.OptionFromNullOr(Schema.String), diff --git a/packages/web-domain/src/Video.ts b/packages/web-domain/src/Video.ts index b04317fe0..3c0f68845 100644 --- a/packages/web-domain/src/Video.ts +++ b/packages/web-domain/src/Video.ts @@ -1,10 +1,11 @@ import { Rpc, RpcGroup } from "@effect/rpc"; import { Context, Effect, Option, Schema } from "effect"; -import { RpcAuthMiddleware } from "./Authentication"; -import { InternalError } from "./Errors"; -import { FolderId } from "./Folder"; -import { PolicyDeniedError } from "./Policy"; -import { S3BucketId } from "./S3Bucket"; + +import { RpcAuthMiddleware } from "./Authentication.ts"; +import { InternalError } from "./Errors.ts"; +import { FolderId } from "./Folder.ts"; +import { PolicyDeniedError } from "./Policy.ts"; +import { S3BucketId } from "./S3Bucket.ts"; export const VideoId = Schema.String.pipe(Schema.brand("VideoId")); export type VideoId = typeof VideoId.Type; @@ -13,25 +14,48 @@ export type VideoId = typeof VideoId.Type; export class Video extends Schema.Class