feat: git registries

src/build/mod.ts

@@ -23,6 +23,7 @@ import { Module, Script } from "../project/mod.ts";
 import { shutdownAllPools } from "../utils/worker_pool.ts";
 import { migrateDev } from "../migrate/dev.ts";
 import { compileModuleTypeHelper } from "./gen.ts";
+import { migrateDeploy } from "../migrate/deploy.ts";
 
 /**
  * Which format to use for building.
@@ -237,18 +238,26 @@ async function buildSteps(
 	project: Project,
 	opts: BuildOpts,
 ) {
-	const modules = Array.from(project.modules.values());
-
-	buildStep(buildState, {
-		name: "Prisma schema",
-		files: modules.map((module) => join(module.path, "db", "schema.prisma")),
-		async build() {
-			await migrateDev(project, modules, {
-				createOnly: false,
+	// TODO: This does not reuse the Prisma dir or the database connection, so is extra slow on the first run. Figure out how to make this use one `migrateDev` command and pass in any modified modules For now, these need to be migrated individually because `prisma migrate dev` is an interactive command. Also, making a database change and waiting for all other databases to re-apply will take a long tim.e
+	for (const module of project.modules.values()) {
+		if (module.db) {
+			buildStep(buildState, {
+				name: `Migrate (${module.name})`,
+				module,
+				files: [join(module.path, "db", "schema.prisma")],
+				async build() {
+					if ('directory' in module.registry) {
+						// Update migrations
+						await migrateDev(project, [module], { createOnly: false });
+					} else {
+						// Do not alter migrations, only deploy them
+						await migrateDeploy(project, [module]);
+					}
+				},
 			});
-		},
-	});
-	
+		}
+	}
+
 	buildStep(buildState, {
 		name: "Inflate runtime",
 		// TODO: Add way to compare runtime version

src/cli/commands/db.ts

@@ -23,7 +23,8 @@ dbCommand.command("reset").action(async (opts) => {
 });
 
 dbCommand.command("deploy").action(async (opts) => {
-	await migrateDeploy(await initProject(opts));
+	const project = await initProject(opts);
+	await migrateDeploy(project, [...project.modules.values()]);
 });
 
 // TODO: https://github.com/rivet-gg/open-game-services-engine/issues/84

src/config/project.ts

@@ -7,17 +7,13 @@ export interface ProjectConfig extends Record<string, unknown> {
 	modules: { [name: string]: ProjectModuleConfig };
 }
 
-export interface RegistryConfig extends Record<string, unknown> {
-	directory?: string;
-	// git?: string;
-	// branch?: string;
-	// rev?: string;
+export type RegistryConfig = { local: RegistryConfigLocal } | { git: RegistryConfigGit };
+
+export interface RegistryConfigLocal {
+	directory: string;
 }
 
-// export interface RegistryConfig extends Record<string, unknown> {
-// 	name: string;
-// 	url: string;
-// }
+export type RegistryConfigGit = { url: string, directory?: string } & ({ branch: string } | { tag: string } | { rev: string });
 
 export interface ProjectModuleConfig extends Record<string, unknown> {
 	/**

src/migrate/deploy.ts

@@ -1,10 +1,10 @@
 // Wrapper around `prisma migrate deploy`
 
-import { Project } from "../project/mod.ts";
+import { Module, Project } from "../project/mod.ts";
 import { forEachPrismaSchema } from "./mod.ts";
 
-export async function migrateDeploy(project: Project) {
-	await forEachPrismaSchema(project, [...project.modules.values()], async ({ databaseUrl, tempDir }) => {
+export async function migrateDeploy(project: Project, modules: Module[]) {
+	await forEachPrismaSchema(project, modules, async ({ databaseUrl, tempDir }) => {
 		// Generate migrations & client
 		const status = await new Deno.Command("deno", {
 			args: ["run", "-A", "npm:prisma@5.9.1", "migrate", "deploy"],

src/project/deps.ts

@@ -1,2 +1,3 @@
 export * as glob from "npm:glob@^10.3.10";
-export * as tjs from "npm:typescript-json-schema@^0.62.0";
+export * as tjs from "npm:typescript-json-schema@^0.62.0";
+export { bold, brightRed } from "https://deno.land/std@0.208.0/fmt/colors.ts";

src/project/module.ts

@@ -4,11 +4,13 @@ import { readConfig as readModuleConfig } from "../config/module.ts";
 import { ModuleConfig } from "../config/module.ts";
 import { Script } from "./script.ts";
 import { Project } from "./project.ts";
+import { Registry } from "./registry.ts";
 
 export interface Module {
 	path: string;
 	name: string;
 	config: ModuleConfig;
+	registry: Registry,
 	scripts: Map<string, Script>;
 	db?: ModuleDatabase;
 }
@@ -20,6 +22,7 @@ export interface ModuleDatabase {
 export async function loadModule(
 	modulePath: string,
 	name: string,
+	registry: Registry,
 ): Promise<Module> {
 	// Read config
 	const config = await readModuleConfig(modulePath);
@@ -79,6 +82,7 @@ export async function loadModule(
 		path: modulePath,
 		name,
 		config,
+		registry,
 		scripts,
 		db,
 	};

src/project/project.ts

@@ -1,14 +1,15 @@
-import { exists, join } from "../deps.ts";
+import { dirname, exists, join, copy } from "../deps.ts";
+import { bold, brightRed } from "./deps.ts";
 import { readConfig as readProjectConfig } from "../config/project.ts";
 import { ProjectConfig } from "../config/project.ts";
 import { loadModule, Module } from "./module.ts";
-import { RegistryConfig } from "../config/project.ts";
+import { loadRegistry, Registry } from "./registry.ts";
 import { ProjectModuleConfig } from "../config/project.ts";
-import { bold, brightRed } from "https://deno.land/std@0.208.0/fmt/colors.ts";
 
 export interface Project {
 	path: string;
-	projectConfig: ProjectConfig;
+	config: ProjectConfig;
+	registries: Map<string, Registry>;
 	modules: Map<string, Module>;
 }
 
@@ -19,26 +20,36 @@ export interface LoadProjectOpts {
 export async function loadProject(opts: LoadProjectOpts): Promise<Project> {
 	const projectRoot = join(Deno.cwd(), opts.path ?? ".");
 
-	// console.log("Loading project", projectRoot);
-
 	// Read project config
 	const projectConfig = await readProjectConfig(
 		projectRoot,
 	);
 
+	// Load registries
+	const registries = new Map();
+	for (const registryName in projectConfig.registries) {
+		const registryConfig = projectConfig.registries[registryName];
+		const registry = await loadRegistry(
+			projectRoot,
+			registryName,
+			registryConfig,
+		);
+		registries.set(registryName, registry);
+	}
+
 	// Load modules
 	const modules = new Map<string, Module>();
 	for (const projectModuleName in projectConfig.modules) {
-		const modulePath = await fetchAndResolveModule(
+		const { path, registry } = await fetchAndResolveModule(
 			projectRoot,
 			projectConfig,
+			registries,
 			projectModuleName,
 		);
-		const module = await loadModule(modulePath, projectModuleName);
+		const module = await loadModule(path, projectModuleName, registry);
 		modules.set(projectModuleName, module);
 	}
 
-
 	// Verify existince of module dependencies
 	const missingDepsByModule = new Map<string, string[]>();
 	for (const module of modules.values()) {
@@ -81,23 +92,7 @@ export async function loadProject(opts: LoadProjectOpts): Promise<Project> {
 
 
 
-	return { path: projectRoot, projectConfig, modules };
-}
-
-/**
- * Clones a registry to the local machine if required and returns the path.
- */
-async function fetchAndResolveRegistry(
-	projectRoot: string,
-	registry: RegistryConfig,
-): Promise<string> {
-	// TODO: Impl git cloning
-	if (!registry.directory) throw new Error("Registry directory not set");
-	const registryPath = join(projectRoot, registry.directory);
-	if (!await exists(registryPath)) {
-		throw new Error(`Registry not found at ${registryPath}`);
-	}
-	return registryPath;
+	return { path: projectRoot, config: projectConfig, registries, modules };
 }
 
 /**
@@ -106,29 +101,24 @@ async function fetchAndResolveRegistry(
 async function fetchAndResolveModule(
 	projectRoot: string,
 	projectConfig: ProjectConfig,
+	registries: Map<string, Registry>,
 	moduleName: string,
-): Promise<string> {
+): Promise<{ path: string; registry: Registry }> {
 	// Lookup module
 	const module = projectConfig.modules[moduleName];
 	if (!module) throw new Error(`Module not found ${moduleName}`);
 
 	// Lookup registry
-	const registryName = registryForModule(module);
-	const registryConfig = projectConfig.registries[registryName];
-	if (!registryConfig) {
+	const registryName = registryNameForModule(module);
+	const registry = registries.get(registryName);
+	if (!registry) {
 		throw new Error(`Registry ${registryName} not found for module ${module}`);
 	}
-	const registryPath = await fetchAndResolveRegistry(
-		projectRoot,
-		registryConfig,
-	);
 
 	// Resolve module path
 	const pathModuleName = moduleNameInRegistry(moduleName, module);
-	const modulePath = join(registryPath, pathModuleName);
-	if (await exists(join(modulePath, "module.yaml"))) {
-		return modulePath;
-	} else {
+	const modulePath = join(registry.path, pathModuleName);
+	if (!await exists(join(modulePath, "module.yaml"))) {
 		if (pathModuleName != moduleName) {
 			// Has alias
 			throw new Error(
@@ -141,9 +131,26 @@ async function fetchAndResolveModule(
 			);
 		}
 	}
+
+	// Copy to gen dir
+	//
+	// We do this so generate files into the gen dir without modifying the
+	// original module. For example. if multiple projects are using the same
+	// local registry, we don't want conflicting generated files.
+	const dstPath = join(
+		projectRoot,
+		"_gen",
+		"modules",
+		registryName,
+		moduleName,
+	);
+	await Deno.mkdir(dirname(dstPath), { recursive: true });
+	await copy(modulePath, dstPath, { overwrite: true });
+
+	return { path: dstPath, registry };
 }
 
-function registryForModule(module: ProjectModuleConfig): string {
+function registryNameForModule(module: ProjectModuleConfig): string {
 	return module.registry ?? "default";
 }