From 2e8a41b7e331ec977c6ecdc47307014c4d3b34e8 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Tue, 5 Mar 2024 16:20:49 -0800 Subject: [PATCH] feat: git registries --- src/build/mod.ts | 31 ++++--- src/cli/commands/db.ts | 3 +- src/config/project.ts | 14 ++- src/migrate/deploy.ts | 6 +- src/project/deps.ts | 3 +- src/project/module.ts | 4 + src/project/project.ts | 85 ++++++++++-------- src/project/registry.ts | 157 +++++++++++++++++++++++++++++++++ tests/test_project/opengb.yaml | 11 +-- 9 files changed, 245 insertions(+), 69 deletions(-) create mode 100644 src/project/registry.ts diff --git a/src/build/mod.ts b/src/build/mod.ts index 5b2a4b1d..e99abe0e 100644 --- a/src/build/mod.ts +++ b/src/build/mod.ts @@ -23,6 +23,7 @@ import { Module, Script } from "../project/mod.ts"; import { shutdownAllPools } from "../utils/worker_pool.ts"; import { migrateDev } from "../migrate/dev.ts"; import { compileModuleTypeHelper } from "./gen.ts"; +import { migrateDeploy } from "../migrate/deploy.ts"; /** * Which format to use for building. @@ -237,18 +238,26 @@ async function buildSteps( project: Project, opts: BuildOpts, ) { - const modules = Array.from(project.modules.values()); - - buildStep(buildState, { - name: "Prisma schema", - files: modules.map((module) => join(module.path, "db", "schema.prisma")), - async build() { - await migrateDev(project, modules, { - createOnly: false, + // TODO: This does not reuse the Prisma dir or the database connection, so is extra slow on the first run. Figure out how to make this use one `migrateDev` command and pass in any modified modules For now, these need to be migrated individually because `prisma migrate dev` is an interactive command. Also, making a database change and waiting for all other databases to re-apply will take a long tim.e + for (const module of project.modules.values()) { + if (module.db) { + buildStep(buildState, { + name: `Migrate (${module.name})`, + module, + files: [join(module.path, "db", "schema.prisma")], + async build() { + if ('directory' in module.registry) { + // Update migrations + await migrateDev(project, [module], { createOnly: false }); + } else { + // Do not alter migrations, only deploy them + await migrateDeploy(project, [module]); + } + }, }); - }, - }); - + } + } + buildStep(buildState, { name: "Inflate runtime", // TODO: Add way to compare runtime version diff --git a/src/cli/commands/db.ts b/src/cli/commands/db.ts index fde70970..e3a7ffb2 100644 --- a/src/cli/commands/db.ts +++ b/src/cli/commands/db.ts @@ -23,7 +23,8 @@ dbCommand.command("reset").action(async (opts) => { }); dbCommand.command("deploy").action(async (opts) => { - await migrateDeploy(await initProject(opts)); + const project = await initProject(opts); + await migrateDeploy(await initProject(opts), [...project.modules.values()]); }); // TODO: https://github.com/rivet-gg/open-game-services-engine/issues/84 diff --git a/src/config/project.ts b/src/config/project.ts index 5ae7f51b..74a4d101 100644 --- a/src/config/project.ts +++ b/src/config/project.ts @@ -7,17 +7,13 @@ export interface ProjectConfig extends Record { modules: { [name: string]: ProjectModuleConfig }; } -export interface RegistryConfig extends Record { - directory?: string; - // git?: string; - // branch?: string; - // rev?: string; +export type RegistryConfig = { local: RegistryConfigLocal } | { git: RegistryConfigGit }; + +export interface RegistryConfigLocal { + directory: string; } -// export interface RegistryConfig extends Record { -// name: string; -// url: string; -// } +export type RegistryConfigGit = { url: string, directory?: string } & ({ branch: string } | { tag: string } | { rev: string }); export interface ProjectModuleConfig extends Record { /** diff --git a/src/migrate/deploy.ts b/src/migrate/deploy.ts index c3518a40..8f2719ea 100644 --- a/src/migrate/deploy.ts +++ b/src/migrate/deploy.ts @@ -1,10 +1,10 @@ // Wrapper around `prisma migrate deploy` -import { Project } from "../project/mod.ts"; +import { Module, Project } from "../project/mod.ts"; import { forEachPrismaSchema } from "./mod.ts"; -export async function migrateDeploy(project: Project) { - await forEachPrismaSchema(project, [...project.modules.values()], async ({ databaseUrl, tempDir }) => { +export async function migrateDeploy(project: Project, modules: Module[]) { + await forEachPrismaSchema(project, modules, async ({ databaseUrl, tempDir }) => { // Generate migrations & client const status = await new Deno.Command("deno", { args: ["run", "-A", "npm:prisma@5.9.1", "migrate", "deploy"], diff --git a/src/project/deps.ts b/src/project/deps.ts index eba747bc..92851593 100644 --- a/src/project/deps.ts +++ b/src/project/deps.ts @@ -1,2 +1,3 @@ export * as glob from "npm:glob@^10.3.10"; -export * as tjs from "npm:typescript-json-schema@^0.62.0"; \ No newline at end of file +export * as tjs from "npm:typescript-json-schema@^0.62.0"; +export { bold, brightRed } from "https://deno.land/std@0.208.0/fmt/colors.ts"; diff --git a/src/project/module.ts b/src/project/module.ts index 34ae256d..325b43b7 100644 --- a/src/project/module.ts +++ b/src/project/module.ts @@ -4,11 +4,13 @@ import { readConfig as readModuleConfig } from "../config/module.ts"; import { ModuleConfig } from "../config/module.ts"; import { Script } from "./script.ts"; import { Project } from "./project.ts"; +import { Registry } from "./registry.ts"; export interface Module { path: string; name: string; config: ModuleConfig; + registry: Registry, scripts: Map; db?: ModuleDatabase; } @@ -20,6 +22,7 @@ export interface ModuleDatabase { export async function loadModule( modulePath: string, name: string, + registry: Registry, ): Promise { // Read config const config = await readModuleConfig(modulePath); @@ -79,6 +82,7 @@ export async function loadModule( path: modulePath, name, config, + registry, scripts, db, }; diff --git a/src/project/project.ts b/src/project/project.ts index 030a629a..0cced978 100644 --- a/src/project/project.ts +++ b/src/project/project.ts @@ -1,14 +1,15 @@ -import { exists, join } from "../deps.ts"; +import { dirname, exists, join, copy } from "../deps.ts"; +import { bold, brightRed } from "./deps.ts"; import { readConfig as readProjectConfig } from "../config/project.ts"; import { ProjectConfig } from "../config/project.ts"; import { loadModule, Module } from "./module.ts"; -import { RegistryConfig } from "../config/project.ts"; +import { loadRegistry, Registry } from "./registry.ts"; import { ProjectModuleConfig } from "../config/project.ts"; -import { bold, brightRed } from "https://deno.land/std@0.208.0/fmt/colors.ts"; export interface Project { path: string; - projectConfig: ProjectConfig; + config: ProjectConfig; + registries: Map; modules: Map; } @@ -19,26 +20,36 @@ export interface LoadProjectOpts { export async function loadProject(opts: LoadProjectOpts): Promise { const projectRoot = join(Deno.cwd(), opts.path ?? "."); - // console.log("Loading project", projectRoot); - // Read project config const projectConfig = await readProjectConfig( projectRoot, ); + // Load registries + const registries = new Map(); + for (const registryName in projectConfig.registries) { + const registryConfig = projectConfig.registries[registryName]; + const registry = await loadRegistry( + projectRoot, + registryName, + registryConfig, + ); + registries.set(registryName, registry); + } + // Load modules const modules = new Map(); for (const projectModuleName in projectConfig.modules) { - const modulePath = await fetchAndResolveModule( + const { path, registry } = await fetchAndResolveModule( projectRoot, projectConfig, + registries, projectModuleName, ); - const module = await loadModule(modulePath, projectModuleName); + const module = await loadModule(path, projectModuleName, registry); modules.set(projectModuleName, module); } - // Verify existince of module dependencies const missingDepsByModule = new Map(); for (const module of modules.values()) { @@ -81,23 +92,7 @@ export async function loadProject(opts: LoadProjectOpts): Promise { - return { path: projectRoot, projectConfig, modules }; -} - -/** - * Clones a registry to the local machine if required and returns the path. - */ -async function fetchAndResolveRegistry( - projectRoot: string, - registry: RegistryConfig, -): Promise { - // TODO: Impl git cloning - if (!registry.directory) throw new Error("Registry directory not set"); - const registryPath = join(projectRoot, registry.directory); - if (!await exists(registryPath)) { - throw new Error(`Registry not found at ${registryPath}`); - } - return registryPath; + return { path: projectRoot, config: projectConfig, registries, modules }; } /** @@ -106,29 +101,24 @@ async function fetchAndResolveRegistry( async function fetchAndResolveModule( projectRoot: string, projectConfig: ProjectConfig, + registries: Map, moduleName: string, -): Promise { +): Promise<{ path: string; registry: Registry }> { // Lookup module const module = projectConfig.modules[moduleName]; if (!module) throw new Error(`Module not found ${moduleName}`); // Lookup registry - const registryName = registryForModule(module); - const registryConfig = projectConfig.registries[registryName]; - if (!registryConfig) { + const registryName = registryNameForModule(module); + const registry = registries.get(registryName); + if (!registry) { throw new Error(`Registry ${registryName} not found for module ${module}`); } - const registryPath = await fetchAndResolveRegistry( - projectRoot, - registryConfig, - ); // Resolve module path const pathModuleName = moduleNameInRegistry(moduleName, module); - const modulePath = join(registryPath, pathModuleName); - if (await exists(join(modulePath, "module.yaml"))) { - return modulePath; - } else { + const modulePath = join(registry.path, pathModuleName); + if (!await exists(join(modulePath, "module.yaml"))) { if (pathModuleName != moduleName) { // Has alias throw new Error( @@ -141,9 +131,26 @@ async function fetchAndResolveModule( ); } } + + // Copy to gen dir + // + // We do this so generate files into the gen dir without modifying the + // original module. For example. if multiple projects are using the same + // local registry, we don't want conflicting generated files. + const dstPath = join( + projectRoot, + "_gen", + "modules", + registryName, + moduleName, + ); + await Deno.mkdir(dirname(dstPath), { recursive: true }); + await copy(modulePath, dstPath, { overwrite: true }); + + return { path: dstPath, registry }; } -function registryForModule(module: ProjectModuleConfig): string { +function registryNameForModule(module: ProjectModuleConfig): string { return module.registry ?? "default"; } diff --git a/src/project/registry.ts b/src/project/registry.ts new file mode 100644 index 00000000..46a77517 --- /dev/null +++ b/src/project/registry.ts @@ -0,0 +1,157 @@ +import { exists, join, emptyDir } from "../deps.ts"; +import { + RegistryConfig, + RegistryConfigGit, + RegistryConfigLocal, +} from "../config/project.ts"; + +export interface Registry { + path: string; + name: string; + config: RegistryConfig; +} + +/** + * Clones a registry to the local machine if required and returns the path. + */ +export async function loadRegistry( + projectRoot: string, + name: string, + config: RegistryConfig, +): Promise { + let path: string; + if ("local" in config) { + path = await resolveRegistryLocal(projectRoot, config.local); + } else if ("git" in config) { + path = await resolveRegistryGit(projectRoot, name, config.git); + } else { + // Unknown project config + throw new Error("Unreachable"); + } + + return { + path, + name, + config, + }; +} + +async function resolveRegistryLocal( + projectRoot: string, + config: RegistryConfigLocal, +): Promise { + // Check that registry exists + const path = join(projectRoot, config.directory); + if (!await exists(path)) { + throw new Error(`Registry not found at ${path}`); + } + return path; +} + +async function resolveRegistryGit( + projectRoot: string, + name: string, + config: RegistryConfigGit, +): Promise { + const repoPath = join(projectRoot, "_gen", "git_registries", name); + const gitRef = resolveGitRef(config); + + // Clone repo if needed + if (!await exists(join(repoPath, ".git"))) { + console.log('📦 Cloning git registry', config.url) + + // Remove potentially dirty existing directory + await emptyDir(repoPath); + + // Clone repo + const cloneOutput = await new Deno.Command("git", { + args: ["clone", "--single-branch", config.url, repoPath], + }).output(); + if (!cloneOutput.success) { + throw new Error( + `Failed to clone registry ${config.url}:\n${ + new TextDecoder().decode(cloneOutput.stderr) + }`, + ); + } + } + + // Discard any changes + const unstagedDiffOutput = await new Deno.Command("git", { + cwd: repoPath, + args: ["diff", "--quiet"], + }).output(); + const stagedDiffOutput = await new Deno.Command("git", { + cwd: repoPath, + args: ["diff", "--quiet", "--cached"], + }).output(); + if (!unstagedDiffOutput.success || !stagedDiffOutput.success) { + console.warn("💣 Discarding changes in git registry", name); + + const resetOutput = await new Deno.Command("git", { + cwd: repoPath, + args: ["reset", "--hard"], + }).output(); + if (!resetOutput.success) { + throw new Error( + `Failed to reset registry ${config.url}:\n${ + new TextDecoder().decode(resetOutput.stderr) + }`, + ); + } + } + + // Check if rev exists locally, if not try fetch it + const catOutput = await new Deno.Command("git", { + cwd: repoPath, + args: ["cat-file", "-t", gitRef], + }).output(); + if (!catOutput.success) { + console.log('📦 Fetching git registry', config.url, gitRef); + + const fetchOutput = await new Deno.Command("git", { + cwd: repoPath, + args: ["fetch", "origin", gitRef], + }).output(); + if (!fetchOutput.success) { + throw new Error( + `Failed to fetch registry ${config.url} at ${gitRef}:\n${ + new TextDecoder().decode(fetchOutput.stderr) + }`, + ); + } + } + + // Checkout commit + const checkoutOutput = await new Deno.Command("git", { + cwd: repoPath, + args: ["checkout", gitRef], + }).output(); + if (!checkoutOutput.success) { + throw new Error( + `Failed to checkout registry ${config.url} at ${gitRef}:\n${ + new TextDecoder().decode(checkoutOutput.stderr) + }`, + ); + } + + // Join sub directory + const path = join(repoPath, config.directory ?? ""); + if (!await exists(path)) { + throw new Error(`Registry not found at ${path}`); + } + + return path; +} + +function resolveGitRef(registryConfig: RegistryConfigGit): string { + if ('rev' in registryConfig) { + return registryConfig.rev; + } else if ('branch' in registryConfig) { + return registryConfig.branch; + } else if ('tag' in registryConfig) { + return `tags/${registryConfig.tag}`; + } else { + throw new Error("Unreachable"); + } +} \ No newline at end of file diff --git a/tests/test_project/opengb.yaml b/tests/test_project/opengb.yaml index ab527cf5..fd883b74 100644 --- a/tests/test_project/opengb.yaml +++ b/tests/test_project/opengb.yaml @@ -1,11 +1,12 @@ registries: - # default: - # git: https://github.com/rivet-gg/open-game-services.git - # branch: main default: - directory: ../../../opengb-registry/modules + git: + url: https://github.com/rivet-gg/open-game-services.git + branch: main + directory: ./modules local: - directory: ./modules + local: + directory: ./modules modules: # currency: {} # friends: {}