Reworked dynamic module loading, fixing startup issues for migration and compact jobs

.changeset/nasty-snakes-double.md

@@ -0,0 +1,6 @@
+---
+'@powersync/service-core': patch
+'@powersync/service-image': patch
+---
+
+- Rework dynamic module loading, fixing startup issues for migration and compact jobs in 1.18.0 / 1.18.1

packages/service-core/src/modules/ModuleManager.ts

@@ -1,11 +1,13 @@
 import { logger } from '@powersync/lib-services-framework';
 import * as system from '../system/system-index.js';
 import { AbstractModule, TearDownOptions } from './AbstractModule.js';
+import { loadModules, ModuleLoaders } from './loader.js';
 /**
  *  The module manager keeps track of activated modules
  */
 export class ModuleManager {
   private readonly modules: Map<string, AbstractModule> = new Map();
+  private moduleLoaders: ModuleLoaders | undefined;
 
   public register(modules: AbstractModule[]) {
     for (const module of modules) {
@@ -18,7 +20,18 @@ export class ModuleManager {
     }
   }
 
+  public registerDynamicModules(moduleLoaders: ModuleLoaders) {
+    this.moduleLoaders = moduleLoaders;
+  }
+
   async initialize(serviceContext: system.ServiceContextContainer) {
+    logger.info(`Loading dynamic modules...`);
+    if (this.moduleLoaders) {
+      const dynamicModules = await loadModules(serviceContext.configuration, this.moduleLoaders);
+      this.register(dynamicModules);
+    }
+    logger.info(`Successfully loaded dynamic modules.`);
+
     logger.info(`Initializing modules...`);
     for (const module of this.modules.values()) {
       await module.initialize(serviceContext);

service/src/entry.ts

@@ -6,13 +6,15 @@ import { startServer } from './runners/server.js';
 import { startStreamRunner } from './runners/stream-worker.js';
 import { startUnifiedRunner } from './runners/unified-runner.js';
 import { createSentryReporter } from './util/alerting.js';
+import { DYNAMIC_MODULES } from './util/modules.js';
 
 // Initialize framework components
 container.registerDefaults();
 container.register(ContainerImplementation.REPORTER, createSentryReporter());
 
 const moduleManager = new core.modules.ModuleManager();
 moduleManager.register([new CoreModule()]);
+moduleManager.registerDynamicModules(DYNAMIC_MODULES);
 // This is a bit of a hack. Commands such as the teardown command or even migrations might
 // want access to the ModuleManager in order to use modules
 container.register(core.ModuleManager, moduleManager);

service/src/runners/server.ts

@@ -2,7 +2,6 @@ import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
 
 import { logBooting } from '../util/version.js';
-import { DYNAMIC_MODULES } from '../util/modules.js';
 
 /**
  * Starts an API server
@@ -13,10 +12,6 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) {
   const config = await core.utils.loadConfig(runnerConfig);
 
   const moduleManager = container.getImplementation(core.modules.ModuleManager);
-  const modules = await core.loadModules(config, DYNAMIC_MODULES);
-  if (modules.length > 0) {
-    moduleManager.register(modules);
-  }
 
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.API,

service/src/runners/stream-worker.ts

@@ -2,7 +2,6 @@ import { container, logger } from '@powersync/lib-services-framework';
 import * as core from '@powersync/service-core';
 
 import { logBooting } from '../util/version.js';
-import { DYNAMIC_MODULES } from '../util/modules.js';
 
 /**
  * Configures the replication portion on a {@link serviceContext}
@@ -24,10 +23,6 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) =
   const config = await core.utils.loadConfig(runnerConfig);
 
   const moduleManager = container.getImplementation(core.modules.ModuleManager);
-  const modules = await core.loadModules(config, DYNAMIC_MODULES);
-  if (modules.length > 0) {
-    moduleManager.register(modules);
-  }
 
   // Self-hosted version allows for automatic migrations
   const serviceContext = new core.system.ServiceContextContainer({

service/src/runners/unified-runner.ts

@@ -3,7 +3,6 @@ import * as core from '@powersync/service-core';
 
 import { logBooting } from '../util/version.js';
 import { registerReplicationServices } from './stream-worker.js';
-import { DYNAMIC_MODULES } from '../util/modules.js';
 
 /**
  * Starts an API server
@@ -14,10 +13,6 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig)
   const config = await core.utils.loadConfig(runnerConfig);
 
   const moduleManager = container.getImplementation(core.modules.ModuleManager);
-  const modules = await core.loadModules(config, DYNAMIC_MODULES);
-  if (modules.length > 0) {
-    moduleManager.register(modules);
-  }
 
   const serviceContext = new core.system.ServiceContextContainer({
     serviceMode: core.system.ServiceContextMode.UNIFIED,