Several Import improvements

* Ensure imported markers are added to the backup database
* Remove circular dependency between IntroEditor and ImportExport by
  explicitly reinitializing the necessary caches instead of a broader
  server soft reset.
* Integrate import_db into ServerCommands instead of its own category
* Tweak QueryParse to make formRaw return the full form entry (name,
  data, and any other fields) instead of just the data field, adding
  formString to get the raw data value of the form entry.

Server/FormDataParse.js

@@ -3,7 +3,8 @@ import ServerError from './ServerError.js';
 
 /** @typedef {!import('http').IncomingMessage} IncomingMessage */
 
-/** @typedef {{ [name: string]: { name : string, data : string, [optionalKeys: string]: string? } }} ParsedFormData */
+/** @typedef {{ name : string, data : string, [optionalKeys: string]: string? }} ParsedFormField */
+/** @typedef {{ [name: string]: ParsedFormField }} ParsedFormData */
 
 const Log = new ContextualLog('FormData');
 

Server/ImportExport.js

@@ -3,21 +3,22 @@ import { join } from 'path';
 
 import { ContextualLog } from '../Shared/ConsoleLog.js';
 
+import { BackupManager, MarkerBackupManager } from './MarkerBackupManager.js';
+import { MarkerConflictResolution, MarkerData } from '../Shared/PlexTypes.js';
 import { MetadataType, PlexQueries } from './PlexQueryManager.js';
 import DatabaseWrapper from './DatabaseWrapper.js';
-import FormDataParse from './FormDataParse.js';
-import { MarkerConflictResolution } from '../Shared/PlexTypes.js';
+import LegacyMarkerBreakdown from './LegacyMarkerBreakdown.js';
+import { MarkerCacheManager } from './MarkerCacheManager.js';
 import MarkerEditCache from './MarkerEditCache.js';
 import { ProjectRoot } from './IntroEditorConfig.js';
-import { sendJsonError } from './ServerHelpers.js';
 import ServerError from './ServerError.js';
-import { softRestart } from './IntroEditor.js';
 import TransactionBuilder from './TransactionBuilder.js';
 
 /** @typedef {!import('http').IncomingMessage} IncomingMessage */
 /** @typedef {!import('http').ServerResponse} ServerResponse */
 
 /** @typedef {!import('./FormDataParse.js').ParsedFormData} ParsedFormData */
+/** @typedef {!import('./FormDataParse.js').ParsedFormField} ParsedFormField */
 /** @typedef {!import('./MarkerCacheManager').MarkerQueryResult} MarkerQueryResult */
 /** @typedef {!import('../Shared/PlexTypes').MarkerAction} MarkerAction */
 
@@ -199,53 +200,39 @@ WHERE t.tag_id=$tagId`;
     }
 
     /**
-     * Import the markers in the database uploaded in the request.
-     * @param {IncomingMessage} request
-     * @param {ServerResponse} response */
-    static async importDatabase(request, response) {
-        try {
-            // 32 MiB max
-            const formData = await FormDataParse.parseRequest(request, 1024 * 1024 * 32);
-            if (!formData.database
-                || !formData.database.filename
-                || !formData.database.data
-                || !formData.sectionId
-                || isNaN(parseInt(formData.sectionId.data))
-                || !formData.resolveType
-                || isNaN(parseInt(formData.resolveType.data))
-                || Object.keys(MarkerConflictResolution).filter(
-                    k => MarkerConflictResolution[k] == parseInt(formData.resolveType.data)).length == 0) {
-                throw new ServerError(`Invalid parameters for import_db`);
-            }
+     * @param {ParsedFormField} database
+     * @param {number} sectionId
+     * @param {number} resolveType */
+    static async importDatabase(database, sectionId, resolveType) {
+        if (!database.filename) {
+            throw new ServerError(`importDatabase: no filename provided for database`);
+        }
 
-            // Form data looks good. Write the database to a real file.
-            const backupDir = join(ProjectRoot(), 'Backup', 'MarkerExports');
-            mkdirSync(backupDir, { recursive : true });
-            const dbData = Buffer.from(formData.database.data, 'binary');
-            const fullPath = join(backupDir, `Import-${formData.database.filename}`);
-            writeFileSync(fullPath, dbData);
-
-            const stats = await DatabaseImportExport.#doImport(
-                fullPath,
-                parseInt(formData.sectionId.data),
-                parseInt(formData.resolveType.data));
-
-            // Try to delete the temporarily uploaded file. Not a big deal if we can't though
-            try {
-                rmSync(fullPath);
-            } catch (err) {
-                Log.warn(err.message, `Unable to clean up uploaded database file`);
-            }
+        if (Object.keys(MarkerConflictResolution).filter(k => MarkerConflictResolution[k] == resolveType).length == 0) {
+            throw new ServerError(`importDatabase: resolveType must be a MarkerConflictResolution type, found ${resolveType}`);
+        }
+
+        const backupDir = join(ProjectRoot(), 'Backup', 'MarkerExports');
+        mkdirSync(backupDir, { recursive : true });
+        const dbData = Buffer.from(database.data, 'binary');
+        const fullPath = join(backupDir, `Import-${database.filename}`);
+        writeFileSync(fullPath, dbData);
 
-            // Force a mini-reload, as it's easier than trying to perfectly account for the right
-            // marker deltas, and import isn't expected to be a common scenario, so I don't really care
-            // about the slightly worse user experience. Wait until the reload completes before sending
-            // the response.
-            await softRestart(response, stats);
+        const stats = await DatabaseImportExport.#doImport(fullPath, sectionId, resolveType);
 
+        // Try to delete the temporarily uploaded file. Not a big deal if we can't though
+        try {
+            rmSync(fullPath);
         } catch (err) {
-            return sendJsonError(response, err);
+            Log.warn(err.message, `Unable to clean up uploaded database file`);
         }
+
+        // Success. Instead of trying to properly adjust everything, rebuild necessary caches from
+        // scratch, since this shouldn't be a common action, so efficiency isn't super important.
+        LegacyMarkerBreakdown.Clear();
+        await Promise.all([MarkerCacheManager.Reinitialize(), MarkerBackupManager.Reinitialize()]);
+
+        return stats;
     }
 
     /**
@@ -374,6 +361,28 @@ WHERE (base.metadata_type=1 OR base.metadata_type=4)`;
                 parseInt(sectionId),
                 sectionInfo.sectionType,
                 resolveType);
+
+            // Add changed markers to the backup database. While we'll clear out the BackupManager after this
+            // action, we still want the database to know about these changes so they can be restored if needed.
+            await BackupManager.recordAdds(restoredMarkerData.newMarkers.map(x => new MarkerData(x)));
+            await BackupManager.recordDeletes(restoredMarkerData.deletedMarkers.map(x => new MarkerData(x)));
+            const oldMarkerTimings = {};
+            const editedMarkers = [];
+            // Copied from MarkerBackupManager. Can this be shared?
+            for (const mod of restoredMarkerData.modifiedMarkers) {
+                const edited = mod.marker;
+                const newData = mod.newData;
+                oldMarkerTimings[edited.id] = { start : edited.start, end : edited.end };
+                edited.start = newData.newStart;
+                edited.end = newData.newEnd;
+                edited.modified_date = newData.newModified;
+                edited.marker_type = newData.newType;
+                edited.final = newData.newFinal;
+                editedMarkers.push(new MarkerData(edited));
+            }
+
+            await BackupManager.recordEdits(editedMarkers, oldMarkerTimings);
+
             stats.added += restoredMarkerData.newMarkers.length;
             stats.identical += restoredMarkerData.identicalMarkers.length;
             stats.deleted += restoredMarkerData.deletedMarkers.length;

Server/IntroEditor.js

@@ -289,37 +289,6 @@ async function userReload(res) {
     run();
 }
 
-/**
- * Do a soft internal restart to rebuild all internal caches
- * and reconnect to databases, usually after a large operation where
- * it's easier to just rebuild everything from scratch.
- *
- * TODO: How much of this can be moved to a different file instead of Main?
- *
- * @param {ServerResponse?} response The response to send when the reload completes.
- * @param {*?} data The data to send alongside the response, if any. */
-async function softRestart(response, data) {
-    Log.info('Soft reset started. Rebuilding everything.');
-    if (GetServerState() != ServerState.Running) {
-        Log.warn(`Attempting a soft reset when the server isn't running. Ignoring it.`);
-        return;
-    }
-
-    SetServerState(ServerState.Suspended);
-    await cleanupForShutdown(false /*fullShutdown*/);
-    Log.assert(GetServerState() == ServerState.Suspended, 'Server state changed during cleanup, that\'s not right!');
-    SetServerState(ServerState.SoftBoot);
-    if (response) {
-        ResumeResponse = response;
-    }
-
-    if (data) {
-        ResumeData = data;
-    }
-
-    run();
-}
-
 /** Creates the server. Called after verifying the config file and database. */
 async function launchServer() {
     if (!shouldCreateServer()) {
@@ -421,14 +390,6 @@ const ServerActionMap = {
     reload   : (res) => userReload(res),
 };
 
-/**
- * Map of actions that require more direct access to the underlying request and response.
- * Instead of adjusting ServerCommands to accommodate these, have a separate map.
- * @type {[endpoint: string]: (req: IncomingMessage, res: ServerResponse) => Promise<any>} */
-const RawActions = {
-    import_db : async (req, res) => await DatabaseImportExport.importDatabase(req, res),
-};
-
 /**
  * Handle POST requests, used to return JSON data queried by the client.
  * @param {IncomingMessage} req
@@ -446,14 +407,6 @@ async function handlePost(req, res) {
         return ServerActionMap[endpoint](res);
     }
 
-    if (RawActions[endpoint]) {
-        try {
-            return await RawActions[endpoint](req, res);
-        } catch (err) {
-            return sendJsonError(res, err);
-        }
-    }
-
     try {
         const response = await ServerCommands.runCommand(endpoint, req);
         sendJsonSuccess(res, response);
@@ -494,5 +447,3 @@ function checkTestData() {
 
     return testData;
 }
-
-export { softRestart };

Server/MarkerBackupManager.js

@@ -9,6 +9,7 @@ import { ContextualLog } from '../Shared/ConsoleLog.js';
 // Server dependencies/typedefs
 import { ExtraData, MetadataType, PlexQueries } from './PlexQueryManager.js';
 import { MarkerEnum, MarkerType } from '../Shared/MarkerType.js';
+import { Config } from './IntroEditorConfig.js';
 import DatabaseWrapper from './DatabaseWrapper.js';
 import { MarkerCache } from './MarkerCacheManager.js';
 import MarkerEditCache from './MarkerEditCache.js';
@@ -422,6 +423,12 @@ class MarkerBackupManager {
     /** Clear out the singleton backup manager instance. */
     static async Close() { await Instance?.close(); Instance = null; }
 
+    /**
+     * Clear out and rebuild purged marker information. */
+    static async Reinitialize() {
+        await Instance?.reinitialize();
+    }
+
     /**
      * @param {{[sectionId: number]: string}} uuids A map of section ids to UUIDs to uniquely identify a section across severs.
      * @param {{[sectionId: number]: number}} sectionTypes A map of section ids to the type of library it is.
@@ -437,6 +444,17 @@ class MarkerBackupManager {
         this.#buildMarkerEditDataCache();
     }
 
+    /**
+     * Clear out and rebuild purged marker information. */
+    async reinitialize() {
+        MarkerEditCache.clear();
+        await this.#buildMarkerEditDataCache();
+        this.#purgeCache = null;
+        if (Config.extendedMarkerStats()) {
+            await this.buildAllPurges();
+        }
+    }
+
     /** Closes the database connection. */
     async close() {
         Log.verbose('Shutting down backup database connection...');

Server/MarkerCacheManager.js

@@ -178,6 +178,11 @@ class MarkerCacheManager {
         return Instance;
     }
 
+    /** Clear out any cached data and rebuild it from scratch. */
+    static async Reinitialize() {
+        Instance?.reinitialize();
+    }
+
     static Close() { Instance = null; }
 
     /** All markers in the database.
@@ -208,6 +213,15 @@ class MarkerCacheManager {
         this.#tagId = tagId;
     }
 
+    /**
+     * Clear out and rebuild the marker cache. */
+    async reinitialize() {
+        this.#allMarkers = {};
+        this.#markerHierarchy = {};
+        this.#allBaseItems = new Set();
+        await this.buildCache();
+    }
+
     /**
      * Build the marker cache for the entire Plex server. */
     async buildCache() {

Server/QueryParse.js

@@ -98,7 +98,7 @@ class QueryParser {
      * Retrieve a value from the request's form-data.
      * @param {string} key */
     async formInt(key) {
-        const value = parseInt(await this.formRaw(key));
+        const value = parseInt((await this.formRaw(key)).data);
         if (isNaN(value)) {
             throw new QueryParameterException(`Expected an integer for '${key}', found something else.`);
         }
@@ -112,7 +112,14 @@ class QueryParser {
      * @param {(v: string) => any} transform The function that transforms the raw string to a custom object. */
     async formCustom(key, transform) {
         // transform should take care of any exceptions.
-        return transform(await this.formRaw(key));
+        return transform((await this.formRaw(key)).data);
+    }
+
+    /**
+     * Retrieve a string from the request's form data.
+     * @param {string} key The form field to retrieve. */
+    async formString(key) {
+        return (await this.formRaw(key)).data;
     }
 
     /**
@@ -129,7 +136,7 @@ class QueryParser {
             throw new QueryParameterException(`Form data field '${key} not found.`);
         }
 
-        return value.data;
+        return value;
     }
 }
 

Server/ServerCommands.js

@@ -1,6 +1,7 @@
 /** @typedef {!import('http').IncomingMessage} IncomingMessage */
 
 import { CoreCommands, GeneralCommands, PurgeCommands, QueryCommands } from './Commands/AllCommands.js';
+import DatabaseImportExport from './ImportExport.js';
 import LegacyMarkerBreakdown from './LegacyMarkerBreakdown.js';
 import QueryParser from './QueryParse.js';
 import ServerError from './ServerError.js';
@@ -24,7 +25,7 @@ class ServerCommands {
         bulk_add      : async (params) => await CoreCommands.bulkAdd(params.raw('type'), ...params.ints('id', 'start', 'end', 'resolveType'), params.ia('ignored')),
         add_custom    : async (params) => await CoreCommands.bulkAddCustom(
                                                                 await params.formInt('id'),
-                                                                await params.formRaw('type'),
+                                                                await params.formString('type'),
                                                                 await params.formCustom('markers', CoreCommands.parseCustomMarkerData),
                                                                 await params.formInt('resolveType')),
 
@@ -47,6 +48,7 @@ class ServerCommands {
         restore_purge : async (params) => await PurgeCommands.restoreMarkers(params.ia('markerIds'), ...params.ints('sectionId', 'resolveType')),
         ignore_purge  : async (params) => await PurgeCommands.ignorePurgedMarkers(params.ia('markerIds'), params.i('sectionId')),
 
+        import_db     : async (params) => await DatabaseImportExport.importDatabase(await params.formRaw('database'), await params.formInt('sectionId'), await params.formInt('resolveType')),
         nuke_section  : async (params) => await PurgeCommands.nukeSection(...params.ints('sectionId', 'deleteType')),
     };
     /* eslint-enable */