Sync from old repo 2025-12-03

.git-blame-ignore-revs

@@ -1,3 +1,4 @@
 # Format Commits
 9077b5b81d266818af4a25b90c78e0e7d03620a3
-9fba4ebed0b37eac1bb113a43698274be4e2f0b2
+9fba4ebed0b37eac1bb113a43698274be4e2f0b2
+84de82b25fff81f7b28c1a83789d1a9247fa73b4

.github/workflows/lint-code.yaml

@@ -0,0 +1,21 @@
+name: Lint Source Code
+on:
+  push:
+    branches:
+      - main
+      - 'release_*'
+  pull_request:
+  schedule:
+    - cron: '11 13 * * 1-5' # Run M-F at 13:11
+  workflow_dispatch:
+
+jobs:
+  runLinter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          package-manager-cache: false
+      - run: npm i
+      - run: npm run lint:required

CONTRIBUTING.md

@@ -21,6 +21,16 @@ Run unit tests using `npm run test:unit <unit-test-file>` or `npm run test:unit:
 
 > Unit tests currently use [Mocha](https://mochajs.org/) as the test runner, but since they are implemented in TypeScript and are sometimes executing TypeScript source code, it also uses [TSX](https://tsx.is/) for compilation and execution. The npm script `test:unit` sets the appropriate env vars and mocha configuration file. Make sure that the `TSX_TSCONFIG_PATH` environment variable points to the correct `tsconfig.json` file for the unit tests (i.e. `./unitTests/tsconfig.json`) and not the root-level `tsconfig.json`.
 
+## Code Formatting & Linting
+
+We currently use [prettier](https://prettier.io) and [eslint](https://eslint.org) to enforce code formatting and
+linting, respectively. While we do enforce conformity to prettier's ruleset in CI, we're taking an incremental approach
+with eslint. Rules that can be globally enforced are enabled in the `eslint.required.config.mjs` file, while the
+aspirational rules are in `eslint.config.mjs`. This is because when editing or linting new code locally, we want to
+adhere to the full ruleset in `eslint.config.mjs`. But in CI we only want to enforce those rules enabled in
+`eslint.required.config.mjs`. We will enable additional rules in `eslint.required.config.mjs` over time until there is
+one ruleset that is enforced everywhere. PRs that allow enabling additional eslint rules are welcome!
+
 ## Dependency Version Updates
 
 We use [Renovate](https://www.mend.io/renovate/) to automatically update dependencies on a regular schedule (with a cooldown period to help guard against supply chain attacks).
@@ -56,9 +66,9 @@ These are the steps @Ethan-Arrowood has been following to synchronize the reposi
    - The start-commit will be excluded so always include the last commit from `old/main` that was synchronized previously.
    - The hash for that commit is recorded at the end of this section and should be updated after each synchronization
 5. Go through commits one-by-one, from the top to the bottom of the `commits-to-pick.txt` file, following these steps:
-   1. If `git rev-parse $(commit)^2 &>/dev/null 2>&1` exits with 0 then its a merge commit
+   1. If `git rev-parse $(commit)^2 &>/dev/null 2>&1` exits with 0 then it's a merge commit
       1. Execute `git cherry-pick -m 1 $(commit)`
-   2. Else its a regular commit
+   2. Else it's a regular commit
       1. Execute `git cherry-pick $(commit)`
    3. If either cherry-pick command results in a non-zero exit code that means there is a merge conflict
       1. If the conflict is a content, resolve it manually and `git add` the file
@@ -67,7 +77,7 @@ These are the steps @Ethan-Arrowood has been following to synchronize the reposi
          - Example: `CONFLICT (modify/delete): unitTests/bin/copyDB-test.js deleted in HEAD and modified in f75d9170b`
       3. Then check `git status`, if there is nothing you can `git cherry-pick --skip`
          - Note: in this circumstance, running `git cherry-pick --continue` results in a non-zero exit code with the message `The previous cherry-pick is now empty, possibly due to conflict resolution.` Maybe we use this to then run `--skip`? Or maybe there is a way to parse the output of previous `git status` step?
-6. After all commits have been picked, manually check that everything brought over was supposed to be. Look out for any source code we do not want open sourced or things like unit tests which we are actively migrating separately (and will eventually include as part of the synchronization process)
+6. After all commits have been picked, manually check that everything brought over was supposed to be. Look out for any source code we do not want open-sourced or things like unit tests which we are actively migrating separately (and will eventually include as part of the synchronization process)
    - The GitHub PR UI is useful for this step; but make sure to leave the PR as a draft until all synchronization steps are complete
 7. Once everything looks good, run `npm run format:write` to ensure formatting is correct
 8. Commit the formatting changes
@@ -77,7 +87,7 @@ These are the steps @Ethan-Arrowood has been following to synchronize the reposi
 12. Push all changes and open the PR for review
 13. Merge using a Merge Commit so that all relative history is retained and things like the formatting change hash stays the same as recorded.
 
-Last Synchronized Commit: `2030325c105abede4c324511d92d8f0c6357877b`
+Last Synchronized Commit: `5edb685e036c88941cba58f8294dea0888137596`
 
 ## Code of Conduct
 

bin/run.js

@@ -96,7 +96,13 @@ async function initialize(calledByInstall = false, calledByMain = false) {
 	// we do not need to update/backup the config file on run.
 	if (!calledByInstall) {
 		// If run is called with cmd/env vars we create a backup of config and update config file.
-		const parsedArgs = assignCMDENVVariables(Object.keys(terms.CONFIG_PARAM_MAP), true);
+		let parsedArgs = assignCMDENVVariables(Object.keys(terms.CONFIG_PARAM_MAP), true);
+
+		// If HARPER_SET_CONFIG is present, filter out any config keys that are set in it
+		// to prevent individual env vars from overriding explicit runtime configuration
+		const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars.ts');
+		parsedArgs = filterArgsAgainstRuntimeConfig(parsedArgs);
+
 		if (!hdbUtils.isEmpty(parsedArgs) && !hdbUtils.isEmptyOrZeroLength(Object.keys(parsedArgs))) {
 			configUtils.updateConfigValue(undefined, undefined, parsedArgs, true, true);
 		}

components/Application.ts

@@ -123,7 +123,7 @@
 	if (application.payload) {
 		// Given a payload, create a Readable from the Buffer or string
 		tarball = Readable.from(
 			application.payload instanceof Buffer ? application.payload : Buffer.from(application.payload, 'base64')
 		);
 	} else {
 		// Given a package, there are a a couple options
@@ -444,7 +444,7 @@
 
 	const harperApplicationLockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json');
 
-	let harperApplicationLock: any = { application: {} };
+	let harperApplicationLock: { applications: Record<string, ApplicationConfig> } = { applications: {} };
 	try {
 		harperApplicationLock = JSON.parse(await readFile(harperApplicationLockPath, 'utf8'));
 	} catch (error) {
@@ -461,29 +461,33 @@
 			continue;
 		}
 
-		// Then do proper error-based validation with TypeScript `asserts` to provide type safety
-		// This will throw if the config is invalid
-		assertApplicationConfig(name, applicationConfig);
+		try {
+			// Then do proper error-based validation with TypeScript `asserts` to provide type safety
+			// This will throw if the config is invalid
+			assertApplicationConfig(name, applicationConfig);
 
-		const application = new Application({
-			name,
-			packageIdentifier: applicationConfig.package,
-			install: applicationConfig.install,
-		});
+			const application = new Application({
+				name,
+				packageIdentifier: applicationConfig.package,
+				install: applicationConfig.install,
+			});
 
-		// Lock check: only install if not already installed with matching configuration
-		if (
-			existsSync(application.dirPath) &&
-			harperApplicationLock.applications[name] &&
-			JSON.stringify(harperApplicationLock.applications[name]) === JSON.stringify(applicationConfig)
-		) {
-			logger.info(`Application ${name} is already installed with matching configuration; skipping installation`);
-			continue;
-		}
+			// Lock check: only install if not already installed with matching configuration
+			if (
+				existsSync(application.dirPath) &&
+				harperApplicationLock.applications[name] &&
+				JSON.stringify(harperApplicationLock.applications[name]) === JSON.stringify(applicationConfig)
+			) {
+				logger.info(`Application ${name} is already installed with matching configuration; skipping installation`);
+				continue;
+			}
 
-		applicationInstallationPromises.push(prepareApplication(application));
+			applicationInstallationPromises.push(prepareApplication(application));
 
-		harperApplicationLock.applications[name] = applicationConfig;
+			harperApplicationLock.applications[name] = applicationConfig;
+		} catch (error) {
+			logger.error(`Skipping installation of application ${name} due to invalid configuration: ${error.message}`);
+		}
 	}
 
 	const applicationInstallationStatuses = await Promise.allSettled(applicationInstallationPromises);

components/componentLoader.ts

@@ -74,7 +74,7 @@
 	});
 }
 
-const TRUSTED_RESOURCE_PLUGINS = {
+export const TRUSTED_RESOURCE_PLUGINS = {
 	REST, // for backwards compatibility with older configs
 	rest: REST,
 	graphql: graphqlQueryHandler,
@@ -235,7 +235,6 @@
 		if (existsSync(configPath)) {
 			config = isRoot ? getConfigObj() : parseDocument(readFileSync(configPath, 'utf8')).toJSON();
 			// if not found, look for the generic config.yaml, the config filename we have historically used, but only if not the root
-			// eslint-disable-next-line sonarjs/no-nested-assignment
 		} else if (!isRoot && existsSync((configPath = join(componentDirectory, 'config.yaml')))) {
 			config = parseDocument(readFileSync(configPath, 'utf8')).toJSON();
 		} else {

components/operations.js

@@ -364,6 +364,17 @@ async function deployComponent(req) {
 	// Write to root config if the request contains a package identifier
 	// TODO: how can we keep record of the `payload`? Its often too large to stuff into a config file; especially the root config. Maybe we can write it to a file and reference that way?
 	if (req.package) {
+		// Check if trying to overwrite a core component (requires force)
+		// Lazy-load to avoid circular dependency with componentLoader
+		const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts');
+		if (TRUSTED_RESOURCE_PLUGINS[req.project] && !req.force) {
+			throw handleHDBError(
+				new Error(),
+				`Cannot deploy component with name '${req.project}': this is a protected core component name. Use force: true to overwrite.`,
+				HTTP_STATUS_CODES.CONFLICT
+			);
+		}
+
 		const applicationConfig = { package: req.package };
 		// Avoid writing an empty `install:` block
 		if (req.install_command || req.install_timeout) {

components/operationsValidation.js

@@ -239,6 +239,7 @@ function deployComponentValidator(req) {
 		restart: Joi.alternatives().try(Joi.boolean(), Joi.string().valid('rolling')).optional(),
 		install_command: Joi.string().optional(),
 		install_timeout: Joi.number().optional(),
+		force: Joi.boolean().optional(),
 	});
 
 	return validator.validateBySchema(req, deployProjSchema);

config/configUtils.js

@@ -120,13 +120,15 @@ function createConfigFile(args, skipFsValidation = false) {
 
 	if (schemasArgs) setSchemasConfig(configDoc, schemasArgs);
 
-	// Validates config doc and if required sets default values for some parameters.
-	validateConfig(configDoc, skipFsValidation);
-
-	// Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG environment variables
+	// Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG environment variables BEFORE validation
+	// This allows runtime env vars to resolve port conflicts before validation
 	// Must be called AFTER rootPath is set in configDoc
 	// Mutates configDoc in place
 	applyRuntimeEnvVarConfig(configDoc, null, { isInstall: true });
+
+	// Validates config doc and if required sets default values for some parameters.
+	validateConfig(configDoc, skipFsValidation);
+
 	const configObj = configDoc.toJSON();
 	flatConfigObj = flattenConfig(configObj);
 
@@ -757,9 +759,16 @@ function applyRuntimeEnvVarConfig(configDoc, configFilePath, options = {}) {
 		// Apply env vars with source tracking and drift detection
 		applyRuntimeEnvConfig(configObj, rootPath, options);
 
-		// Convert back to YAML document and write to file
+		// Update the YAML document's contents
+		// We update only the 'contents' property to preserve the Document instance and its methods
 		const mergedDoc = YAML.parseDocument(YAML.stringify(configObj), { simpleKeys: true });
-		Object.assign(configDoc, mergedDoc);
+
+		// Check for YAML parsing errors
+		if (mergedDoc.errors?.length > 0) {
+			throw new Error(`Error parsing harperdb-config.yaml: ${mergedDoc.errors}`);
+		}
+
+		configDoc.contents = mergedDoc.contents;
 	} catch (error) {
 		logger.error(`Failed to apply runtime env config: ${error.message}`);
 		throw error;

config/harperConfigEnvVars.ts

@@ -103,6 +103,71 @@ function isPlainObject(value: any): value is Record<string, any> {
 	);
 }
 
+/**
+ * Filters out arguments that are already set in HARPER_SET_CONFIG.
+ * This prevents individual environment variables from overriding runtime configuration.
+ *
+ * Note: Only filters against HARPER_SET_CONFIG, not HARPER_DEFAULT_CONFIG, since
+ * HARPER_DEFAULT_CONFIG sets defaults that can be overridden by individual env vars.
+ *
+ * @param args - Object containing individual env var arguments (e.g., from assignCMDENVVariables)
+ * @returns Filtered args object with HARPER_SET_CONFIG keys removed
+ *
+ * @example
+ * // If HARPER_SET_CONFIG sets operationsApi.network.port
+ * const args = { operationsapi_network_port: '9925', rootpath: '/var/hdb' };
+ * const filtered = filterArgsAgainstRuntimeConfig(args);
+ * // Returns: { rootpath: '/var/hdb' }
+ */
+export function filterArgsAgainstRuntimeConfig(args: Record<string, any>): Record<string, any> {
+	// Only filter against HARPER_SET_CONFIG (not HARPER_DEFAULT_CONFIG)
+	if (!process.env.HARPER_SET_CONFIG) {
+		return args;
+	}
+
+	// Parse HARPER_SET_CONFIG
+	let setConfig: ConfigObject;
+	try {
+		setConfig = JSON.parse(process.env.HARPER_SET_CONFIG);
+	} catch (err) {
+		// If parsing fails, log warning and return args unchanged
+		const logger = getLogger();
+		logger.warn('Failed to parse HARPER_SET_CONFIG for arg filtering', err);
+		return args;
+	}
+
+	// If no valid config, return args unchanged
+	if (Object.keys(setConfig).length === 0) {
+		return args;
+	}
+
+	// Flatten HARPER_SET_CONFIG to get all keys
+	const flattenSetConfig = (obj: ConfigObject, prefix = ''): Set<string> => {
+		const keys = new Set<string>();
+		for (const key in obj) {
+			const newKey = prefix ? `${prefix}_${key}` : key;
+			if (obj[key] !== null && typeof obj[key] === 'object' && !Array.isArray(obj[key])) {
+				flattenSetConfig(obj[key], newKey).forEach((k) => keys.add(k));
+			} else {
+				keys.add(newKey.toLowerCase());
+			}
+		}
+		return keys;
+	};
+
+	const setConfigKeys = flattenSetConfig(setConfig);
+
+	// Filter out args that are in HARPER_SET_CONFIG
+	const filteredArgs: Record<string, any> = {};
+	for (const key in args) {
+		if (!setConfigKeys.has(key.toLowerCase())) {
+			filteredArgs[key] = args[key];
+		}
+	}
+
+	return filteredArgs;
+}
+
 /**
  * Flatten nested object to dot-notation paths
  */

dataLayer/export.js

@@ -101,17 +101,17 @@ async function confirmPath(directoryPath) {
 	} catch (err) {
 		let errorMessage;
 		if (err.code === 'ENOENT') {
-			errorMessage = `path '${directory_path}' does not exist`;
+			errorMessage = `path '${directoryPath}' does not exist`;
 		} else if (err.code === 'EACCES') {
-			errorMessage = `access to path '${directory_path}' is denied`;
+			errorMessage = `access to path '${directoryPath}' is denied`;
 		} else {
 			errorMessage = err.message;
 		}
 		hdbLogger.error(errorMessage);
 		throw handleHDBError(new Error(), errorMessage, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
 	}
 	if (!stats.isDirectory()) {
-		let err = `path '${directory_path}' is not a directory, please supply a valid folder path`;
+		let err = `path '${directoryPath}' is not a directory, please supply a valid folder path`;
 		hdbLogger.error(err);
 		throw handleHDBError(new Error(), err, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
 	}

dataLayer/getBackup.js

@@ -5,7 +5,6 @@ const harperBridge = require('./harperBridge/harperBridge.js');
 const GetBackupObject = require('./GetBackupObject.js');
 const hdbUtils = require('../utility/common_utils.js');
 const hdbTerms = require('../utility/hdbTerms.ts');
-const envMgr = require('../utility/environment/environmentManager.js');
 const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.js');
 const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
 
@@ -37,5 +36,5 @@ async function getBackup(getBackupObject) {
 		);
 	}
 
-	return await harperBridge.getBackup(readAuditLogObject);
+	return harperBridge.getBackup(getBackupObject);
 }

dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js

@@ -80,7 +80,7 @@ async function lmdbCreateAttribute(createAttributeObj) {
 		);
 		if (env.dbis[createAttributeObj.attribute] !== undefined) {
 			throw new Error(
-				`attribute '${create_attribute_obj.attribute}' already exists in ${createAttributeObj.schema}.${createAttributeObj.table}`
+				`attribute '${createAttributeObj.attribute}' already exists in ${createAttributeObj.schema}.${createAttributeObj.table}`
 			);
 		}
 		environmentUtility.createDBI(
@@ -97,7 +97,11 @@ async function lmdbCreateAttribute(createAttributeObj) {
 
 		let { written_hashes, skipped_hashes } = await writeUtility.insertRecords(
 			hdbAttributeEnv,
+			// I'm not sure what else to do with these for now, but I do want to eslint to check the rest of the codebase
+			// for undefined vars. - WSM 2025-11-26
+			// eslint-disable-next-line no-undef
 			HDB_TABLE_INFO.hash_attribute,
+			// eslint-disable-next-line no-undef
 			hdbAttributeAttributes,
 			[record]
 		);

dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js

@@ -13,7 +13,7 @@ const createTxnEnvironments = require('../lmdbUtility/lmdbCreateTransactionsAudi
 module.exports = lmdbCreateTable;
 
 /**
- * Writes new table data to the system tables creates the enivronment file and creates two datastores to track created and updated
+ * Writes new table data to the system tables creates the environment file and creates two datastores to track created and updated
  * timestamps for new table data.
  * @param tableSystemData
  * @param tableCreateObj
@@ -55,9 +55,16 @@ async function lmdbCreateTable(tableSystemData, tableCreateObj) {
 			);
 
 			//add the meta data to system.hdb_table
-			await writeUtility.insertRecords(hdbTableEnv, HDB_TABLE_INFO.hash_attribute, hdbTableAttributes, [
-				tableSystemData,
-			]);
+			await writeUtility.insertRecords(
+				hdbTableEnv,
+				// I'm not sure what else to do with these for now, but I do want to eslint to check the rest of the codebase
+				// for undefined vars. - WSM 2025-11-26
+				// eslint-disable-next-line no-undef
+				HDB_TABLE_INFO.hash_attribute,
+				// eslint-disable-next-line no-undef
+				hdbTableAttributes,
+				[tableSystemData]
+			);
 			//create attributes for hash attribute created/updated time stamps
 			createdTimeAttr.skip_table_check = true;
 			updatedTimeAttr.skip_table_check = true;