From b27ec09127e2afac2d951256449f01172d023f20 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 6 Apr 2024 18:22:12 -0400 Subject: [PATCH 1/5] config type safety --- server/src/entities/system-config.entity.ts | 225 +++++++++++--------- 1 file changed, 121 insertions(+), 104 deletions(-) diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts index 9b03eb93f3e3d..d034ed942fc91 100644 --- a/server/src/entities/system-config.entity.ts +++ b/server/src/entities/system-config.entity.ts @@ -1,118 +1,135 @@ import { ConcurrentQueueName } from 'src/interfaces/job.interface'; import { Column, Entity, PrimaryColumn } from 'typeorm'; +export type SystemConfigValue = string | string[] | number | boolean; + +// https://stackoverflow.com/a/47058976 +// https://stackoverflow.com/a/70692231 +type PathsToStringProps = T extends SystemConfigValue + ? [] + : { + [K in keyof T]: [K, ...PathsToStringProps]; + }[keyof T]; + +type Join = T extends [] + ? never + : T extends [infer F] + ? F + : T extends [infer F, ...infer R] + ? F extends string + ? `${F}${D}${Join, D>}` + : never + : string; + @Entity('system_config') export class SystemConfigEntity { @PrimaryColumn() - key!: SystemConfigKey; + key!: (typeof SystemConfigKey)[keyof typeof SystemConfigKey]; @Column({ type: 'varchar', nullable: true, transformer: { to: JSON.stringify, from: JSON.parse } }) - value!: T | T[]; + value!: T; } -export type SystemConfigValue = string | number | boolean; - // dot notation matches path in `SystemConfig` -export enum SystemConfigKey { - FFMPEG_CRF = 'ffmpeg.crf', - FFMPEG_THREADS = 'ffmpeg.threads', - FFMPEG_PRESET = 'ffmpeg.preset', - FFMPEG_TARGET_VIDEO_CODEC = 'ffmpeg.targetVideoCodec', - FFMPEG_ACCEPTED_VIDEO_CODECS = 'ffmpeg.acceptedVideoCodecs', - FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec', - FFMPEG_ACCEPTED_AUDIO_CODECS = 'ffmpeg.acceptedAudioCodecs', - FFMPEG_TARGET_RESOLUTION = 'ffmpeg.targetResolution', - FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate', - FFMPEG_BFRAMES = 'ffmpeg.bframes', - FFMPEG_REFS = 'ffmpeg.refs', - FFMPEG_GOP_SIZE = 'ffmpeg.gopSize', - FFMPEG_NPL = 'ffmpeg.npl', - FFMPEG_TEMPORAL_AQ = 'ffmpeg.temporalAQ', - FFMPEG_CQ_MODE = 'ffmpeg.cqMode', - FFMPEG_TWO_PASS = 'ffmpeg.twoPass', - FFMPEG_PREFERRED_HW_DEVICE = 'ffmpeg.preferredHwDevice', - FFMPEG_TRANSCODE = 'ffmpeg.transcode', - FFMPEG_ACCEL = 'ffmpeg.accel', - FFMPEG_TONEMAP = 'ffmpeg.tonemap', - - JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency', - JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency', - JOB_VIDEO_CONVERSION_CONCURRENCY = 'job.videoConversion.concurrency', - JOB_FACE_DETECTION_CONCURRENCY = 'job.faceDetection.concurrency', - JOB_CLIP_ENCODING_CONCURRENCY = 'job.smartSearch.concurrency', - JOB_BACKGROUND_TASK_CONCURRENCY = 'job.backgroundTask.concurrency', - JOB_STORAGE_TEMPLATE_MIGRATION_CONCURRENCY = 'job.storageTemplateMigration.concurrency', - JOB_SEARCH_CONCURRENCY = 'job.search.concurrency', - JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency', - JOB_LIBRARY_CONCURRENCY = 'job.library.concurrency', - JOB_MIGRATION_CONCURRENCY = 'job.migration.concurrency', - - LIBRARY_SCAN_ENABLED = 'library.scan.enabled', - LIBRARY_SCAN_CRON_EXPRESSION = 'library.scan.cronExpression', - - LIBRARY_WATCH_ENABLED = 'library.watch.enabled', - - LOGGING_ENABLED = 'logging.enabled', - LOGGING_LEVEL = 'logging.level', - - MACHINE_LEARNING_ENABLED = 'machineLearning.enabled', - MACHINE_LEARNING_URL = 'machineLearning.url', - - MACHINE_LEARNING_CLIP_ENABLED = 'machineLearning.clip.enabled', - MACHINE_LEARNING_CLIP_MODEL_NAME = 'machineLearning.clip.modelName', - - MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED = 'machineLearning.facialRecognition.enabled', - MACHINE_LEARNING_FACIAL_RECOGNITION_MODEL_NAME = 'machineLearning.facialRecognition.modelName', - MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_SCORE = 'machineLearning.facialRecognition.minScore', - MACHINE_LEARNING_FACIAL_RECOGNITION_MAX_DISTANCE = 'machineLearning.facialRecognition.maxDistance', - MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES = 'machineLearning.facialRecognition.minFaces', - - MAP_ENABLED = 'map.enabled', - MAP_LIGHT_STYLE = 'map.lightStyle', - MAP_DARK_STYLE = 'map.darkStyle', - - REVERSE_GEOCODING_ENABLED = 'reverseGeocoding.enabled', - - NEW_VERSION_CHECK_ENABLED = 'newVersionCheck.enabled', - - OAUTH_AUTO_LAUNCH = 'oauth.autoLaunch', - OAUTH_AUTO_REGISTER = 'oauth.autoRegister', - OAUTH_BUTTON_TEXT = 'oauth.buttonText', - OAUTH_CLIENT_ID = 'oauth.clientId', - OAUTH_CLIENT_SECRET = 'oauth.clientSecret', - OAUTH_DEFAULT_STORAGE_QUOTA = 'oauth.defaultStorageQuota', - OAUTH_ENABLED = 'oauth.enabled', - OAUTH_ISSUER_URL = 'oauth.issuerUrl', - OAUTH_MOBILE_OVERRIDE_ENABLED = 'oauth.mobileOverrideEnabled', - OAUTH_MOBILE_REDIRECT_URI = 'oauth.mobileRedirectUri', - OAUTH_SCOPE = 'oauth.scope', - OAUTH_SIGNING_ALGORITHM = 'oauth.signingAlgorithm', - OAUTH_STORAGE_LABEL_CLAIM = 'oauth.storageLabelClaim', - OAUTH_STORAGE_QUOTA_CLAIM = 'oauth.storageQuotaClaim', - - PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled', - - SERVER_EXTERNAL_DOMAIN = 'server.externalDomain', - SERVER_LOGIN_PAGE_MESSAGE = 'server.loginPageMessage', - - STORAGE_TEMPLATE_ENABLED = 'storageTemplate.enabled', - STORAGE_TEMPLATE_HASH_VERIFICATION_ENABLED = 'storageTemplate.hashVerificationEnabled', - STORAGE_TEMPLATE = 'storageTemplate.template', - - IMAGE_THUMBNAIL_FORMAT = 'image.thumbnailFormat', - IMAGE_THUMBNAIL_SIZE = 'image.thumbnailSize', - IMAGE_PREVIEW_FORMAT = 'image.previewFormat', - IMAGE_PREVIEW_SIZE = 'image.previewSize', - IMAGE_QUALITY = 'image.quality', - IMAGE_COLORSPACE = 'image.colorspace', - - TRASH_ENABLED = 'trash.enabled', - TRASH_DAYS = 'trash.days', - - THEME_CUSTOM_CSS = 'theme.customCss', - - USER_DELETE_DELAY = 'user.deleteDelay', -} +export const SystemConfigKey: Record, '.'>> = { + FFMPEG_CRF: 'ffmpeg.crf', + FFMPEG_THREADS: 'ffmpeg.threads', + FFMPEG_PRESET: 'ffmpeg.preset', + FFMPEG_TARGET_VIDEO_CODEC: 'ffmpeg.targetVideoCodec', + FFMPEG_ACCEPTED_VIDEO_CODECS: 'ffmpeg.acceptedVideoCodecs', + FFMPEG_TARGET_AUDIO_CODEC: 'ffmpeg.targetAudioCodec', + FFMPEG_ACCEPTED_AUDIO_CODECS: 'ffmpeg.acceptedAudioCodecs', + FFMPEG_TARGET_RESOLUTION: 'ffmpeg.targetResolution', + FFMPEG_MAX_BITRATE: 'ffmpeg.maxBitrate', + FFMPEG_BFRAMES: 'ffmpeg.bframes', + FFMPEG_REFS: 'ffmpeg.refs', + FFMPEG_GOP_SIZE: 'ffmpeg.gopSize', + FFMPEG_NPL: 'ffmpeg.npl', + FFMPEG_TEMPORAL_AQ: 'ffmpeg.temporalAQ', + FFMPEG_CQ_MODE: 'ffmpeg.cqMode', + FFMPEG_TWO_PASS: 'ffmpeg.twoPass', + FFMPEG_PREFERRED_HW_DEVICE: 'ffmpeg.preferredHwDevice', + FFMPEG_TRANSCODE: 'ffmpeg.transcode', + FFMPEG_ACCEL: 'ffmpeg.accel', + FFMPEG_TONEMAP: 'ffmpeg.tonemap', + + JOB_THUMBNAIL_GENERATION_CONCURRENCY: 'job.thumbnailGeneration.concurrency', + JOB_METADATA_EXTRACTION_CONCURRENCY: 'job.metadataExtraction.concurrency', + JOB_VIDEO_CONVERSION_CONCURRENCY: 'job.videoConversion.concurrency', + JOB_FACE_DETECTION_CONCURRENCY: 'job.faceDetection.concurrency', + JOB_CLIP_ENCODING_CONCURRENCY: 'job.smartSearch.concurrency', + JOB_BACKGROUND_TASK_CONCURRENCY: 'job.backgroundTask.concurrency', + JOB_SEARCH_CONCURRENCY: 'job.search.concurrency', + JOB_SIDECAR_CONCURRENCY: 'job.sidecar.concurrency', + JOB_LIBRARY_CONCURRENCY: 'job.library.concurrency', + JOB_MIGRATION_CONCURRENCY: 'job.migration.concurrency', + + LIBRARY_SCAN_ENABLED: 'library.scan.enabled', + LIBRARY_SCAN_CRON_EXPRESSION: 'library.scan.cronExpression', + + LIBRARY_WATCH_ENABLED: 'library.watch.enabled', + + LOGGING_ENABLED: 'logging.enabled', + LOGGING_LEVEL: 'logging.level', + + MACHINE_LEARNING_ENABLED: 'machineLearning.enabled', + MACHINE_LEARNING_URL: 'machineLearning.url', + + MACHINE_LEARNING_CLIP_ENABLED: 'machineLearning.clip.enabled', + MACHINE_LEARNING_CLIP_MODEL_NAME: 'machineLearning.clip.modelName', + + MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED: 'machineLearning.facialRecognition.enabled', + MACHINE_LEARNING_FACIAL_RECOGNITION_MODEL_NAME: 'machineLearning.facialRecognition.modelName', + MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_SCORE: 'machineLearning.facialRecognition.minScore', + MACHINE_LEARNING_FACIAL_RECOGNITION_MAX_DISTANCE: 'machineLearning.facialRecognition.maxDistance', + MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES: 'machineLearning.facialRecognition.minFaces', + + MAP_ENABLED: 'map.enabled', + MAP_LIGHT_STYLE: 'map.lightStyle', + MAP_DARK_STYLE: 'map.darkStyle', + + REVERSE_GEOCODING_ENABLED: 'reverseGeocoding.enabled', + + NEW_VERSION_CHECK_ENABLED: 'newVersionCheck.enabled', + + OAUTH_AUTO_LAUNCH: 'oauth.autoLaunch', + OAUTH_AUTO_REGISTER: 'oauth.autoRegister', + OAUTH_BUTTON_TEXT: 'oauth.buttonText', + OAUTH_CLIENT_ID: 'oauth.clientId', + OAUTH_CLIENT_SECRET: 'oauth.clientSecret', + OAUTH_DEFAULT_STORAGE_QUOTA: 'oauth.defaultStorageQuota', + OAUTH_ENABLED: 'oauth.enabled', + OAUTH_ISSUER_URL: 'oauth.issuerUrl', + OAUTH_MOBILE_OVERRIDE_ENABLED: 'oauth.mobileOverrideEnabled', + OAUTH_MOBILE_REDIRECT_URI: 'oauth.mobileRedirectUri', + OAUTH_SCOPE: 'oauth.scope', + OAUTH_SIGNING_ALGORITHM: 'oauth.signingAlgorithm', + OAUTH_STORAGE_LABEL_CLAIM: 'oauth.storageLabelClaim', + OAUTH_STORAGE_QUOTA_CLAIM: 'oauth.storageQuotaClaim', + + PASSWORD_LOGIN_ENABLED: 'passwordLogin.enabled', + + SERVER_EXTERNAL_DOMAIN: 'server.externalDomain', + SERVER_LOGIN_PAGE_MESSAGE: 'server.loginPageMessage', + + STORAGE_TEMPLATE_ENABLED: 'storageTemplate.enabled', + STORAGE_TEMPLATE_HASH_VERIFICATION_ENABLED: 'storageTemplate.hashVerificationEnabled', + STORAGE_TEMPLATE: 'storageTemplate.template', + + IMAGE_THUMBNAIL_FORMAT: 'image.thumbnailFormat', + IMAGE_THUMBNAIL_SIZE: 'image.thumbnailSize', + IMAGE_PREVIEW_FORMAT: 'image.previewFormat', + IMAGE_PREVIEW_SIZE: 'image.previewSize', + IMAGE_QUALITY: 'image.quality', + IMAGE_COLORSPACE: 'image.colorspace', + + TRASH_ENABLED: 'trash.enabled', + TRASH_DAYS: 'trash.days', + + THEME_CUSTOM_CSS: 'theme.customCss', + + USER_DELETE_DELAY: 'user.deleteDelay', +} as const; export enum TranscodePolicy { ALL = 'all', From 1aa7207e1b6929676c4546af20fe1bd03832413f Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 6 Apr 2024 19:22:33 -0400 Subject: [PATCH 2/5] typeorm fix --- server/src/entities/system-config.entity.ts | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts index d034ed942fc91..33a11fd485bb4 100644 --- a/server/src/entities/system-config.entity.ts +++ b/server/src/entities/system-config.entity.ts @@ -1,11 +1,11 @@ import { ConcurrentQueueName } from 'src/interfaces/job.interface'; import { Column, Entity, PrimaryColumn } from 'typeorm'; -export type SystemConfigValue = string | string[] | number | boolean; +export type SystemConfigValue = string | number | boolean; // https://stackoverflow.com/a/47058976 // https://stackoverflow.com/a/70692231 -type PathsToStringProps = T extends SystemConfigValue +type PathsToStringProps = T extends SystemConfigValue | string[] ? [] : { [K in keyof T]: [K, ...PathsToStringProps]; @@ -21,15 +21,6 @@ type Join = T extends [] : never : string; -@Entity('system_config') -export class SystemConfigEntity { - @PrimaryColumn() - key!: (typeof SystemConfigKey)[keyof typeof SystemConfigKey]; - - @Column({ type: 'varchar', nullable: true, transformer: { to: JSON.stringify, from: JSON.parse } }) - value!: T; -} - // dot notation matches path in `SystemConfig` export const SystemConfigKey: Record, '.'>> = { FFMPEG_CRF: 'ffmpeg.crf', @@ -131,6 +122,15 @@ export const SystemConfigKey: Record { + @PrimaryColumn({ type: 'enum', enum: Object.values(SystemConfigKey), enumName: 'SystemConfigKey' }) + key!: (typeof SystemConfigKey)[keyof typeof SystemConfigKey]; + + @Column({ type: 'varchar', nullable: true, transformer: { to: JSON.stringify, from: JSON.parse } }) + value!: T; +} + export enum TranscodePolicy { ALL = 'all', OPTIMAL = 'optimal', From d960973c15ca5b71a949abce05f31412b5cc65f1 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 6 Apr 2024 20:00:31 -0400 Subject: [PATCH 3/5] typing fixes --- server/src/entities/system-config.entity.ts | 12 +++++++----- server/src/services/job.service.spec.ts | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts index 33a11fd485bb4..428ca9bd0dfcb 100644 --- a/server/src/entities/system-config.entity.ts +++ b/server/src/entities/system-config.entity.ts @@ -1,11 +1,11 @@ import { ConcurrentQueueName } from 'src/interfaces/job.interface'; import { Column, Entity, PrimaryColumn } from 'typeorm'; -export type SystemConfigValue = string | number | boolean; +export type SystemConfigValue = string | string[] | number | boolean; // https://stackoverflow.com/a/47058976 // https://stackoverflow.com/a/70692231 -type PathsToStringProps = T extends SystemConfigValue | string[] +type PathsToStringProps = T extends SystemConfigValue ? [] : { [K in keyof T]: [K, ...PathsToStringProps]; @@ -22,7 +22,7 @@ type Join = T extends [] : string; // dot notation matches path in `SystemConfig` -export const SystemConfigKey: Record, '.'>> = { +export const SystemConfigKey = { FFMPEG_CRF: 'ffmpeg.crf', FFMPEG_THREADS: 'ffmpeg.threads', FFMPEG_PRESET: 'ffmpeg.preset', @@ -120,12 +120,14 @@ export const SystemConfigKey: Record, '.'>>; + +export type SystemConfigKeyPaths = (typeof SystemConfigKey)[keyof typeof SystemConfigKey]; @Entity('system_config') export class SystemConfigEntity { @PrimaryColumn({ type: 'enum', enum: Object.values(SystemConfigKey), enumName: 'SystemConfigKey' }) - key!: (typeof SystemConfigKey)[keyof typeof SystemConfigKey]; + key!: SystemConfigKeyPaths; @Column({ type: 'varchar', nullable: true, transformer: { to: JSON.stringify, from: JSON.parse } }) value!: T; diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index ac0e502ae890e..c141586dbeacf 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -1,6 +1,6 @@ import { BadRequestException } from '@nestjs/common'; import { FeatureFlag, SystemConfigCore } from 'src/cores/system-config.core'; -import { SystemConfig, SystemConfigKey } from 'src/entities/system-config.entity'; +import { SystemConfig, SystemConfigKey, SystemConfigKeyPaths } from 'src/entities/system-config.entity'; import { IAssetRepository } from 'src/interfaces/asset.interface'; import { IEventRepository } from 'src/interfaces/event.interface'; import { @@ -360,7 +360,7 @@ describe(JobService.name, () => { }); } - const featureTests: Array<{ queue: QueueName; feature: FeatureFlag; configKey: SystemConfigKey }> = [ + const featureTests: Array<{ queue: QueueName; feature: FeatureFlag; configKey: SystemConfigKeyPaths }> = [ { queue: QueueName.SMART_SEARCH, feature: FeatureFlag.SMART_SEARCH, From 031c569179bec554f23832c50d1ec58c264fabd8 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 6 Apr 2024 20:26:39 -0400 Subject: [PATCH 4/5] don't use enum in db --- server/src/entities/system-config.entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts index 428ca9bd0dfcb..68a73e4311e1c 100644 --- a/server/src/entities/system-config.entity.ts +++ b/server/src/entities/system-config.entity.ts @@ -126,7 +126,7 @@ export type SystemConfigKeyPaths = (typeof SystemConfigKey)[keyof typeof SystemC @Entity('system_config') export class SystemConfigEntity { - @PrimaryColumn({ type: 'enum', enum: Object.values(SystemConfigKey), enumName: 'SystemConfigKey' }) + @PrimaryColumn({ type: 'varchar' }) key!: SystemConfigKeyPaths; @Column({ type: 'varchar', nullable: true, transformer: { to: JSON.stringify, from: JSON.parse } }) From 3597791dfefb5211c8f1c20f2dcb5a52d8b5015c Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 6 Apr 2024 21:42:14 -0400 Subject: [PATCH 5/5] add todo --- server/src/entities/system-config.entity.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/entities/system-config.entity.ts b/server/src/entities/system-config.entity.ts index 68a73e4311e1c..1ddd6baff39fc 100644 --- a/server/src/entities/system-config.entity.ts +++ b/server/src/entities/system-config.entity.ts @@ -22,6 +22,7 @@ type Join = T extends [] : string; // dot notation matches path in `SystemConfig` +// TODO: migrate to key value per section export const SystemConfigKey = { FFMPEG_CRF: 'ffmpeg.crf', FFMPEG_THREADS: 'ffmpeg.threads',