diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 4046c36..0e92657 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -68,6 +68,12 @@ }, "tilesStorageProvider": "TILES_STORAGE_PROVIDER", "gpkgStorageProvider": "GPKG_STORAGE_PROVIDER", + "storage": { + "internalPvc": { + "mountPath": "INTERNAL_PVC_MOUNT_PATH", + "gpkgSubPath": "GPKG_SUBPATH" + } + }, "ingestionSourcesDirPath": "INGESTION_SOURCES_DIR_PATH", "servicesUrl": { "mapproxyApi": "MAPPROXY_API_URL", @@ -185,8 +191,7 @@ "cleanupExpirationDays": { "__name": "EXPORT_CLEANUP_EXPIRATION_DAYS", "__format": "number" - }, - "gpkgsRootDir": "EXPORT_GPKGS_ROOT_DIR" + } } }, "tasks": { diff --git a/config/default.json b/config/default.json index 0a531e7..9510b2a 100644 --- a/config/default.json +++ b/config/default.json @@ -45,6 +45,12 @@ "disableHttpClientLogs": true, "tilesStorageProvider": "FS", "gpkgStorageProvider": "FS", + "storage": { + "internalPvc": { + "mountPath": "", + "gpkgSubPath": "" + } + }, "ingestionSourcesDirPath": "", "linkTemplatesPath": "config/linkTemplates.template", "servicesUrl": { @@ -121,8 +127,7 @@ "pollingJobs": { "export": { "type": "Export", - "cleanupExpirationDays": 14, - "gpkgsRootDir": "gpkgs" + "cleanupExpirationDays": 14 } }, "tasks": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 89ab083..f415ed5 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -54,7 +54,8 @@ data: INGESTION_SEED_JOB_TYPE : {{ $jobDefinitions.jobs.seed.type | quote }} EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }} EXPORT_CLEANUP_EXPIRATION_DAYS: {{ $jobDefinitions.jobs.export.cleanupExpirationDays | quote }} - EXPORT_GPKGS_ROOT_DIR: {{ $jobDefinitions.jobs.export.gpkgsRootDir | quote }} + INTERNAL_PVC_MOUNT_PATH: {{ $storage.fs.internalPvc.mountPath | quote }} + GPKG_SUBPATH: {{ $storage.fs.internalPvc.gpkgSubPath | quote }} TILES_MERGING_TASK_TYPE: {{ $jobDefinitions.tasks.merge.type | quote }} TILES_MERGING_TILE_BATCH_SIZE: {{ $jobDefinitions.tasks.merge.tileBatchSize | quote }} TILES_MERGING_TASK_BATCH_SIZE: {{ $jobDefinitions.tasks.merge.taskBatchSize | quote }} diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 366e134..1dec834 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -12,8 +12,6 @@ {{- $samePvc := and $storage.fs.ingestionSourcePvc.enabled $storage.fs.internalPvc.enabled (eq $storage.fs.internalPvc.name $storage.fs.ingestionSourcePvc.name) }} {{- $internalVolumeName := ternary "ingestion-storage" "internal-storage" $samePvc }} -{{ $gpkgPath := (printf "%s/%s" $storage.fs.internalPvc.mountPath $storage.fs.internalPvc.gpkgSubPath) }} - {{- if .Values.enabled -}} apiVersion: apps/v1 kind: Deployment @@ -89,8 +87,6 @@ spec: env: - name: SERVER_PORT value: {{ .Values.env.targetPort | quote }} - - name: GPKGS_LOCATION - value: {{ $gpkgPath }} {{- if .Values.global.ca.secretName }} - name: REQUESTS_CA_BUNDLE value: {{ printf "%s/%s" .Values.global.ca.path .Values.global.ca.key | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index ea7d2d8..79cbb64 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -141,7 +141,6 @@ jobDefinitions: export: type: "" cleanupExpirationDays: 14 - gpkgsRootDir: "gpkgs" tasks: createTasks: type: "" diff --git a/src/job/models/export/exportJobHandler.ts b/src/job/models/export/exportJobHandler.ts index 5c3c329..751bb46 100644 --- a/src/job/models/export/exportJobHandler.ts +++ b/src/job/models/export/exportJobHandler.ts @@ -47,15 +47,13 @@ import { CallbackClient } from '../../../httpClients/callbackClient'; import { JobTrackerClient } from '../../../httpClients/jobTrackerClient'; import { PolygonPartsMangerClient } from '../../../httpClients/polygonPartsMangerClient'; import { convertObjectKeysToSnakeCase } from '../../../utils/db/dbUtils'; -import { buildUrl } from '../../../utils/url'; +import { ArtifactPathBuilder } from '../../../utils/storage/artifactPathBuilder'; @injectable() export class ExportJobHandler extends JobHandler implements IJobHandler { private readonly exportTaskType: string; - private readonly gpkgsRootDir: string; private readonly isS3GpkgProvider: boolean; private readonly cleanupExpirationDays: number; - private readonly downloadServerUrl: string; public constructor( @inject(SERVICES.LOGGER) logger: Logger, @inject(SERVICES.CONFIG) config: IConfig, @@ -68,16 +66,15 @@ export class ExportJobHandler extends JobHandler implements IJobHandler('jobManagement.export.tasks.tilesExporting.type'); - this.gpkgsRootDir = config.get('jobManagement.export.pollingJobs.export.gpkgsRootDir'); // eslint-disable-next-line @typescript-eslint/naming-convention const gpkgProvider = config.get('gpkgStorageProvider'); this.isS3GpkgProvider = gpkgProvider === StorageProvider.S3; this.cleanupExpirationDays = config.get('jobManagement.export.pollingJobs.export.cleanupExpirationDays'); - this.downloadServerUrl = config.get('servicesUrl.downloadServerPublicDNS'); } public async handleJobInit(job: ExportJob, task: ExportInitTask): Promise { await context.with(trace.setSpan(context.active(), this.tracer.startSpan(`${ExportJobHandler.name}.${this.handleJobInit.name}`)), async () => { @@ -171,7 +168,7 @@ export class ExportJobHandler extends JobHandler implements IJobHandler { - const { gpkgFilePath } = paths; - const gpkgS3Key = path.posix.join(this.gpkgsRootDir, paths.gpkgRelativePath); - const jsonFilePath = gpkgFilePath.replace(/\.gpkg$/, '.json'); //TODO: In future, we will remove the json metadata file and support only gpkg - const jsonS3Key = gpkgS3Key.replace(/\.gpkg$/, '.json'); //TODO: In future, we will remove the json metadata file and support only gpkg + const { gpkgFilePath, gpkgRelativePath } = paths; + const gpkgS3Key = this.pathBuilder.gpkgS3Key(gpkgRelativePath); + const jsonFilePath = this.pathBuilder.jsonLocalPath(gpkgRelativePath); //TODO: In future, we will remove the json metadata file and support only gpkg + const jsonS3Key = this.pathBuilder.jsonS3Key(gpkgRelativePath); //TODO: In future, we will remove the json metadata file and support only gpkg await this.s3Service.uploadFiles([ { filePath: gpkgFilePath, s3Key: gpkgS3Key, contentType: GPKG_CONTENT_TYPE }, @@ -444,8 +441,8 @@ export class ExportJobHandler extends JobHandler implements IJobHandler('storage.internalPvc.mountPath'); + this.gpkgSubPath = config.get('storage.internalPvc.gpkgSubPath'); + this.downloadServerUrl = config.get('servicesUrl.downloadServerPublicDNS'); + this.gpkgPrefix = path.posix.basename(this.gpkgSubPath); + } + + //GPKGs + public gpkgLocalPath(rel: string): string { + return path.join(this.internalMountPath, this.gpkgSubPath, rel); + } + + public gpkgS3Key(rel: string): string { + return path.posix.join(this.gpkgPrefix, rel); + } + + public gpkgDownloadUrl(rel: string): string { + return buildUrl(this.downloadServerUrl, this.gpkgPrefix, rel); + } + + //JSONs(metadata) + public jsonLocalPath(rel: string): string { + return this.gpkgLocalPath(rel).replace(ARTIFACTS_EXTENSION.GPKG, ARTIFACTS_EXTENSION.JSON); + } + + public jsonS3Key(rel: string): string { + return this.gpkgS3Key(rel).replace(ARTIFACTS_EXTENSION.GPKG, ARTIFACTS_EXTENSION.JSON); + } + + public jsonDownloadUrl(rel: string): string { + return this.gpkgDownloadUrl(rel).replace(ARTIFACTS_EXTENSION.GPKG, ARTIFACTS_EXTENSION.JSON); + } +} diff --git a/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts b/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts index 2561f27..3146547 100644 --- a/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts +++ b/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts @@ -112,15 +112,14 @@ describe('ExportJobHandler', () => { }); describe('handleJobFinalize', () => { - const gpkgsRootDir = 'gpkgs'; + const mountPath = '/outputs'; + const gpkgSubPath = 'raster/artifacts/gpkgs'; const gpkgRelativePath = 'package.gpkg'; - const gpkgFilePath = path.join('/', gpkgsRootDir, gpkgRelativePath); + const gpkgFilePath = path.join(mountPath, gpkgSubPath, gpkgRelativePath); const jsonFilePath = gpkgFilePath.replace('.gpkg', '.json'); const gpkgDirPath = '/path/to/gpkgs'; - let joinSpy: jest.SpyInstance; let dirnameSpy: jest.SpyInstance; beforeEach(() => { - joinSpy = jest.spyOn(path, 'join').mockReturnValue(gpkgFilePath); dirnameSpy = jest.spyOn(path, 'dirname').mockReturnValue(gpkgDirPath); }); @@ -165,7 +164,6 @@ describe('ExportJobHandler', () => { await exportJobHandler.handleJobFinalize(job, task); // Verify path methods were called correctly - expect(joinSpy).toHaveBeenCalledWith('/', gpkgsRootDir, gpkgRelativePath); expect(dirnameSpy).toHaveBeenCalledWith(gpkgFilePath); // Verify metadata processing @@ -257,8 +255,6 @@ describe('ExportJobHandler', () => { await exportJobHandler.handleJobFinalize(job, task); - expect(joinSpy).toHaveBeenCalledWith('/', gpkgsRootDir, gpkgRelativePath); - expect(s3ServiceMock.uploadFiles).toHaveBeenCalledWith([ { filePath: gpkgFilePath, diff --git a/tests/unit/job/exportJobHandler/exportJobHandlerSetup.ts b/tests/unit/job/exportJobHandler/exportJobHandlerSetup.ts index 511091e..af4c388 100644 --- a/tests/unit/job/exportJobHandler/exportJobHandlerSetup.ts +++ b/tests/unit/job/exportJobHandler/exportJobHandlerSetup.ts @@ -13,6 +13,7 @@ import { FSService } from '../../../../src/utils/storage/fsService'; import { CallbackClient } from '../../../../src/httpClients/callbackClient'; import { JobTrackerClient } from '../../../../src/httpClients/jobTrackerClient'; import { PolygonPartsMangerClient } from '../../../../src/httpClients/polygonPartsMangerClient'; +import { ArtifactPathBuilder } from '../../../../src/utils/storage/artifactPathBuilder'; export interface ExportJobHandlerTestContext { configMock: IConfig; @@ -57,6 +58,8 @@ export const setupExportJobHandlerTest = (): ExportJobHandlerTestContext => { getAggregatedLayerMetadata: jest.fn(), } as unknown as jest.Mocked; + const pathBuilder = new ArtifactPathBuilder(configMock); + const exportJobHandler = new ExportJobHandler( jsLogger({ enabled: false }), configMock, @@ -69,7 +72,8 @@ export const setupExportJobHandlerTest = (): ExportJobHandlerTestContext => { fsServiceMock, callbackClientMock, taskMetricsMock, - polygonPartsManagerClientMock + polygonPartsManagerClientMock, + pathBuilder ); return { diff --git a/tests/unit/mocks/configMock.ts b/tests/unit/mocks/configMock.ts index de4d56a..cffc738 100644 --- a/tests/unit/mocks/configMock.ts +++ b/tests/unit/mocks/configMock.ts @@ -99,6 +99,12 @@ const registerDefaultConfig = (): void => { ingestionSourcesDirPath: '/layerSources', tilesStorageProvider: 'FS', gpkgStorageProvider: 'FS', + storage: { + internalPvc: { + mountPath: '/outputs', + gpkgSubPath: 'raster/artifacts/gpkgs', + }, + }, disableHttpClientLogs: true, linkTemplatesPath: 'config/linkTemplates.template', servicesUrl: { @@ -170,7 +176,6 @@ const registerDefaultConfig = (): void => { pollingJobs: { export: { type: 'Export', - gpkgsRootDir: 'gpkgs', }, }, tasks: { diff --git a/tests/unit/utils/artifactPathBuilder.spec.ts b/tests/unit/utils/artifactPathBuilder.spec.ts new file mode 100644 index 0000000..d428b05 --- /dev/null +++ b/tests/unit/utils/artifactPathBuilder.spec.ts @@ -0,0 +1,59 @@ +import path from 'path'; +import { faker } from '@faker-js/faker'; +import { ArtifactPathBuilder, ARTIFACTS_EXTENSION } from '../../../src/utils/storage/artifactPathBuilder'; +import { configMock, registerDefaultConfig } from '../mocks/configMock'; + +describe('ArtifactPathBuilder', () => { + let internalMountPath: string; + let gpkgSubPath: string; + let downloadServerPublicDNS: string; + let gpkgPrefix: string; + + beforeEach(() => { + registerDefaultConfig(); + internalMountPath = configMock.get('storage.internalPvc.mountPath'); + gpkgSubPath = configMock.get('storage.internalPvc.gpkgSubPath'); + downloadServerPublicDNS = configMock.get('servicesUrl.downloadServerPublicDNS'); + gpkgPrefix = path.posix.basename(gpkgSubPath); + }); + + const rel = `${faker.string.uuid()}/package.gpkg`; + + describe('gpkg paths', () => { + it('builds the local FS path under mountPath + gpkgSubPath', () => { + const builder = new ArtifactPathBuilder(configMock); + expect(builder.gpkgLocalPath(rel)).toBe(`${internalMountPath}/${gpkgSubPath}/${rel}`); + }); + + it('builds the S3 key relative to the artifacts anchor', () => { + const builder = new ArtifactPathBuilder(configMock); + expect(builder.gpkgS3Key(rel)).toBe(`${gpkgPrefix}/${rel}`); + }); + + it('builds the public download URL', () => { + const builder = new ArtifactPathBuilder(configMock); + expect(builder.gpkgDownloadUrl(rel)).toBe(`${downloadServerPublicDNS}/${gpkgPrefix}/${rel}`); + }); + }); + + describe('json sidecar paths', () => { + it('swaps .gpkg → .json in the local path', () => { + const builder = new ArtifactPathBuilder(configMock); + expect(builder.jsonLocalPath(rel)).toBe( + `${internalMountPath}/${gpkgSubPath}/${rel.replace(ARTIFACTS_EXTENSION.GPKG, ARTIFACTS_EXTENSION.JSON)}` + ); + }); + + it('swaps .gpkg → .json in the S3 key', () => { + const builder = new ArtifactPathBuilder(configMock); + expect(builder.jsonS3Key(rel)).toBe(`${gpkgPrefix}/${rel.replace(ARTIFACTS_EXTENSION.GPKG, ARTIFACTS_EXTENSION.JSON)}`); + }); + + it('swaps .gpkg → .json in the download URL', () => { + const builder = new ArtifactPathBuilder(configMock); + expect(builder.jsonDownloadUrl(rel)).toBe( + `${downloadServerPublicDNS}/${gpkgPrefix}/${rel.replace(ARTIFACTS_EXTENSION.GPKG, ARTIFACTS_EXTENSION.JSON)}` + ); + }); + }); +});