diff --git a/functions/src/clean-temp.ts b/functions/src/clean-temp.ts new file mode 100644 index 000000000..0e177458f --- /dev/null +++ b/functions/src/clean-temp.ts @@ -0,0 +1,36 @@ +/** + * Copyright 2026 The Ground Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getStorageBucket } from './common/context'; +import { TEMP_MAX_AGE_MS, TEMP_PREFIX } from './common/temp-storage'; + +/** + * Deletes temporary files older than MAX_AGE_MS from the temp/ prefix in the + * default storage bucket. + */ +export async function cleanTempHandler() { + const bucket = getStorageBucket(); + const [files] = await bucket.getFiles({ prefix: TEMP_PREFIX }); + const cutoff = Date.now() - TEMP_MAX_AGE_MS; + const deletions = files + .filter(f => { + const updated = f.metadata?.updated; + return updated && new Date(updated).getTime() < cutoff; + }) + .map(f => f.delete()); + await Promise.all(deletions); + console.log(`Deleted ${deletions.length} expired temp file(s).`); +} diff --git a/functions/src/common/context.ts b/functions/src/common/context.ts index ac99f3a3b..df613490c 100644 --- a/functions/src/common/context.ts +++ b/functions/src/common/context.ts @@ -18,6 +18,8 @@ import { Datastore } from './datastore'; import { MailService } from './mail-service'; import { getApp, initializeApp } from 'firebase-admin/app'; import { getFirestore } from 'firebase-admin/firestore'; +import { getStorage } from 'firebase-admin/storage'; +import { randomUUID } from 'crypto'; let datastore: Datastore | undefined; let mailService: MailService | undefined; @@ -49,3 +51,26 @@ export async function getMailService(): Promise { export function resetDatastore() { datastore = undefined; } + +export function getStorageBucket() { + initializeFirebaseApp(); + return getStorage().bucket(); +} + +/** + * Sets a Firebase Storage download token on the given file and returns a + * download URL that does not require IAM signing permissions. + */ +export async function getFirebaseDownloadUrl(file: { + name: string; + bucket: { name: string }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + setMetadata: (metadata: any) => Promise; +}): Promise { + const token = randomUUID(); + await file.setMetadata({ + metadata: { firebaseStorageDownloadTokens: token }, + }); + const encoded = encodeURIComponent(file.name); + return `https://firebasestorage.googleapis.com/v0/b/${file.bucket.name}/o/${encoded}?alt=media&token=${token}`; +} diff --git a/functions/src/common/temp-storage.ts b/functions/src/common/temp-storage.ts new file mode 100644 index 000000000..af040ec3f --- /dev/null +++ b/functions/src/common/temp-storage.ts @@ -0,0 +1,22 @@ +/** + * Copyright 2026 The Ground Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const TEMP_PREFIX = 'temp/'; +export const TEMP_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour + +export function getTempFilePath(userId: string, filename: string): string { + return `${TEMP_PREFIX}${userId}/${filename}`; +} diff --git a/functions/src/export-csv.spec.ts b/functions/src/export-csv.spec.ts index da5e45552..5a603041e 100644 --- a/functions/src/export-csv.spec.ts +++ b/functions/src/export-csv.spec.ts @@ -23,9 +23,10 @@ import { createResponseSpy, } from './testing/http-test-helpers'; import { DecodedIdToken } from 'firebase-admin/auth'; -import { StatusCodes } from 'http-status-codes'; import { SURVEY_ORGANIZER_ROLE } from './common/auth'; import { getDatastore, resetDatastore } from './common/context'; +import * as context from './common/context'; +import { PassThrough } from 'stream'; import { Firestore, QueryDocumentSnapshot } from 'firebase-admin/firestore'; import { exportCsvHandler } from './export-csv'; import { registry } from '@ground/lib'; @@ -94,6 +95,10 @@ async function* fetchLoisSubmissionsFromMock( describe('exportCsv()', () => { let mockFirestore: Firestore; + let storageChunks: string[]; + let mockFile: jasmine.SpyObj; + const FIREBASE_DOWNLOAD_URL_PREFIX = + 'https://firebasestorage.googleapis.com/v0/b/test-bucket/o/'; const email = 'somebody@test.it'; const userId = 'user5000'; const survey = { @@ -342,6 +347,26 @@ describe('exportCsv()', () => { beforeEach(() => { mockFirestore = createMockFirestore(); stubAdminApi(mockFirestore); + storageChunks = []; + const writeStream = new PassThrough(); + writeStream.on('data', (chunk: Buffer) => + storageChunks.push(chunk.toString()) + ); + mockFile = jasmine.createSpyObj('file', [ + 'createWriteStream', + 'setMetadata', + ]); + mockFile.createWriteStream.and.returnValue(writeStream); + mockFile.setMetadata.and.resolveTo([{}]); + Object.defineProperty(mockFile, 'name', { + value: 'temp/user5000/job.csv', + }); + Object.defineProperty(mockFile, 'bucket', { + value: { name: 'test-bucket' }, + }); + const mockBucket = jasmine.createSpyObj('bucket', ['file']); + mockBucket.file.and.returnValue(mockFile); + spyOn(context, 'getStorageBucket').and.returnValue(mockBucket); spyOn(getDatastore(), 'fetchPartialLocationsOfInterest').and.callFake( (surveyId: string, jobId: string) => { const emptyQuery: any = { @@ -402,20 +427,25 @@ describe('exportCsv()', () => { job: jobId, }, }); - const chunks: string[] = []; - const res = createResponseSpy(chunks); + const res = createResponseSpy(); // Run export CSV handler. await exportCsvHandler(req, res, { email } as DecodedIdToken); // Check post-conditions. - expect(res.status).toHaveBeenCalledOnceWith(StatusCodes.OK); - expect(res.type).toHaveBeenCalledOnceWith('text/csv'); - expect(res.setHeader).toHaveBeenCalledOnceWith( - 'Content-Disposition', - `attachment; filename=${expectedFilename}` + expect(res.redirect as jasmine.Spy).toHaveBeenCalledTimes(1); + const redirectUrl: string = ( + res.redirect as jasmine.Spy + ).calls.mostRecent().args[0]; + expect(redirectUrl).toContain(FIREBASE_DOWNLOAD_URL_PREFIX); + expect(mockFile.createWriteStream).toHaveBeenCalledWith( + jasmine.objectContaining({ + metadata: jasmine.objectContaining({ + contentDisposition: `attachment; filename=${expectedFilename}`, + }), + }) ); - const output = chunks.join('').trim(); + const output = storageChunks.join('').trim(); const lines = output.split('\n'); expect(lines).toEqual(expectedCsv); }) diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts index d7e431118..0656af162 100644 --- a/functions/src/export-csv.ts +++ b/functions/src/export-csv.ts @@ -20,7 +20,12 @@ import * as csv from '@fast-csv/format'; import { canExport, hasOrganizerRole } from './common/auth'; import { isAccessibleLoi } from './common/utils'; import { geojsonToWKT } from '@terraformer/wkt'; -import { getDatastore } from './common/context'; +import { + getDatastore, + getFirebaseDownloadUrl, + getStorageBucket, +} from './common/context'; +import { getTempFilePath } from './common/temp-storage'; import { DecodedIdToken } from 'firebase-admin/auth'; import { QueryDocumentSnapshot } from 'firebase-admin/firestore'; import { StatusCodes } from 'http-status-codes'; @@ -103,11 +108,15 @@ export async function exportCsvHandler( const headers = getHeaders(tasks, loiProperties); - res.type('text/csv'); - res.setHeader( - 'Content-Disposition', - 'attachment; filename=' + getFileName(jobName) - ); + const fileName = getFileName(jobName); + const bucket = getStorageBucket(); + const file = bucket.file(getTempFilePath(userId, `${Date.now()}.csv`)); + const writeStream = file.createWriteStream({ + metadata: { + contentType: 'text/csv', + contentDisposition: `attachment; filename=${fileName}`, + }, + }); const csvStream = csv.format({ delimiter: ',', @@ -116,7 +125,7 @@ export async function exportCsvHandler( includeEndRowDelimiter: true, // Add \n to last row in CSV quote: false, }); - csvStream.pipe(res); + csvStream.pipe(writeStream); const rows = await db.fetchLoisSubmissions( surveyId, @@ -142,8 +151,14 @@ export async function exportCsvHandler( } } - res.status(StatusCodes.OK); csvStream.end(); + + await new Promise((resolve, reject) => { + writeStream.on('finish', resolve); + writeStream.on('error', reject); + }); + + res.redirect(await getFirebaseDownloadUrl(file)); } function getHeaders(tasks: Pb.ITask[], loiProperties: Set): string[] { diff --git a/functions/src/export-geojson.spec.ts b/functions/src/export-geojson.spec.ts index 607c93e70..6520f0426 100644 --- a/functions/src/export-geojson.spec.ts +++ b/functions/src/export-geojson.spec.ts @@ -23,9 +23,10 @@ import { createResponseSpy, } from './testing/http-test-helpers'; import { DecodedIdToken } from 'firebase-admin/auth'; -import { StatusCodes } from 'http-status-codes'; import { DATA_COLLECTOR_ROLE } from './common/auth'; import { resetDatastore } from './common/context'; +import * as context from './common/context'; +import { PassThrough } from 'stream'; import { Firestore } from 'firebase-admin/firestore'; import { exportGeojsonHandler } from './export-geojson'; import { registry } from '@ground/lib'; @@ -45,6 +46,10 @@ const op = registry.getFieldIds(Pb.Task.MultipleChoiceQuestion.Option); describe('export()', () => { let mockFirestore: Firestore; + let storageChunks: string[]; + let mockFile: jasmine.SpyObj; + const FIREBASE_DOWNLOAD_URL_PREFIX = + 'https://firebasestorage.googleapis.com/v0/b/test-bucket/o/'; const email = 'somebody@test.it'; const userId = 'user5000'; const survey = { @@ -174,6 +179,26 @@ describe('export()', () => { beforeEach(() => { mockFirestore = createMockFirestore(); stubAdminApi(mockFirestore); + storageChunks = []; + const writeStream = new PassThrough(); + writeStream.on('data', (chunk: Buffer) => + storageChunks.push(chunk.toString()) + ); + mockFile = jasmine.createSpyObj('file', [ + 'createWriteStream', + 'setMetadata', + ]); + mockFile.createWriteStream.and.returnValue(writeStream); + mockFile.setMetadata.and.resolveTo([{}]); + Object.defineProperty(mockFile, 'name', { + value: 'temp/user5000/job.geojson', + }); + Object.defineProperty(mockFile, 'bucket', { + value: { name: 'test-bucket' }, + }); + const mockBucket = jasmine.createSpyObj('bucket', ['file']); + mockBucket.file.and.returnValue(mockFile); + spyOn(context, 'getStorageBucket').and.returnValue(mockBucket); }); afterEach(() => { @@ -200,8 +225,7 @@ describe('export()', () => { job: jobId, }, }); - const chunks: string[] = []; - const res = createResponseSpy(chunks); + const res = createResponseSpy(); // Run export handler. await exportGeojsonHandler(req, res, { @@ -210,13 +234,19 @@ describe('export()', () => { } as DecodedIdToken); // Check post-conditions. - expect(res.status).toHaveBeenCalledOnceWith(StatusCodes.OK); - expect(res.type).toHaveBeenCalledOnceWith('application/json'); - expect(res.setHeader).toHaveBeenCalledOnceWith( - 'Content-Disposition', - `attachment; filename=${expectedFilename}` + expect(res.redirect as jasmine.Spy).toHaveBeenCalledTimes(1); + const redirectUrl: string = ( + res.redirect as jasmine.Spy + ).calls.mostRecent().args[0]; + expect(redirectUrl).toContain(FIREBASE_DOWNLOAD_URL_PREFIX); + expect(mockFile.createWriteStream).toHaveBeenCalledWith( + jasmine.objectContaining({ + metadata: jasmine.objectContaining({ + contentDisposition: `attachment; filename=${expectedFilename}`, + }), + }) ); - const output = JSON.parse(chunks.join('')); + const output = JSON.parse(storageChunks.join('')); expect(output).toEqual(expectedGeojson); expect(JSON.stringify(output)).toEqual(JSON.stringify(expectedGeojson)); }) diff --git a/functions/src/export-geojson.ts b/functions/src/export-geojson.ts index 063858c35..0ac8643d2 100644 --- a/functions/src/export-geojson.ts +++ b/functions/src/export-geojson.ts @@ -17,7 +17,12 @@ import { Request } from 'firebase-functions/v2/https'; import type { Response } from 'express'; import { canExport, hasOrganizerRole } from './common/auth'; -import { getDatastore } from './common/context'; +import { + getDatastore, + getFirebaseDownloadUrl, + getStorageBucket, +} from './common/context'; +import { getTempFilePath } from './common/temp-storage'; import { isAccessibleLoi } from './common/utils'; import { DecodedIdToken } from 'firebase-admin/auth'; import { StatusCodes } from 'http-status-codes'; @@ -79,15 +84,18 @@ export async function exportGeojsonHandler( const ownerIdFilter = canViewAll ? null : userId; - res.type('application/json'); - res.setHeader( - 'Content-Disposition', - 'attachment; filename=' + getFileName(jobName) - ); - res.status(StatusCodes.OK); + const fileName = getFileName(jobName); + const bucket = getStorageBucket(); + const file = bucket.file(getTempFilePath(userId, `${Date.now()}.geojson`)); + const writeStream = file.createWriteStream({ + metadata: { + contentType: 'application/json', + contentDisposition: `attachment; filename=${fileName}`, + }, + }); // Write opening of FeatureCollection manually - res.write('{\n "type": "FeatureCollection",\n "features": [\n'); + writeStream.write('{\n "type": "FeatureCollection",\n "features": [\n'); // Fetch all locations of interest const rows = await db.fetchLocationsOfInterest(surveyId, jobId); @@ -103,13 +111,13 @@ export async function exportGeojsonHandler( // Manually write the separator comma before each feature except the first one. if (!first) { - res.write(',\n'); + writeStream.write(',\n'); } else { first = false; } // Use JSON.stringify to convert the feature object to a string and write it. - res.write(JSON.stringify(feature, null, 2)); + writeStream.write(JSON.stringify(feature, null, 2)); } } catch (e) { console.debug('Skipping row', e); @@ -117,8 +125,15 @@ export async function exportGeojsonHandler( } // Close the FeatureCollection after the loop completes. - res.write('\n ]\n}'); - res.end(); + writeStream.write('\n ]\n}'); + writeStream.end(); + + await new Promise((resolve, reject) => { + writeStream.on('finish', resolve); + writeStream.on('error', reject); + }); + + res.redirect(await getFirebaseDownloadUrl(file)); } function buildFeature(loi: Pb.LocationOfInterest) { diff --git a/functions/src/index.ts b/functions/src/index.ts index 75b966178..0e7595bfd 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -15,6 +15,7 @@ */ import 'module-alias/register'; +import { onSchedule } from 'firebase-functions/scheduler'; import { onDocumentCreated, onDocumentWritten, @@ -25,6 +26,7 @@ import { sessionLoginHandler } from './session-login'; import { importGeoJsonCallback } from './import-geojson'; import { exportCsvHandler } from './export-csv'; import { exportGeojsonHandler } from './export-geojson'; +import { cleanTempHandler } from './clean-temp'; import { onCall } from 'firebase-functions/v2/https'; import { onCreateLoiHandler } from './on-create-loi'; import { onCreatePasslistEntryHandler } from './on-create-passlist-entry'; @@ -106,3 +108,5 @@ export const onWriteSurvey = onDocumentWritten( ); export const sessionLogin = onHttpsRequest(sessionLoginHandler); + +export const cleanTemp = onSchedule('every 1 hours', cleanTempHandler); diff --git a/functions/src/testing/http-test-helpers.ts b/functions/src/testing/http-test-helpers.ts index b9a840f1f..f19b4cdc3 100644 --- a/functions/src/testing/http-test-helpers.ts +++ b/functions/src/testing/http-test-helpers.ts @@ -48,6 +48,7 @@ export function createResponseSpy(chunks?: string[]): Response { 'write', 'type', 'setHeader', + 'redirect', 'on', 'once', 'emit', diff --git a/storage/storage.rules b/storage/storage.rules index 8437b36a6..161d57153 100644 --- a/storage/storage.rules +++ b/storage/storage.rules @@ -1,19 +1,19 @@ /** * Copyright 2024 The Ground Authors. * - * Licensed under the Apache License, Version 2.0 (the 'License'); + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, + * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + rules_version = '2'; service firebase.storage { match /b/{bucket}/o { @@ -74,6 +74,11 @@ service firebase.storage { ])); } + match /temp/{userId}/{allPaths=**} { + // Only the owning user can read their own temporary files. + allow read: if isSignedIn() && request.auth.uid == userId; + } + match /offline-imagery/{allPaths=**} { // All authenticated users can read. allow read: if isSignedIn();