Skip to content

Commit

Permalink
feat: 🎨 utilise redirect url for getFile request
Browse files Browse the repository at this point in the history
  • Loading branch information
ignazio-bovo committed May 16, 2024
1 parent c80488d commit c269c10
Show file tree
Hide file tree
Showing 5 changed files with 762 additions and 783 deletions.
3 changes: 2 additions & 1 deletion storage-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
"bugs": "https://github.com/Joystream/joystream/issues",
"dependencies": {
"@apollo/client": "^3.3.21",
"@aws-sdk/client-s3": "^3.569.0",
"@aws-sdk/client-s3": "^3.577.0",
"@aws-sdk/s3-request-presigner": "^3.577.0",
"@elastic/ecs-winston-format": "^1.3.1",
"@joystream/metadata-protobuf": "^2.15.0",
"@joystream/opentelemetry": "1.0.0",
Expand Down
10 changes: 2 additions & 8 deletions storage-node/src/services/storageProviders/IConnectionHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ export interface IConnectionHandler {
/**
* Asynchronously retrieves a file from the remote bucket.
* @param filename - The key of the file to retrieve from the remote bucket.
* @returns A promise that resolves with the retrieved file data or rejects with an error.
* @returns A promise that resolves in the presigned URL of the file or rejects with an error, 1h expiry
*/
getFileFromRemoteBucket(filename: string): Promise<StorageProviderGetObjectResponse>
getRedirectUrlForObject(filename: string): Promise<string>

/**
* Asynchronously lists ALL files in the remote bucket, to be used during cache initialization only as it can be very slow.
Expand All @@ -27,9 +27,3 @@ export interface IConnectionHandler {
}

export type ColossusFileStream = Readable

export type StorageProviderGetObjectResponse = {
Body: ColossusFileStream
ContentType: string
ContentLength: number
}
37 changes: 5 additions & 32 deletions storage-node/src/services/storageProviders/awsConnectionHandler.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { IConnectionHandler, ColossusFileStream, StorageProviderGetObjectResponse } from './IConnectionHandler'
import { IConnectionHandler, ColossusFileStream } from './IConnectionHandler'
import {
GetObjectCommand,
ListObjectsCommand,
ListObjectsCommandInput,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3'
import { Readable } from 'stream'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'

export type AwsConnectionHandlerParams = {
accessKeyId: string
Expand Down Expand Up @@ -35,14 +35,6 @@ export class AwsConnectionHandler implements IConnectionHandler {
return response.$metadata.httpStatusCode === 200
}

// no extra connection setup needed for aws
async connect(): Promise<void> {}

get isReady(): boolean {
// Implement isReady method here
return false
}

async uploadFileToRemoteBucket(filename: string, filestream: ColossusFileStream): Promise<any> {
// Setting up S3 upload parameters

Expand All @@ -60,34 +52,15 @@ export class AwsConnectionHandler implements IConnectionHandler {
}
}

async getFileFromRemoteBucket(filename: string): Promise<StorageProviderGetObjectResponse> {
// Implement getFileFromRemoteBucket method here
async getRedirectUrlForObject(filename: string): Promise<string> {
// Implement getRedirectUrlForObject method here
const input = {
Bucket: this.bucket,
Key: filename,
}

const command = new GetObjectCommand(input)
const response = await this.client.send(command)

if (!this.isSuccessfulResponse(response)) {
throw new Error('Failed to get file from S3')
}

if (!response.Body || !response.ContentType || !response.ContentLength) {
throw new Error('Response body, content type, or content length is undefined')
}

return {
Body: new Readable({
read() {
this.push(response.Body)
this.push(null)
},
}),
ContentType: response.ContentType!,
ContentLength: response.ContentLength!,
}
return await getSignedUrl(this.client, command, { expiresIn: 60 * 60 })
}

async listFilesOnRemoteBucket(): Promise<string[]> {
Expand Down
19 changes: 3 additions & 16 deletions storage-node/src/services/webApi/controllers/filesApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,23 +92,10 @@ export async function getFile(
}
const connection = getStorageProviderConnection()!

const { Body: stream, ContentType, ContentLength } = await connection.getFileFromRemoteBucket(dataObjectId)
stream.on('headers', (res) => {
// serve all files for download
res.setHeader('Content-Disposition', 'inline')
res.setHeader('Content-Type', ContentType)
res.setHeader('Content-Length', ContentLength)
})
const url = await connection.getRedirectUrlForObject(dataObjectId)

stream.on('error', (err) => {
sendResponseWithError(res, next, err, 'files')
unpin()
})

stream.on('end', () => {
unpin()
})
stream.pipe(res)
// Redirect to the remote file
res.redirect(url)
}
} catch (err) {
sendResponseWithError(res, next, err, 'files')
Expand Down
Loading

0 comments on commit c269c10

Please sign in to comment.