Skip to content

Commit 385971d

Browse files
authored
fix: register file systems correctly when using duplicate roleArns BM-1055 (#3367)
### Motivation We have been seeing some tiff files intermitently fail to load when loading across AWS accounts, this was tracked down a failure to register a AWS roleArn against the target bucket when the roleArn was used in multiple different buckets ### Modifications Use the more general "onFileSystemFound" as "onFileSystemCreated" only happens once per roleArn. ### Verification Tested locally
1 parent af656bf commit 385971d

File tree

3 files changed

+18
-14
lines changed

3 files changed

+18
-14
lines changed

package-lock.json

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/shared/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
"@basemaps/config": "^7.11.0",
3030
"@basemaps/geo": "^7.11.0",
3131
"@chunkd/fs": "^11.2.0",
32-
"@chunkd/fs-aws": "^11.2.2",
32+
"@chunkd/fs-aws": "^11.3.0",
3333
"@chunkd/middleware": "^11.1.0",
3434
"@cogeotiff/core": "^9.0.3",
3535
"@cotar/core": "^6.0.1",

packages/shared/src/file.system.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url';
22

33
import { S3Client } from '@aws-sdk/client-s3';
44
import { sha256base58 } from '@basemaps/config';
5-
import { FileSystem, fsa, FsHttp } from '@chunkd/fs';
5+
import { fsa, FsHttp } from '@chunkd/fs';
66
import { AwsCredentialConfig, AwsS3CredentialProvider, FsAwsS3 } from '@chunkd/fs-aws';
77
import { SourceCache, SourceChunk } from '@chunkd/middleware';
88
import type { SourceCallback, SourceRequest } from '@chunkd/source';
@@ -36,17 +36,21 @@ export const s3FsPublic = new FsAwsS3(
3636
/** Ensure middleware are added to all s3 clients that are created */
3737
function applyS3MiddleWare(fs: FsAwsS3): void {
3838
if (fs.s3 == null) return;
39-
fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
39+
const stacks = fs.s3.middlewareStack.identify();
40+
if (stacks.find((f) => f.startsWith('FQDN - ')) == null) {
41+
fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
42+
}
4043
}
4144

4245
applyS3MiddleWare(s3FsPublic);
4346
applyS3MiddleWare(s3Fs);
4447

4548
const credentials = new AwsS3CredentialProvider();
4649

47-
credentials.onFileSystemCreated = (acc: AwsCredentialConfig, fs: FileSystem): void => {
48-
LogConfig.get().info({ prefix: acc.prefix, roleArn: acc.roleArn }, 'FileSystem:Register');
49-
applyS3MiddleWare(fs as FsAwsS3);
50+
credentials.onFileSystemFound = (acc: AwsCredentialConfig, fs?: FsAwsS3, path?: URL): void => {
51+
if (fs == null) return;
52+
LogConfig.get().info({ prefix: acc.prefix, roleArn: acc.roleArn, path: path?.href }, 'FileSystem:Register');
53+
applyS3MiddleWare(fs);
5054
fsa.register(acc.prefix, fs);
5155
};
5256

0 commit comments

Comments
 (0)