Skip to content

Commit

Permalink
Bump minio to v8.0.0 (#991)
Browse files Browse the repository at this point in the history
  • Loading branch information
@SleepyLeslie
SleepyLeslie committed Jun 3, 2024
1 parent c27512f commit 9c90da7
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 43 deletions.
55 changes: 38 additions & 17 deletions app/server/lib/MinIOExternalStorage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,39 @@ import {IncomingMessage} from 'http';
import * as fse from 'fs-extra';
import * as minio from 'minio';

// The minio typings appear to be quite stale. Extend them here to avoid
// lots of casts to any.
type MinIOClient = minio.Client & {
statObject(bucket: string, key: string, options: {versionId?: string}): Promise<MinIOBucketItemStat>;
getObject(bucket: string, key: string, options: {versionId?: string}): Promise<IncomingMessage>;
listObjects(bucket: string, key: string, recursive: boolean,
options: {IncludeVersion?: boolean}): minio.BucketStream<minio.BucketItem>;
removeObjects(bucket: string, versions: Array<{name: string, versionId: string}>): Promise<void>;
};

type MinIOBucketItemStat = minio.BucketItemStat & {
versionId?: string;
metaData?: Record<string, string>;
};
// The minio-js v8.0.0 typings are sometimes incorrect. Here are some workarounds.
interface MinIOClient extends
// Some of them are not directly extendable, must be omitted first and then redefined.
Omit<minio.Client, "listObjects" | "getBucketVersioning" | "removeObjects">
{
// The official typing returns `Promise<Readable>`, dropping some useful metadata.
getObject(bucket: string, key: string, options: {versionId?: string}): Promise<IncomingMessage>;
// The official typing dropped "options" in their .d.ts file, but it is present in the underlying impl.
listObjects(bucket: string, key: string, recursive: boolean,
options: {IncludeVersion?: boolean}): minio.BucketStream<minio.BucketItem>;
// The released v8.0.0 wrongly returns `Promise<void>`; borrowed from PR #1297
getBucketVersioning(bucketName: string): Promise<MinIOVersioningStatus>;
// The released v8.0.0 typing is outdated; copied over from commit 8633968.
removeObjects(bucketName: string, objectsList: RemoveObjectsParam): Promise<RemoveObjectsResponse[]>
}

type MinIOVersioningStatus = "" | {
Status: "Enabled" | "Suspended",
MFADelete?: string,
ExcludeFolders?: boolean,
ExcludedPrefixes?: {Prefix: string}[]
}

type RemoveObjectsParam = string[] | { name: string, versionId?: string }[]

type RemoveObjectsResponse = null | undefined | {
Error?: {
Code?: string
Message?: string
Key?: string
VersionId?: string
}
}

/**
* An external store implemented using the MinIO client, which
Expand All @@ -38,7 +57,7 @@ export class MinIOExternalStorage implements ExternalStorage {
region: string
},
private _batchSize?: number,
private _s3 = new minio.Client(options) as MinIOClient
private _s3 = new minio.Client(options) as unknown as MinIOClient
) {
}

Expand Down Expand Up @@ -70,7 +89,7 @@ export class MinIOExternalStorage implements ExternalStorage {
public async upload(key: string, fname: string, metadata?: ObjMetadata) {
const stream = fse.createReadStream(fname);
const result = await this._s3.putObject(
this.bucket, key, stream,
this.bucket, key, stream, undefined,
metadata ? {Metadata: toExternalMetadata(metadata)} : undefined
);
// Empirically VersionId is available in result for buckets with versioning enabled.
Expand Down Expand Up @@ -111,7 +130,9 @@ export class MinIOExternalStorage implements ExternalStorage {

public async hasVersioning(): Promise<Boolean> {
const versioning = await this._s3.getBucketVersioning(this.bucket);
return versioning && versioning.Status === 'Enabled';
// getBucketVersioning() may return an empty string when versioning has never been enabled.
// This situation is not addressed in minio-js v8.0.0, but included in our workaround.
return versioning !== '' && versioning?.Status === 'Enabled';
}

public async versions(key: string, options?: { includeDeleteMarkers?: boolean }) {
Expand Down
3 changes: 1 addition & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@
"@types/lru-cache": "5.1.1",
"@types/marked": "4.0.8",
"@types/mime-types": "2.1.0",
"@types/minio": "7.0.15",
"@types/mocha": "10.0.1",
"@types/moment-timezone": "0.5.9",
"@types/mousetrap": "1.6.2",
Expand Down Expand Up @@ -167,7 +166,7 @@
"locale-currency": "0.0.2",
"lodash": "4.17.21",
"marked": "4.2.12",
"minio": "7.1.3",
"minio": "8.0.0",
"moment": "2.29.4",
"moment-timezone": "0.5.35",
"morgan": "1.9.1",
Expand Down
53 changes: 29 additions & 24 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -964,13 +964,6 @@
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==

"@types/minio@7.0.15":
version "7.0.15"
resolved "https://registry.yarnpkg.com/@types/minio/-/minio-7.0.15.tgz#6fbf2e17aeae172cbf181ea52b1faa05a601ce42"
integrity sha512-1VR05lWJDuxkn/C7d87MPAJs0p+onKnkUN3nyQ0xrrtaziZQmONy/nxXRaAVWheEyIb6sl0TTi77I/GAQDN5Lw==
dependencies:
"@types/node" "*"

"@types/mocha@10.0.1":
version "10.0.1"
resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.1.tgz#2f4f65bb08bc368ac39c96da7b2f09140b26851b"
Expand Down Expand Up @@ -2166,6 +2159,11 @@ buffer-crc32@^0.2.1, buffer-crc32@^0.2.13:
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==

buffer-crc32@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-1.0.0.tgz#a10993b9055081d55304bd9feb4a072de179f405"
integrity sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==

buffer-equal-constant-time@1.0.1:
version "1.0.1"
resolved "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz"
Expand Down Expand Up @@ -3673,6 +3671,11 @@ eventemitter3@^4.0.0:
resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz"
integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==

eventemitter3@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4"
integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==

events@^1.1.1, events@~1.1.0:
version "1.1.1"
resolved "https://registry.npmjs.org/events/-/events-1.1.1.tgz"
Expand Down Expand Up @@ -5136,11 +5139,6 @@ json-stable-stringify@~0.0.0:
dependencies:
jsonify "~0.0.0"

json-stream@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/json-stream/-/json-stream-1.0.0.tgz#1a3854e28d2bbeeab31cc7ddf683d2ddc5652708"
integrity sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg==

json5@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
Expand Down Expand Up @@ -5674,24 +5672,24 @@ minimist@^1.1.0, minimist@^1.2.0, minimist@^1.2.5:
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==

minio@7.1.3:
version "7.1.3"
resolved "https://registry.yarnpkg.com/minio/-/minio-7.1.3.tgz#86dc95f3671045d6956920db757bb63f25bf20ee"
integrity sha512-xPrLjWkTT5E7H7VnzOjF//xBp9I40jYB4aWhb2xTFopXXfw+Wo82DDWngdUju7Doy3Wk7R8C4LAgwhLHHnf0wA==
minio@8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/minio/-/minio-8.0.0.tgz#c712bace51232e20c30568ca160a41a5ac1d13ce"
integrity sha512-GkM/lk+Gzwd4fAQvLlB+cy3NV3PRADe0tNXnH9JD5BmdAHKIp+5vypptbjdkU85xWBIQsa2xK35GpXjmYXBBYA==
dependencies:
async "^3.2.4"
block-stream2 "^2.1.0"
browser-or-node "^2.1.1"
buffer-crc32 "^0.2.13"
buffer-crc32 "^1.0.0"
eventemitter3 "^5.0.1"
fast-xml-parser "^4.2.2"
ipaddr.js "^2.0.1"
json-stream "^1.0.0"
lodash "^4.17.21"
mime-types "^2.1.35"
query-string "^7.1.3"
stream-json "^1.8.0"
through2 "^4.0.2"
web-encoding "^1.1.5"
xml "^1.0.1"
xml2js "^0.5.0"

minipass-collect@^1.0.2:
Expand Down Expand Up @@ -7491,6 +7489,11 @@ stream-browserify@^2.0.0:
inherits "~2.0.1"
readable-stream "^2.0.2"

stream-chain@^2.2.5:
version "2.2.5"
resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09"
integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==

stream-combiner2@^1.1.1:
version "1.1.1"
resolved "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz"
Expand All @@ -7517,6 +7520,13 @@ stream-http@^2.0.0:
to-arraybuffer "^1.0.0"
xtend "^4.0.0"

stream-json@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c"
integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==
dependencies:
stream-chain "^2.2.5"

stream-splicer@^2.0.0:
version "2.0.1"
resolved "https://registry.npmjs.org/stream-splicer/-/stream-splicer-2.0.1.tgz"
Expand Down Expand Up @@ -8632,11 +8642,6 @@ xml2js@^0.5.0:
sax ">=0.6.0"
xmlbuilder "~11.0.0"

xml@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==

xmlbuilder2@^2.4.0:
version "2.4.1"
resolved "https://registry.yarnpkg.com/xmlbuilder2/-/xmlbuilder2-2.4.1.tgz#899c783a833188c5a5aa6f3c5428a3963f3e479d"
Expand Down

0 comments on commit 9c90da7

Please sign in to comment.