feat: Add support for "slash tmp" to s3 mounts

This PR does so for IBM mounts

part of #7721

Author: starpit

packages/core/src/core/userdata.ts

@@ -183,3 +183,18 @@ export const setPreference = async (key: string, value: string): Promise<string>
   await fsyncPreferences(prefs)
   return value
 }
+
+export async function getOrSetPreference(
+  key: string,
+  defaultValue: string | (() => string | Promise<string>)
+): Promise<string> {
+  const prefs = await preferences()
+  if (prefs[key]) {
+    return prefs[key]
+  } else {
+    const value = typeof defaultValue === 'string' ? defaultValue : await defaultValue()
+    prefs[key] = value
+    await fsyncPreferences(prefs)
+    return value
+  }
+}

packages/core/src/index.ts

@@ -250,6 +250,7 @@ export { commandsOffered as commandsOfferedByPlugin, userHome as pluginUserHome
 
 // Settings
 export { userDataDir, uiThemes } from './core/settings'
+export { getOrSetPreference, getPreference, setPreference } from './core/userdata'
 
 // Storage for user data
 export { default as Store } from './models/store'

plugins/plugin-s3/ibm/src/IBMCloudS3Provider.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2021 The Kubernetes Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { S3Provider } from '@kui-shell/plugin-s3'
+
+import Config from './model/Config'
+import Geos, { GeoDefaults } from './model/geos'
+
+export default class IBMCloudS3Provider implements S3Provider {
+  public readonly endPoint: string
+  public readonly directEndPoint: string
+
+  public readonly region: string
+  public readonly accessKey: string
+  public readonly secretKey: string
+
+  public readonly understandsFolders = true
+  public readonly bucketNameNSlashes: 1
+  public readonly listBuckets: S3Provider['listBuckets']
+
+  public isDefault: boolean
+
+  public constructor(
+    private readonly geo: string,
+    public readonly mountName: string,
+    config?: Config,
+    public readonly error?: Error
+  ) {
+    const accessKey = config ? config.AccessKeyID : undefined
+    const secretKey = config ? config.SecretAccessKey : undefined
+
+    this.endPoint = Geos[geo] || (config ? config.endpointForKui : undefined)
+    this.region = geo
+    this.accessKey = accessKey
+    this.secretKey = secretKey
+
+    // fast-path endpoint when executing in a cloud job?
+    // e.g. s3.ap.cloud-object-storage.appdomain.cloud -> s3.direct.ap.cloud-object-storage.appdomain.cloud
+    this.directEndPoint = this.endPoint.replace(/^s3\./, 's3.direct.')
+
+    const defaultRegion = GeoDefaults[config['Default Region']] || config['Default Region']
+    this.isDefault = config && geo === defaultRegion
+
+    // use the closest available endpoint for listBuckets, since it is geo-agnostic
+    this.listBuckets = {
+      endPoint: config ? config.endpointForKui : Geos[geo],
+      region: geo,
+      accessKey,
+      secretKey
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public bucketFilter({ name, locationConstraint }: { name: string; locationConstraint: string }): boolean {
+    return !locationConstraint || locationConstraint.startsWith(this.geo)
+  }
+}

plugins/plugin-s3/ibm/src/baseMountName.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2021 The Kubernetes Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** So that the user can interface via /s3/ibm; this is the "ibm" part of that path */
+const baseMountName = 'ibm'
+
+export default baseMountName

plugins/plugin-s3/ibm/src/s3provider.ts

@@ -15,64 +15,16 @@
  */
 
 import { REPL, eventChannelUnsafe } from '@kui-shell/core'
-import { S3Provider, ProviderInitializer, UnsupportedS3ProviderError } from '@kui-shell/plugin-s3'
+import { ProviderInitializer, UnsupportedS3ProviderError } from '@kui-shell/plugin-s3'
 
+import Geos from './model/geos'
 import Config from './model/Config'
 import updateChannel from './channel'
 import { isGoodConfig } from './controller/local'
-import Geos, { GeoDefaults } from './model/geos'
 
-const baseMountName = 'ibm'
-
-class IBMCloudS3Provider implements S3Provider {
-  public readonly endPoint: string
-  public readonly directEndPoint: string
-
-  public readonly region: string
-  public readonly accessKey: string
-  public readonly secretKey: string
-
-  public readonly understandsFolders = true
-  public readonly bucketNameNSlashes: 1
-  public readonly listBuckets: S3Provider['listBuckets']
-
-  public isDefault: boolean
-
-  public constructor(
-    private readonly geo: string,
-    public readonly mountName: string,
-    config?: Config,
-    public readonly error?: Error
-  ) {
-    const accessKey = config ? config.AccessKeyID : undefined
-    const secretKey = config ? config.SecretAccessKey : undefined
-
-    this.endPoint = Geos[geo] || (config ? config.endpointForKui : undefined)
-    this.region = geo
-    this.accessKey = accessKey
-    this.secretKey = secretKey
-
-    // fast-path endpoint when executing in a cloud job?
-    // e.g. s3.ap.cloud-object-storage.appdomain.cloud -> s3.direct.ap.cloud-object-storage.appdomain.cloud
-    this.directEndPoint = this.endPoint.replace(/^s3\./, 's3.direct.')
-
-    const defaultRegion = GeoDefaults[config['Default Region']] || config['Default Region']
-    this.isDefault = config && geo === defaultRegion
-
-    // use the closest available endpoint for listBuckets, since it is geo-agnostic
-    this.listBuckets = {
-      endPoint: config ? config.endpointForKui : Geos[geo],
-      region: geo,
-      accessKey,
-      secretKey
-    }
-  }
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  public bucketFilter({ name, locationConstraint }: { name: string; locationConstraint: string }): boolean {
-    return !locationConstraint || locationConstraint.startsWith(this.geo)
-  }
-}
+import baseMountName from './baseMountName'
+import IBMCloudS3Provider from './IBMCloudS3Provider'
+import extraProvidersForDefaultRegion from './specialMounts'
 
 /** Listening for reconfigs? */
 let listeningAlready = false
@@ -86,6 +38,7 @@ async function fetchConfig(repl: REPL): Promise<void | Config> {
   return (await currentConfig).content
 }
 
+/** Initialize an S3Provider for the given geo */
 async function init(geo: string, mountName: string, repl: REPL, reinit: () => void) {
   try {
     if (!listeningAlready) {
@@ -110,13 +63,14 @@ async function init(geo: string, mountName: string, repl: REPL, reinit: () => vo
         // TODO: isn't there a race here?
         config['Default Region'] = await repl.qexec('ibmcloud cos config region default')
       }
+
       const provider = new IBMCloudS3Provider(geo, mountName, config)
+
+      // special handling for default geo
       if (provider.isDefault) {
-        // add an /s3/ibm/default mount point
-        const defaultProvider = new IBMCloudS3Provider(geo, 'ibm/default', config)
-        defaultProvider.isDefault = true
+        // e.g. add an /s3/ibm/default mount point
         provider.isDefault = false
-        return [defaultProvider, provider]
+        return [...(await extraProvidersForDefaultRegion(geo, config)), provider]
       } else {
         return provider
       }
@@ -126,6 +80,10 @@ async function init(geo: string, mountName: string, repl: REPL, reinit: () => vo
   }
 }
 
+/**
+ * We want one ProviderInitializer per geo
+ *
+ */
 const initializer: ProviderInitializer[] = Object.keys(Geos)
   // .filter(_ => !/-geo$/.test(_)) // don't manifest geo endpoints in the VFS
   .map(geo => {

plugins/plugin-s3/ibm/src/specialMounts.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021 The Kubernetes Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { v4 } from 'uuid'
+import { getOrSetPreference } from '@kui-shell/core'
+
+import Config from './model/Config'
+import baseMountName from './baseMountName'
+import IBMCloudS3Provider from './IBMCloudS3Provider'
+
+/** The /s3/ibm/default mount point */
+function defaultProvider(geo: string, config: Config) {
+  const provider = new IBMCloudS3Provider(geo, `${baseMountName}/default`, config)
+  provider.isDefault = true
+  return provider
+}
+
+class Pseudo extends IBMCloudS3Provider {
+  public constructor(geo: string, mountName: string, config: Config, public readonly subdir: string) {
+    super(geo, mountName, config)
+  }
+}
+
+/** The /s3/ibm/tmp mount point */
+async function pseudoProvider(geo: string, config: Config, pseudo: string) {
+  const subdir = await getOrSetPreference(`org.kubernetes-sigs.kui/s3/pseudomount/${pseudo}`, `${pseudo}-${v4()}`)
+  const provider = new Pseudo(geo, `${baseMountName}/tmp`, config, subdir)
+  return provider
+}
+
+export default function extraProvidersForDefaultRegion(geo: string, config: Config) {
+  return Promise.all([
+    defaultProvider(geo, config),
+    pseudoProvider(geo, config, 'bin'),
+    pseudoProvider(geo, config, 'tmp')
+  ])
+}

plugins/plugin-s3/src/index.ts

@@ -21,5 +21,6 @@ export {
   ProviderInitializer,
   UnsupportedS3ProviderError
 } from './providers'
-export { minioConfig, mounts as getCurrentMounts, Mount } from './vfs/responders'
+
 export { default as eventBus } from './vfs/events'
+export { minioConfig, mounts as getCurrentMounts, Mount } from './vfs/responders'

plugins/plugin-s3/src/providers/model.ts

@@ -22,18 +22,21 @@ type BucketFilter = (bucket: { name: string; locationConstraint: string }) => bo
 
 type Provider = ClientOptions & {
   /** Optimized path for job execution? */
-  directEndPoint?: string
+  readonly directEndPoint?: string
 
-  mountName: string
-  region?: string
-  error?: Error
-  bucketFilter?: BucketFilter
-  listBuckets?: ClientOptions
-  understandsFolders?: boolean
-  isDefault?: boolean
+  readonly mountName: string
+  readonly region?: string
+  readonly error?: Error
+  readonly bucketFilter?: BucketFilter
+  readonly listBuckets?: ClientOptions
+  readonly understandsFolders?: boolean
+  readonly isDefault?: boolean
 
   /** Does this provider allows access only to public buckets? */
-  publicOnly?: boolean
+  readonly publicOnly?: boolean
+
+  /** Is this mount a subirectory mount of an existing mount? */
+  readonly subdir?: string
 }
 export default Provider
 

plugins/plugin-s3/src/test/s3/pseudo.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2021 The Kubernetes Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { tmpdir } from 'os'
+import { Common } from '@kui-shell/test'
+
+import S3Utils, { README, README3 } from './util'
+
+export default function pseudo(this: Common.ISuite) {
+  // bind the helper routines
+  const { copyFromS3, copyToS3, copyWithinS3, lsExpecting404, rm } = S3Utils.bind(this)()
+
+  // v4 to get unique names; some s3 backends can't recreate a
+  // bucketName immediately after deleting it
+  const bucketName1 = 'tmp'
+  const bucketName2 = 'bin'
+
+  copyToS3(bucketName1)
+  copyWithinS3(bucketName1, '*.md', bucketName2)
+  copyWithinS3(bucketName1, README, bucketName2, README3)
+  copyFromS3(bucketName2, README3, tmpdir()) // copy out the intra-s3 copy
+  rm(bucketName1, README)
+  rm(bucketName2, README)
+  lsExpecting404(bucketName1)
+  lsExpecting404(bucketName2)
+}

plugins/plugin-s3/src/test/s3/s3.ts

@@ -20,6 +20,7 @@ import cd from './cd'
 import S3Utils from './util'
 import basics from './basics'
 import rimraf from './rimraf'
+import pseudo from './pseudo'
 import folders from './folders'
 import wildcardRimraf from './wildcard-rimraf'
 
@@ -37,6 +38,16 @@ if (process.env.NEEDS_MINIO) {
         .catch(Common.oops(this, true)))
   })
 
+  xdescribe('s3 vfs pseudo mounts', function(this: Common.ISuite) {
+    before(Common.before(this))
+    after(Common.after(this))
+
+    const { init } = S3Utils.bind(this)()
+
+    init()
+    pseudo.bind(this)()
+  })
+
   describe('s3 vfs folders', function(this: Common.ISuite) {
     before(Common.before(this))
     after(Common.after(this))