feat(sdk): reuse volumes for sim.Container (#6336)

Another pass at solving #6284

The approach taken in #6295 where we allow users to bind to the state directory is problematic because docker sometimes changes the permissions of directories on the user's file system after mounting them. This was causing `wing test` to fail (fixed in #6320) and `wing pack` to fail as well.

This PR reworks the API so that data can be persisted via named volumes. Specifically, an anonymous docker volume can be specified, and Wing will automatically name it and reuse it across Wing Console runs. The change should fix the previous issues since named volumes are managed by docker.

The main downside of this approach is that deleting your `target` directory or the state files in `.state/` will not clean everything up all resources/files created by Wing from your machine, as Docker volumes may still exist. Eventually we can provide some facilities for automatically cleaning these, but we're leaving it intentionally out of scope until this general approach has been validated.

## Checklist

- [x] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted)
- [x] Description explains motivation and solution
- [ ] Tests added (always)
- [ ] Docs updated (only required for features)
- [ ] Added `pr/e2e-full` label if this feature requires end-to-end testing

*By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*.

examples/tests/sdk_tests/container/mount.test.w

@@ -6,8 +6,6 @@ skipPlatforms:
 bring sim;
 bring util;
 
-// This test was added to check that "wing test" still works when sim.Container is mounted to the state directory
-
 // only relevant in simulator
 if util.env("WING_TARGET") == "sim" {
   let container = new sim.Container(
@@ -17,7 +15,7 @@ if util.env("WING_TARGET") == "sim" {
       POSTGRES_PASSWORD: "password"
     },
     containerPort: 5432,
-    volumes: ["$WING_STATE_DIR:/var/lib/postgresql/data"],
+    volumes: ["/var/lib/postgresql/data"],
   );
 
   test "my test" {

libs/wingsdk/src/target-sim/container.inflight.ts

@@ -1,5 +1,8 @@
+import * as fs from "fs";
+import { join } from "path";
 import { IContainerClient, HOST_PORT_ATTR } from "./container";
 import { ContainerAttributes, ContainerSchema } from "./schema-resources";
+import { exists } from "./util";
 import { isPath, runCommand, shell } from "../shared/misc";
 import {
   ISimulatorContext,
@@ -9,17 +12,30 @@ import {
 import { Duration, TraceType } from "../std";
 import { Util } from "../util";
 
-export const WING_STATE_DIR_ENV = "WING_STATE_DIR";
+const STATE_FILENAME = "state.json";
+
+/**
+ * Contents of the state file for this resource.
+ */
+interface StateFileContents {
+  /**
+   * A mapping of volume paths to the Wing-managed volume names, which will be reused
+   * across simulator runs.
+   */
+  readonly managedVolumes?: Record<string, string>;
+}
 
 export class Container implements IContainerClient, ISimulatorResourceInstance {
   private readonly imageTag: string;
   private readonly containerName: string;
   private _context: ISimulatorContext | undefined;
+  private managedVolumes: Record<string, string>;
 
   public constructor(private readonly props: ContainerSchema) {
     this.imageTag = props.imageTag;
 
     this.containerName = `wing-container-${Util.ulid()}`;
+    this.managedVolumes = {};
   }
 
   private get context(): ISimulatorContext {
@@ -31,6 +47,14 @@ export class Container implements IContainerClient, ISimulatorResourceInstance {
 
   public async init(context: ISimulatorContext): Promise<ContainerAttributes> {
     this._context = context;
+
+    // Check for a previous state file to see if there was a port that was previously being used
+    // if so, try to use it out of convenience
+    const state: StateFileContents = await this.loadState();
+    if (state.managedVolumes) {
+      this.managedVolumes = state.managedVolumes;
+    }
+
     // if this a reference to a local directory, build the image from a docker file
     if (isPath(this.props.image)) {
       // check if the image is already built
@@ -79,7 +103,21 @@ export class Container implements IContainerClient, ISimulatorResourceInstance {
 
     for (const volume of this.props.volumes ?? []) {
       dockerRun.push("-v");
-      dockerRun.push(volume);
+
+      // if the user specified an anonymous volume
+      if (volume.startsWith("/") && !volume.includes(":")) {
+        // check if we have a managed volume for this path from a previous run
+        if (this.managedVolumes[volume]) {
+          const volumeName = this.managedVolumes[volume];
+          dockerRun.push(`${volumeName}:${volume}`);
+        } else {
+          const volumeName = `wing-volume-${Util.ulid()}`;
+          dockerRun.push(`${volumeName}:${volume}`);
+          this.managedVolumes[volume] = volumeName;
+        }
+      } else {
+        dockerRun.push(volume);
+      }
     }
 
     dockerRun.push(this.imageTag);
@@ -91,12 +129,7 @@ export class Container implements IContainerClient, ISimulatorResourceInstance {
     this.log(`starting container from image ${this.imageTag}`);
     this.log(`docker ${dockerRun.join(" ")}`);
 
-    await shell("docker", dockerRun, {
-      env: {
-        ...process.env,
-        [WING_STATE_DIR_ENV]: this.context.statedir,
-      },
-    });
+    await shell("docker", dockerRun);
 
     this.log(`containerName=${this.containerName}`);
 
@@ -137,7 +170,31 @@ export class Container implements IContainerClient, ISimulatorResourceInstance {
     await runCommand("docker", ["rm", "-f", this.containerName]);
   }
 
-  public async save(): Promise<void> {}
+  public async save(): Promise<void> {
+    await this.saveState({ managedVolumes: this.managedVolumes });
+  }
+
+  private async loadState(): Promise<StateFileContents> {
+    const stateFileExists = await exists(
+      join(this.context.statedir, STATE_FILENAME)
+    );
+    if (stateFileExists) {
+      const stateFileContents = await fs.promises.readFile(
+        join(this.context.statedir, STATE_FILENAME),
+        "utf-8"
+      );
+      return JSON.parse(stateFileContents);
+    } else {
+      return {};
+    }
+  }
+
+  private async saveState(state: StateFileContents): Promise<void> {
+    fs.writeFileSync(
+      join(this.context.statedir, STATE_FILENAME),
+      JSON.stringify(state)
+    );
+  }
 
   public async plan() {
     return UpdatePlan.AUTO;

libs/wingsdk/src/target-sim/container.md

@@ -47,19 +47,19 @@ new sim.Container(
 ### Retaining state
 
 When the Wing Console is closed, all containers are stopped and removed.
-To retain the state of a container across console restarts, you can mount a volume
-to a subdirectory of the resource's simulator state directory, which is available through `$WING_STATE_DIR`:
+To retain the state of a container across console restarts, you can mount an anonymous volume:
 
 ```js
 new sim.Container(
   name: "my-service",
   image: "./my-service",
   containerPort: 8080,
-  volumes: ["$WING_STATE_DIR/volume1:/var/data"],
+  volumes: ["/var/data"],
 );
 ```
 
-`$WING_STATE_DIR` is a directory that is unique to that `sim.Container` instance.
+Wing will automatically name each unnamed volume in `volumes`, and reuse the named
+volumes across console restarts.
 
 ## API
 

libs/wingsdk/test/target-sim/container.test.ts

@@ -174,14 +174,14 @@ test("simple container with a volume", async () => {
   await sim.stop();
 });
 
-test("container can mount a volume to the state directory", async () => {
+test("anonymous volume can be reused across restarts", async () => {
   const app = new SimApp();
 
   const c = new Container(app, "Container", {
     name: "my-app",
     image: join(__dirname, "my-docker-image.mounted-volume"),
     containerPort: 3000,
-    volumes: ["$WING_STATE_DIR:/tmp"],
+    volumes: ["/tmp"],
   });
 
   new Function(
@@ -204,11 +204,17 @@ test("container can mount a volume to the state directory", async () => {
 
   const fn = sim.getResource("root/Function") as IFunctionClient;
   const response = await fn.invoke();
-  expect(response).contains("hello.txt");
-
-  const statedir = sim.getResourceStateDir("root/Container");
-  const files = readdirSync(statedir);
-  expect(files).toEqual(["hello.txt"]);
+  expect(response?.split("\n").filter((s) => s.endsWith(".txt"))).toEqual([
+    "hello.txt",
+  ]);
 
   await sim.stop();
+  await sim.start();
+
+  const fn2 = sim.getResource("root/Function") as IFunctionClient;
+  const response2 = await fn2.invoke();
+  expect(response2?.split("\n").filter((s) => s.endsWith(".txt"))).toEqual([
+    "hello.txt",
+    "world.txt",
+  ]);
 });

libs/wingsdk/test/target-sim/my-docker-image.mounted-volume/index.js

@@ -12,7 +12,13 @@ const server = http.createServer((req, res) => {
   res.end(fs.readdirSync("/tmp").join("\n"));
 });
 
-fs.writeFileSync("/tmp/hello.txt", "Hello, World!", "utf8");
+if (!fs.existsSync("/tmp/hello.txt")) {
+  // on the first run, create a file
+  fs.writeFileSync("/tmp/hello.txt", "Hello, World!", "utf8");
+} else {
+  // on the second run, create a different file
+  fs.writeFileSync("/tmp/world.txt", "Hello, World!", "utf8");
+}
 
 console.log("listening on port 3000");
 server.listen(3000);

tools/hangar/__snapshots__/test_corpus/sdk_tests/container/mount.test.w_test_sim.md

@@ -1,10 +1,5 @@
 # [mount.test.w](../../../../../../examples/tests/sdk_tests/container/mount.test.w) | test | sim
 
-## stderr.log
-```log
-Warning: unable to clean up test directory: Error: EACCES: permission denied, scandir 'target/test/mount.test.wsim/.state/<STATE_FILE>'
-```
-
 ## stdout.log
 ```log
 [INFO] my test | dummy test