From 38fdbab37719ce3d3907f45de8fb467589d4d73e Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 19:15:06 -0400 Subject: [PATCH 01/12] try to use aws cli in place of deno_s3 for files >1GB --- actions/upload/upload.ts | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index 68b6e67c..ad3c6250 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -8,11 +8,14 @@ args: - --allow-read - --allow-env - --allow-write +depdenencies: + aws.amazon.com/cli: ^2 ---*/ -import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils } from "tea" +import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils, porcelain } from "tea" import { decode as base64Decode } from "deno/encoding/base64.ts" const { useOffLicense, usePrefix, useCache } = hooks +const { run } = porcelain import { basename, dirname } from "deno/path/mod.ts" import { set_output } from "../utils/gha.ts" import { sha256 } from "../bottle/bottle.ts" @@ -68,11 +71,28 @@ async function get_versions(key: string, pkg: Package, bucket: S3Bucket): Promis .sort(semver.compare) } -async function put(key_: string, body: string | Path | Uint8Array, bucket: S3Bucket, qaRequired: boolean) { +class ExtBucket { + name: string + bucket: S3Bucket + + constructor(name: string, s3: S3) { + this.bucket = s3.getBucket(name) + this.name = name + } +} + +async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBucket, qaRequired: boolean) { const key = qaRequired ? `qa/${key_}` : key_ console.info({ uploading: body, to: key }) if (!qaRequired) rv.push(`/${key}`) if (body instanceof Path) { + // For really large files, just kick it to the CLI + if (Deno.statSync(body.string).size > 1024 * 1024 * 1024) { + console.log("large file (>1GB), using aws cli") + const filename = body.string + console.log(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`]) + return retry(() => run(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`])) + } body = await Deno.readFile(body.string) } else if (typeof body === "string") { body = encode(body) @@ -91,8 +111,8 @@ const s3 = new S3({ region: "us-east-1", }) -const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!) -const stagingBucket = s3.getBucket(Deno.env.get("AWS_S3_STAGING_BUCKET")!) +const bucket = new ExtBucket(Deno.env.get("AWS_S3_BUCKET")!, s3) +const stagingBucket = new ExtBucket(Deno.env.get("AWS_S3_STAGING_BUCKET")!, s3) const encode = (() => { const e = new TextEncoder() @@ -119,7 +139,7 @@ for (const [index, pkg] of pkgs.entries()) { const signature = base64Decode(signatures[index]) const stowed = cache.decode(bottle)! const key = useOffLicense("s3").key(stowed) - const versions = await get_versions(key, pkg, dst) + const versions = await get_versions(key, pkg, dst.bucket) //FIXME stream the bottle (at least) to S3 await put(key, bottle, dst, qaRequired) @@ -140,7 +160,7 @@ for (const [index, pkg] of pkgs.entries()) { extname: src.extname(), }) const srcChecksum = await sha256(src) - const srcVersions = await get_versions(srcKey, pkg, dst) + const srcVersions = await get_versions(srcKey, pkg, dst.bucket) await put(srcKey, src, dst, qaRequired) await put(`${srcKey}.sha256sum`, `${srcChecksum} ${basename(srcKey)}`, dst, qaRequired) await put(`${dirname(srcKey)}/versions.txt`, srcVersions.join("\n"), dst, qaRequired) From 8adc92e8a50464d466082c2454c879a1746f67c1 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 22:59:54 -0400 Subject: [PATCH 02/12] maybe if i could spell --- actions/upload/upload.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index ad3c6250..ea9ab224 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -8,7 +8,7 @@ args: - --allow-read - --allow-env - --allow-write -depdenencies: +dependencies: aws.amazon.com/cli: ^2 ---*/ @@ -88,9 +88,9 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu if (body instanceof Path) { // For really large files, just kick it to the CLI if (Deno.statSync(body.string).size > 1024 * 1024 * 1024) { - console.log("large file (>1GB), using aws cli") + console.info("large file (>1GB), using aws cli") const filename = body.string - console.log(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`]) + console.info(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`]) return retry(() => run(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`])) } body = await Deno.readFile(body.string) From 14095b07e09d1e02554c5c1a7ba269c173c82bae Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 23:04:26 -0400 Subject: [PATCH 03/12] sigh --- actions/upload/upload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index ea9ab224..f45c4bba 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -98,7 +98,7 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu body = encode(body) } // @ts-ignore typescript doesn't narrow the types properly here - return retry(()=>bucket.putObject(key, body)) + return retry(()=>bucket.bucket.putObject(key, body)) } //------------------------------------------------------------------------- main From 8410d17dd65a844de7e125dfb61898751cc2faf2 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 23:13:50 -0400 Subject: [PATCH 04/12] --allow-run=aws --- actions/upload/upload.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index f45c4bba..47e211b8 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -8,6 +8,7 @@ args: - --allow-read - --allow-env - --allow-write + - --allow-run=aws dependencies: aws.amazon.com/cli: ^2 ---*/ From bea687ed0ff9d102087516371f5dde7290640021 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 23:34:58 -0400 Subject: [PATCH 05/12] we'll get it --- actions/upload/upload.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index 47e211b8..d97198d2 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -91,8 +91,22 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu if (Deno.statSync(body.string).size > 1024 * 1024 * 1024) { console.info("large file (>1GB), using aws cli") const filename = body.string - console.info(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`]) - return retry(() => run(["aws", "s3", "cp", filename, `s3://${bucket.name}/${key}`])) + const cmd = [ + "aws", + "s3", + "cp", + filename, + `s3://${bucket.name}/${key}`, + ] + const env = { + AWS_ACCESS_KEY_ID: Deno.env.get("AWS_ACCESS_KEY_ID")!, + AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, + AWS_DEFAULT_REGION: "us-east-1", + } + console.info(cmd) + return retry(() => run(cmd, { env, stdout: true, stderr: true }).catch((e) => { + console.error("aws call failed:", e) + })) } body = await Deno.readFile(body.string) } else if (typeof body === "string") { From 4bfde9542126048cd266964d63e0956495068902 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 23:42:35 -0400 Subject: [PATCH 06/12] one more --- actions/upload/upload.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index d97198d2..b1dad1e3 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -104,8 +104,10 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu AWS_DEFAULT_REGION: "us-east-1", } console.info(cmd) - return retry(() => run(cmd, { env, stdout: true, stderr: true }).catch((e) => { - console.error("aws call failed:", e) + return retry(() => run(cmd, { env, stdout: true, stderr: true, status: true }).then((x) => { + console.log("status:", x.status) + console.log("stdout:", x.stdout) + console.log("stderr:", x.stderr) })) } body = await Deno.readFile(body.string) From c2211316a633eb801c22fd733a2d92cde6508d1b Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Tue, 13 Jun 2023 23:56:32 -0400 Subject: [PATCH 07/12] wip --- actions/upload/upload.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index b1dad1e3..7c466e2d 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -104,11 +104,12 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu AWS_DEFAULT_REGION: "us-east-1", } console.info(cmd) - return retry(() => run(cmd, { env, stdout: true, stderr: true, status: true }).then((x) => { - console.log("status:", x.status) - console.log("stdout:", x.stdout) - console.log("stderr:", x.stderr) - })) + const foo = await run(cmd, { env, stdout: true, stderr: true, status: true }) + console.error(`status: ${foo.status}`) + console.error(`stdout: ${foo.stdout}`) + console.error(`stderr: ${foo.stderr}`) + return + // return retry(() => run(cmd, { env, stdout: true, stderr: true, status: true })) } body = await Deno.readFile(body.string) } else if (typeof body === "string") { From 6e59ac4693000cb8cd819a77560d69df206b7eaf Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Wed, 14 Jun 2023 00:04:27 -0400 Subject: [PATCH 08/12] maybe this'll play nicer --- actions/upload/upload.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index 7c466e2d..ad4ace1b 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -9,8 +9,6 @@ args: - --allow-env - --allow-write - --allow-run=aws -dependencies: - aws.amazon.com/cli: ^2 ---*/ import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils, porcelain } from "tea" @@ -92,6 +90,7 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu console.info("large file (>1GB), using aws cli") const filename = body.string const cmd = [ + "tea", "aws", "s3", "cp", From 67611ba4815e38f5f153337b60b44bd8035c8c9a Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Wed, 14 Jun 2023 00:12:07 -0400 Subject: [PATCH 09/12] Deno.Command --- actions/upload/upload.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index ad4ace1b..21bef0e3 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -9,6 +9,8 @@ args: - --allow-env - --allow-write - --allow-run=aws +dependencies: + aws.amazon.com/cli: ^2 ---*/ import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils, porcelain } from "tea" @@ -89,9 +91,7 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu if (Deno.statSync(body.string).size > 1024 * 1024 * 1024) { console.info("large file (>1GB), using aws cli") const filename = body.string - const cmd = [ - "tea", - "aws", + const args = [ "s3", "cp", filename, @@ -102,11 +102,10 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, AWS_DEFAULT_REGION: "us-east-1", } - console.info(cmd) - const foo = await run(cmd, { env, stdout: true, stderr: true, status: true }) - console.error(`status: ${foo.status}`) - console.error(`stdout: ${foo.stdout}`) - console.error(`stderr: ${foo.stderr}`) + console.info("aws", args) + const cmd = new Deno.Command("aws", { args, env }).spawn() + const res = await cmd.status() + console.error(`status: ${res}`) return // return retry(() => run(cmd, { env, stdout: true, stderr: true, status: true })) } From 64419ec50e8219b21d36c7a46583e0b49e6db3fa Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Wed, 14 Jun 2023 00:17:55 -0400 Subject: [PATCH 10/12] sigh --- actions/upload/upload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index 21bef0e3..ab30a6a8 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -104,7 +104,7 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu } console.info("aws", args) const cmd = new Deno.Command("aws", { args, env }).spawn() - const res = await cmd.status() + const res = await cmd.status console.error(`status: ${res}`) return // return retry(() => run(cmd, { env, stdout: true, stderr: true, status: true })) From f3987a14c694c060fe17fc7826ef5fe2827d20c9 Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Wed, 14 Jun 2023 00:25:32 -0400 Subject: [PATCH 11/12] clean up --- actions/upload/upload.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index ab30a6a8..de98338f 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -102,12 +102,10 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, AWS_DEFAULT_REGION: "us-east-1", } - console.info("aws", args) - const cmd = new Deno.Command("aws", { args, env }).spawn() - const res = await cmd.status - console.error(`status: ${res}`) - return - // return retry(() => run(cmd, { env, stdout: true, stderr: true, status: true })) + return retry(() => { + const cmd = new Deno.Command("aws", { args, env }).spawn() + return cmd.status + }) } body = await Deno.readFile(body.string) } else if (typeof body === "string") { From 3b2e2c746670f6544605812e701196a620828e1f Mon Sep 17 00:00:00 2001 From: Jacob Heider Date: Wed, 14 Jun 2023 14:29:37 -0400 Subject: [PATCH 12/12] remove arbitrary file size distinction in cli usage --- actions/upload/upload.ts | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/actions/upload/upload.ts b/actions/upload/upload.ts index de98338f..a9ba2cc8 100755 --- a/actions/upload/upload.ts +++ b/actions/upload/upload.ts @@ -87,27 +87,21 @@ async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBu console.info({ uploading: body, to: key }) if (!qaRequired) rv.push(`/${key}`) if (body instanceof Path) { - // For really large files, just kick it to the CLI - if (Deno.statSync(body.string).size > 1024 * 1024 * 1024) { - console.info("large file (>1GB), using aws cli") - const filename = body.string - const args = [ - "s3", - "cp", - filename, - `s3://${bucket.name}/${key}`, - ] - const env = { - AWS_ACCESS_KEY_ID: Deno.env.get("AWS_ACCESS_KEY_ID")!, - AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - AWS_DEFAULT_REGION: "us-east-1", - } - return retry(() => { - const cmd = new Deno.Command("aws", { args, env }).spawn() - return cmd.status - }) + const args = [ + "s3", + "cp", + body.string, + `s3://${bucket.name}/${key}`, + ] + const env = { + AWS_ACCESS_KEY_ID: Deno.env.get("AWS_ACCESS_KEY_ID")!, + AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, + AWS_DEFAULT_REGION: "us-east-1", } - body = await Deno.readFile(body.string) + return retry(() => { + const cmd = new Deno.Command("aws", { args, env }).spawn() + return cmd.status + }) } else if (typeof body === "string") { body = encode(body) }