use aws cli for uploads >1GB

actions/upload/upload.ts

@@ -8,11 +8,15 @@ args:
   - --allow-read
   - --allow-env
   - --allow-write
+  - --allow-run=aws
+dependencies:
+  aws.amazon.com/cli: ^2
 ---*/
 
-import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils } from "tea"
+import { Package, PackageRequirement, SemVer, Path, semver, hooks, utils, porcelain } from "tea"
 import { decode as base64Decode } from "deno/encoding/base64.ts"
 const { useOffLicense, usePrefix, useCache } = hooks
+const { run } = porcelain
 import { basename, dirname } from "deno/path/mod.ts"
 import { set_output } from "../utils/gha.ts"
 import { sha256 } from "../bottle/bottle.ts"
@@ -68,17 +72,41 @@ async function get_versions(key: string, pkg: Package, bucket: S3Bucket): Promis
     .sort(semver.compare)
 }
 
-async function put(key_: string, body: string | Path | Uint8Array, bucket: S3Bucket, qaRequired: boolean) {
+class ExtBucket {
+  name: string
+  bucket: S3Bucket
+
+  constructor(name: string, s3: S3) {
+    this.bucket = s3.getBucket(name)
+    this.name = name
+  }
+}
+
+async function put(key_: string, body: string | Path | Uint8Array, bucket: ExtBucket, qaRequired: boolean) {
   const key = qaRequired ? `qa/${key_}` : key_
   console.info({ uploading: body, to: key })
   if (!qaRequired) rv.push(`/${key}`)
   if (body instanceof Path) {
-    body = await Deno.readFile(body.string)
+    const args = [
+      "s3",
+      "cp",
+      body.string,
+      `s3://${bucket.name}/${key}`,
+    ]
+    const env = {
+      AWS_ACCESS_KEY_ID: Deno.env.get("AWS_ACCESS_KEY_ID")!,
+      AWS_SECRET_ACCESS_KEY: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
+      AWS_DEFAULT_REGION: "us-east-1",
+    }
+    return retry(() => {
+      const cmd = new Deno.Command("aws", { args, env }).spawn()
+      return cmd.status
+    })
   } else if (typeof body === "string") {
     body = encode(body)
   }
   // @ts-ignore typescript doesn't narrow the types properly here
-  return retry(()=>bucket.putObject(key, body))
+  return retry(()=>bucket.bucket.putObject(key, body))
 }
 
 //------------------------------------------------------------------------- main
@@ -91,8 +119,8 @@ const s3 = new S3({
   region: "us-east-1",
 })
 
-const bucket = s3.getBucket(Deno.env.get("AWS_S3_BUCKET")!)
-const stagingBucket = s3.getBucket(Deno.env.get("AWS_S3_STAGING_BUCKET")!)
+const bucket = new ExtBucket(Deno.env.get("AWS_S3_BUCKET")!, s3)
+const stagingBucket = new ExtBucket(Deno.env.get("AWS_S3_STAGING_BUCKET")!, s3)
 
 const encode = (() => {
   const e = new TextEncoder()
@@ -119,7 +147,7 @@ for (const [index, pkg] of pkgs.entries()) {
   const signature = base64Decode(signatures[index])
   const stowed = cache.decode(bottle)!
   const key = useOffLicense("s3").key(stowed)
-  const versions = await get_versions(key, pkg, dst)
+  const versions = await get_versions(key, pkg, dst.bucket)
 
   //FIXME stream the bottle (at least) to S3
   await put(key, bottle, dst, qaRequired)
@@ -140,7 +168,7 @@ for (const [index, pkg] of pkgs.entries()) {
       extname: src.extname(),
     })
     const srcChecksum = await sha256(src)
-    const srcVersions = await get_versions(srcKey, pkg, dst)
+    const srcVersions = await get_versions(srcKey, pkg, dst.bucket)
     await put(srcKey, src, dst, qaRequired)
     await put(`${srcKey}.sha256sum`, `${srcChecksum}  ${basename(srcKey)}`, dst, qaRequired)
     await put(`${dirname(srcKey)}/versions.txt`, srcVersions.join("\n"), dst, qaRequired)