From d5f0c573a172c62a51911d720ab502f9df904700 Mon Sep 17 00:00:00 2001
From: Nathan Flurry <git@nathanflurry.com>
Date: Tue, 1 Apr 2025 21:08:42 +0000
Subject: [PATCH] chore: add script to distribute lz4 binary

---
 package.json                    |  3 +-
 scripts/lz4/Dockerfile.debian11 | 22 +++++++++++
 scripts/lz4/distribute.ts       | 65 +++++++++++++++++++++++++++++++++
 scripts/lz4/package.json        | 16 ++++++++
 4 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 scripts/lz4/Dockerfile.debian11
 create mode 100755 scripts/lz4/distribute.ts
 create mode 100644 scripts/lz4/package.json

diff --git a/package.json b/package.json
index 4058b5a0f5..8063d21fc5 100644
--- a/package.json
+++ b/package.json
@@ -8,7 +8,8 @@
 		"site",
 		"sdks/api/full/typescript",
 		"examples/*",
-		"tests/load"
+		"tests/load",
+		"scripts/lz4"
 	],
 	"scripts": {
 		"start": "npx turbo dev",
diff --git a/scripts/lz4/Dockerfile.debian11 b/scripts/lz4/Dockerfile.debian11
new file mode 100644
index 0000000000..7781faf482
--- /dev/null
+++ b/scripts/lz4/Dockerfile.debian11
@@ -0,0 +1,22 @@
+FROM --platform=linux/amd64 debian:11-slim
+
+# Install required build tools
+RUN apt-get update && \
+    apt-get install -y make gcc curl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /build
+
+# Arguments for LZ4 version
+ARG LZ4_VERSION=1.10.0
+
+# Download and extract LZ4
+RUN curl -L https://github.com/lz4/lz4/releases/download/v${LZ4_VERSION}/lz4-${LZ4_VERSION}.tar.gz | tar xz
+
+# Build LZ4
+WORKDIR /build/lz4-${LZ4_VERSION}
+RUN make
+
+# Output binary is at /build/lz4-${LZ4_VERSION}/lz4
\ No newline at end of file
diff --git a/scripts/lz4/distribute.ts b/scripts/lz4/distribute.ts
new file mode 100755
index 0000000000..806cf16998
--- /dev/null
+++ b/scripts/lz4/distribute.ts
@@ -0,0 +1,65 @@
+/**
+ * Build & push lz4. Most repos are on a significantly older version of lz4 that doesn't support multithreading.
+ */
+
+import { $, cd } from "zx";
+import { mkdtempSync, mkdirSync, rmSync } from "fs";
+import { tmpdir, platform } from "os";
+import { join } from "path";
+import { dirname } from "path";
+import { fileURLToPath } from "url";
+
+const LZ4_VERSION = "1.10.0";
+const DISTRO = "debian11";
+const UPLOAD_PATH = `tools/lz4/${LZ4_VERSION}/${DISTRO}-amd64/lz4`;
+
+// Get the directory where this script is located
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+async function main() {
+  // Get credentials
+  const awsAccessKeyId = process.env.R2_RELEASES_ACCESS_KEY_ID || 
+    (await $({ quiet: true })`op read "op://Engineering/rivet-releases R2 Upload/username"`).stdout.trim();
+  const awsSecretAccessKey = process.env.R2_RELEASES_SECRET_ACCESS_KEY || 
+    (await $({ quiet: true })`op read "op://Engineering/rivet-releases R2 Upload/password"`).stdout.trim();
+
+  // Create temp directory for Docker build
+  const tempDir = mkdtempSync(join(tmpdir(), 'lz4-docker-'));
+  console.log(`Using temp directory: ${tempDir}`);
+  
+  // Create output directory
+  const outputDir = join(tempDir, 'output');
+  mkdirSync(outputDir, { recursive: true });
+
+  // Build Docker image
+  console.log("Building Docker image for Debian 11");
+  await $`docker build -t lz4-builder:debian11 -f ${join(__dirname, 'Dockerfile.debian11')} --build-arg LZ4_VERSION=${LZ4_VERSION} ${__dirname}`;
+
+  // Run container to extract binary
+  console.log("Extracting LZ4 binary from container");
+  await $`docker run --rm -v ${outputDir}:/output lz4-builder:debian11 cp /build/lz4-${LZ4_VERSION}/lz4 /output/`;
+
+  console.log("LZ4 build complete");
+
+  // Upload to S3
+  console.log("Uploading LZ4 binary to S3");
+  
+  // Upload the binary
+  console.log(`Uploading to s3://rivet-releases/${UPLOAD_PATH}`);
+  await $({
+	env: {
+	  ...process.env,
+	  AWS_ACCESS_KEY_ID: awsAccessKeyId,
+	  AWS_SECRET_ACCESS_KEY: awsSecretAccessKey,
+	  AWS_DEFAULT_REGION: "auto",
+	}
+  })`aws s3 cp ${join(outputDir, 'lz4')} ${`s3://rivet-releases/${UPLOAD_PATH}`} --content-type application/octet-stream --endpoint-url https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com`;
+  
+  console.log("Upload complete!");
+}
+
+main().catch(error => {
+  console.error(`Error: ${error}`);
+  process.exit(1);
+});
diff --git a/scripts/lz4/package.json b/scripts/lz4/package.json
new file mode 100644
index 0000000000..b5457f2343
--- /dev/null
+++ b/scripts/lz4/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "@scripts/lz4",
+  "version": "0.0.0",
+  "private": true,
+  "main": "distribute.ts",
+  "type": "module",
+  "scripts": {
+    "distribute": "tsx distribute.ts"
+  },
+  "dependencies": {
+    "zx": "^8.5.0"
+  },
+  "devDependencies": {
+    "tsx": "^4.19.3"
+  }
+}