From a37ec31f5dafe252b358bf667653b0e9f3131c2c Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Wed, 27 Aug 2025 09:10:05 -0700 Subject: [PATCH 01/19] Added local docker deployment --- Makefile | 106 +++++++++++++++++++++++++++++++++++++++++++++ build-local.sh | 50 +++++++++++++++++++++ docker-compose.yml | 52 ++++++++++++++++++++++ run-local.sh | 58 +++++++++++++++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 Makefile create mode 100755 build-local.sh create mode 100644 docker-compose.yml create mode 100755 run-local.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9fdfcde --- /dev/null +++ b/Makefile @@ -0,0 +1,106 @@ +# Makefile for kernel-browser local development +# Using kernel-images native build system + +.PHONY: help build run stop logs clean dev status shell test + +# Default target +help: ## Show this help message + @echo "Kernel Browser - Local Development (using kernel-images build system)" + @echo "==================================================================" + @echo "" + @echo "Available commands:" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-15s %s\n", $$1, $$2}' + +init: ## Initialize submodules (run this first) + git submodule update --init --recursive + @echo "โœ… Submodules initialized" + +build: init ## Build using kernel-images build system + @echo "๐Ÿ”จ Building with kernel-images native build system..." + ./build-local.sh + @echo "โœ… Build complete" + +run: ## Run using kernel-images run system (interactive) + @echo "๐Ÿš€ Starting kernel-browser using native run script..." + ./run-local.sh + +compose-up: build ## Start with docker-compose (background) + @echo "๐Ÿš€ Starting with docker-compose..." + docker-compose up -d + @$(MAKE) --no-print-directory info + @echo "" + @echo "๐Ÿ“Š View logs with: make logs" + +compose-dev: build ## Start with docker-compose (foreground with logs) + @echo "๐Ÿš€ Starting with docker-compose in development mode..." + docker-compose up + +dev: compose-dev ## Alias for compose-dev + +stop: ## Stop all containers + @echo "๐Ÿ›‘ Stopping containers..." + docker-compose down + docker stop kernel-browser-local 2>/dev/null || true + docker rm kernel-browser-local 2>/dev/null || true + @echo "โœ… Containers stopped" + +restart: ## Restart containers + @$(MAKE) --no-print-directory stop + @$(MAKE) --no-print-directory compose-up + +logs: ## Show container logs + docker-compose logs -f kernel-browser || docker logs -f kernel-browser-local + +status: ## Show container status + @echo "Docker Compose Status:" + @docker-compose ps || true + @echo "" + @echo "Direct Container Status:" + @docker ps --filter name=kernel-browser + +shell: ## Get shell access to running container + docker exec -it kernel-browser-local bash || docker-compose exec kernel-browser bash + +info: ## Show connection information + @echo "" + @echo "๐ŸŒ Service Access Points:" + @echo " WebRTC Client: http://localhost:8080" + @echo " Chrome DevTools: http://localhost:9222/json" + @echo " Recording API: http://localhost:444/api" + @echo " Health Check: http://localhost:8080/" + +test: ## Test service endpoints + @echo "๐Ÿงช Testing service endpoints..." + @echo -n "WebRTC Client (8080): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/ || echo "Failed to connect" + @echo "" + @echo -n "Chrome DevTools (9222): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:9222/json/version || echo "Failed to connect" + @echo "" + @echo -n "Recording API (444): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:444/ && echo " (404 is normal - API is running)" || echo "Failed to connect" + @echo "" + @echo "๐ŸŽฏ All services are ready! Access points:" + @echo " WebRTC Client: http://localhost:8080" + @echo " Chrome DevTools: http://localhost:9222/json" + +clean: stop ## Clean up everything + @echo "๐Ÿงน Cleaning up..." + docker-compose down -v 2>/dev/null || true + docker rmi kernel-browser:local 2>/dev/null || true + docker system prune -f + rm -rf recordings/* 2>/dev/null || true + rm -rf kernel-images/images/chromium-headful/.tmp 2>/dev/null || true + @echo "โœ… Cleanup complete" + +# Alternative commands for different approaches +native-build: init ## Build using kernel-images native script directly + cd kernel-images/images/chromium-headful && \ + UKC_TOKEN=dummy-token UKC_METRO=dummy-metro IMAGE=kernel-browser:local ./build-docker.sh + +native-run: ## Run using kernel-images native script directly + cd kernel-images/images/chromium-headful && \ + UKC_TOKEN=dummy-token UKC_METRO=dummy-metro IMAGE=kernel-browser:local NAME=kernel-browser-local ENABLE_WEBRTC=true ./run-docker.sh + +# Quick development workflow +quick: init build compose-up test ## Quick setup: init + build + run + test \ No newline at end of file diff --git a/build-local.sh b/build-local.sh new file mode 100755 index 0000000..975196e --- /dev/null +++ b/build-local.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Local build wrapper for kernel-images chromium-headful +set -e -o pipefail + +echo "๐Ÿ”จ Building kernel-browser using kernel-images build system..." + +# Ensure we're in the right directory +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +cd "$SCRIPT_DIR" + +# Check if kernel-images submodule exists and is initialized +if [ ! -d "kernel-images" ]; then + echo "โŒ Error: kernel-images submodule not found" + echo " Run: git submodule update --init --recursive" + exit 1 +fi + +if [ ! -f "kernel-images/images/chromium-headful/build-docker.sh" ]; then + echo "โŒ Error: kernel-images submodule appears empty" + echo " Run: git submodule update --init --recursive" + exit 1 +fi + +# Change to kernel-images directory and build using their system +echo "๐Ÿ“ Changing to kernel-images directory..." +cd kernel-images/images/chromium-headful + +# Make build script executable +chmod +x build-docker.sh + +# Set image name for local use +export IMAGE="kernel-browser:local" +export NAME="kernel-browser-local" + +# Set dummy UKC variables to bypass cloud requirements (we only need Docker) +export UKC_TOKEN="dummy-token-for-local-build" +export UKC_METRO="dummy-metro-for-local-build" + +echo "๐Ÿš€ Starting build with kernel-images build system..." +echo " Image: $IMAGE" +echo " Bypassing UKC requirements for local Docker build..." + +# Run the official build script +./build-docker.sh + +echo "โœ… Build completed successfully!" +echo " Image built: $IMAGE" +echo "" +echo "๐Ÿƒ To run locally, use: ./run-local.sh" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..42a2744 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3.8' + +services: + kernel-browser: + image: "kernel-browser:local" + container_name: "kernel-browser-local" + privileged: true + shm_size: 2gb + deploy: + resources: + limits: + memory: 8192M + ports: + # Chrome DevTools Protocol (matches kernel-images default) + - "9222:9222" + # Recording API (matches kernel-images default) + - "444:10001" + # WebRTC client interface + - "8080:8080" + # WebRTC UDP port range for local development + - "56000-56100:56000-56100/udp" + environment: + # Display settings + - DISPLAY_NUM=1 + - HEIGHT=768 + - WIDTH=1024 + # WebRTC settings + - ENABLE_WEBRTC=true + - NEKO_WEBRTC_EPR=56000-56100 + - NEKO_WEBRTC_NAT1TO1=127.0.0.1 + # Run as kernel user (not root) + - RUN_AS_ROOT=false + # Mount Chromium flags + - CHROMIUM_FLAGS=--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --start-maximized --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox + volumes: + # Persist recordings in local directory + - "./recordings:/recordings" + # Mount Chromium flags file (will be created by run script) + - "./kernel-images/images/chromium-headful/.tmp/chromium/flags:/chromium/flags:ro" + tmpfs: + - /dev/shm:size=2g + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 120s # Allow more time for startup + +volumes: + recordings: + driver: local \ No newline at end of file diff --git a/run-local.sh b/run-local.sh new file mode 100755 index 0000000..cfab5c4 --- /dev/null +++ b/run-local.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Local run wrapper for kernel-images chromium-headful +set -e -o pipefail + +echo "๐Ÿš€ Starting kernel-browser locally using kernel-images run system..." + +# Ensure we're in the right directory +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +cd "$SCRIPT_DIR" + +# Check if kernel-images submodule exists +if [ ! -d "kernel-images" ] || [ ! -f "kernel-images/images/chromium-headful/run-docker.sh" ]; then + echo "โŒ Error: kernel-images submodule not found or incomplete" + echo " Run: git submodule update --init --recursive" + exit 1 +fi + +# Create local recordings directory +mkdir -p "$SCRIPT_DIR/recordings" + +# Change to kernel-images directory +cd kernel-images/images/chromium-headful + +# Make run script executable +chmod +x run-docker.sh + +# Set environment variables for local development +export IMAGE="kernel-browser:local" +export NAME="kernel-browser-local" +export ENABLE_WEBRTC="true" +export RUN_AS_ROOT="false" + +# Set dummy UKC variables to bypass cloud requirements (we only need Docker) +export UKC_TOKEN="dummy-token-for-local-run" +export UKC_METRO="dummy-metro-for-local-run" + +# Local-friendly Chrome flags (less restrictive than cloud) +export CHROMIUM_FLAGS="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --start-maximized --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox" + +echo "๐Ÿ”ง Configuration:" +echo " Image: $IMAGE" +echo " Container: $NAME" +echo " WebRTC: $ENABLE_WEBRTC" +echo " Run as root: $RUN_AS_ROOT" +echo " Recordings: $SCRIPT_DIR/recordings" +echo "" + +echo "๐Ÿƒ Starting container with kernel-images run system..." + +# Run using the official run script +./run-docker.sh + +echo "" +echo "๐ŸŒ Service should be accessible at:" +echo " WebRTC Client: http://localhost:8080" +echo " Chrome DevTools: http://localhost:9222" +echo " Recording API: http://localhost:444" \ No newline at end of file From a21844bcad8f170270c5245329fd16f3d3143acd Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Sun, 31 Aug 2025 11:13:52 -0700 Subject: [PATCH 02/19] Neko + Devtools run but separately --- Dockerfile.devtools | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Dockerfile.devtools diff --git a/Dockerfile.devtools b/Dockerfile.devtools new file mode 100644 index 0000000..edefa5e --- /dev/null +++ b/Dockerfile.devtools @@ -0,0 +1,63 @@ +# DevTools Frontend build stage using browser-operator-core +FROM --platform=linux/amd64 ubuntu:22.04 AS devtools-builder + +# Install required packages for DevTools frontend build +RUN apt-get update && apt-get install -y \ + curl \ + git \ + python3 \ + python3-pip \ + python-is-python3 \ + wget \ + unzip \ + sudo \ + ca-certificates \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js 18.x +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + apt-get install -y nodejs && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +# Clone depot_tools +RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +ENV PATH="/workspace/depot_tools:${PATH}" +ENV DEPOT_TOOLS_UPDATE=0 + +# Follow README instructions exactly - fetching code +RUN mkdir devtools +WORKDIR /workspace/devtools +RUN fetch devtools-frontend + +# Build steps +WORKDIR /workspace/devtools/devtools-frontend + +RUN gclient sync +RUN /workspace/depot_tools/ensure_bootstrap + +# Build standard DevTools first +RUN npm run build + +# Add Browser Operator fork and switch to it +RUN git remote add upstream https://github.com/BrowserOperator/browser-operator-core.git +RUN git fetch upstream +RUN git checkout upstream/main + +# Build Browser Operator version +RUN npm run build + +# Production stage for DevTools frontend +FROM nginx:alpine AS devtools-frontend +WORKDIR /usr/share/nginx/html + +# Copy the built DevTools frontend from builder +COPY --from=devtools-builder /workspace/devtools/devtools-frontend/out/Default/gen/front_end . + +# Copy nginx config from browser-operator-core +COPY browser-operator-core/docker/nginx.conf /etc/nginx/conf.d/default.conf + +# Create health check endpoint +RUN echo '{"status": "healthy", "service": "browser-operator-devtools"}' > health.json \ No newline at end of file From 50a4a21efae2b7609553bb1a497da867b06d39df Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Sun, 31 Aug 2025 11:55:04 -0700 Subject: [PATCH 03/19] Local setup running with devtools --- Dockerfile.local-extended | 269 ++++++++++++++++++++++++ Makefile | 60 +++++- nginx-devtools.conf | 79 +++++++ run-local-extended.sh | 71 +++++++ supervisor/services/nginx-devtools.conf | 15 ++ 5 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.local-extended create mode 100644 nginx-devtools.conf create mode 100755 run-local-extended.sh create mode 100644 supervisor/services/nginx-devtools.conf diff --git a/Dockerfile.local-extended b/Dockerfile.local-extended new file mode 100644 index 0000000..0bb0ada --- /dev/null +++ b/Dockerfile.local-extended @@ -0,0 +1,269 @@ +# Extended Dockerfile combining kernel-images with DevTools frontend +# This extends the kernel-images base with Browser Operator DevTools static files + +# DevTools Frontend build stage using browser-operator-core +FROM --platform=linux/amd64 ubuntu:22.04 AS devtools-builder + +# Install required packages for DevTools frontend build +RUN apt-get update && apt-get install -y \ + curl \ + git \ + python3 \ + python3-pip \ + python-is-python3 \ + wget \ + unzip \ + sudo \ + ca-certificates \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js 18.x +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + apt-get install -y nodejs && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +# Clone depot_tools +RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +ENV PATH="/workspace/depot_tools:${PATH}" +ENV DEPOT_TOOLS_UPDATE=0 + +# Follow README instructions exactly - fetching code +RUN mkdir devtools +WORKDIR /workspace/devtools +RUN fetch devtools-frontend + +# Build steps +WORKDIR /workspace/devtools/devtools-frontend + +RUN gclient sync +RUN /workspace/depot_tools/ensure_bootstrap + +# Build standard DevTools first +RUN npm run build + +# Add Browser Operator fork and switch to it +RUN git remote add upstream https://github.com/BrowserOperator/browser-operator-core.git +RUN git fetch upstream +RUN git checkout upstream/main + +# Build Browser Operator version +RUN npm run build + +# ============================================================================ +# Use kernel-images base with DevTools integration +# ============================================================================ +FROM docker.io/golang:1.25.0 AS server-builder +WORKDIR /workspace/server + +ARG TARGETOS +ARG TARGETARCH +ENV CGO_ENABLED=0 + +COPY kernel-images/server/go.mod ./ +COPY kernel-images/server/go.sum ./ +RUN go mod download + +COPY kernel-images/server/ . +RUN GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ + go build -ldflags="-s -w" -o /out/kernel-images-api ./cmd/api + +# webrtc client +FROM node:22-bullseye-slim AS client +WORKDIR /src +COPY kernel-images/images/chromium-headful/client/package*.json ./ +RUN npm install +COPY kernel-images/images/chromium-headful/client/ . +RUN npm run build + +# xorg dependencies +FROM docker.io/ubuntu:22.04 AS xorg-deps +WORKDIR /xorg +ENV DEBIAN_FRONTEND=noninteractive +RUN set -eux; \ + apt-get update; \ + apt-get install -y \ + git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev \ + && rm -rf /var/lib/apt/lists/*; +COPY kernel-images/images/chromium-headful/xorg-deps/ /xorg/ +# build xf86-video-dummy v0.3.8 with RandR support +RUN set -eux; \ + cd xf86-video-dummy/v0.3.8; \ + patch -p1 < ../01_v0.3.8_xdummy-randr.patch; \ + autoreconf -v --install; \ + ./configure; \ + make -j$(nproc); \ + make install; +# build custom input driver +RUN set -eux; \ + cd xf86-input-neko; \ + ./autogen.sh --prefix=/usr; \ + ./configure; \ + make -j$(nproc); \ + make install; + +FROM ghcr.io/onkernel/neko/base:3.0.6-v1.0.1 AS neko +# ^--- now has event.SYSTEM_PONG with legacy support to keepalive + +# Final stage: kernel-images base + DevTools static files +FROM docker.io/ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_PRIORITY=high + +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install \ + # UI Requirements + xvfb \ + xterm \ + xdotool \ + scrot \ + imagemagick \ + sudo \ + mutter \ + # Python/pyenv reqs + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + curl \ + git \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libffi-dev \ + liblzma-dev \ + # Network tools + net-tools \ + netcat \ + # PPA req + software-properties-common \ + # Add nginx for DevTools serving + nginx && \ + # Userland apps + sudo add-apt-repository ppa:mozillateam/ppa && \ + sudo apt-get install -y --no-install-recommends \ + chromium-browser \ + libreoffice \ + x11-apps \ + xpdf \ + gedit \ + xpaint \ + tint2 \ + galculator \ + pcmanfm \ + wget \ + xdg-utils \ + libvulkan1 \ + fonts-liberation \ + unzip && \ + apt-get clean + +# install ffmpeg manually since the version available in apt is from the 4.x branch due to #drama. +# as of writing these static builds will be the latest 7.0.x release. +RUN set -eux; \ + URL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz"; \ + echo "Downloading FFmpeg static build from $URL"; \ + curl -fsSL "$URL" -o /tmp/ffmpeg.tar.xz; \ + tar -xJf /tmp/ffmpeg.tar.xz -C /tmp; \ + install -m755 /tmp/ffmpeg-*/ffmpeg /usr/local/bin/ffmpeg; \ + install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe; \ + rm -rf /tmp/ffmpeg* + +# runtime +ENV USERNAME=root +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + wget ca-certificates python2 supervisor xclip xdotool \ + pulseaudio dbus-x11 xserver-xorg-video-dummy \ + libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx7 \ + gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ + gstreamer1.0-pulseaudio gstreamer1.0-omx; \ + # + # install libxcvt0 (not available in debian:bullseye) + ARCH=$(dpkg --print-architecture); \ + wget http://ftp.de.debian.org/debian/pool/main/libx/libxcvt/libxcvt0_0.1.2-1_${ARCH}.deb; \ + apt-get install --no-install-recommends ./libxcvt0_0.1.2-1_${ARCH}.deb; \ + rm ./libxcvt0_0.1.2-1_${ARCH}.deb; \ + # + # workaround for an X11 problem: http://blog.tigerteufel.de/?p=476 + mkdir /tmp/.X11-unix; \ + chmod 1777 /tmp/.X11-unix; \ + chown $USERNAME /tmp/.X11-unix/; \ + # + # make directories for neko + mkdir -p /etc/neko /var/www /var/log/neko \ + /tmp/runtime-$USERNAME \ + /home/$USERNAME/.config/pulse \ + /home/$USERNAME/.local/share/xorg; \ + chmod 1777 /var/log/neko; \ + chown $USERNAME /var/log/neko/ /tmp/runtime-$USERNAME; \ + chown -R $USERNAME:$USERNAME /home/$USERNAME; \ + # clean up + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/ + +# install chromium and sqlite3 for debugging the cookies file +RUN add-apt-repository -y ppa:xtradeb/apps +RUN apt update -y && apt install -y chromium sqlite3 + +# setup desktop env & app +ENV DISPLAY_NUM=1 +ENV HEIGHT=768 +ENV WIDTH=1024 +ENV WITHDOCKER=true + +# Copy kernel-images configuration and binaries +COPY kernel-images/images/chromium-headful/xorg.conf /etc/neko/xorg.conf +COPY kernel-images/images/chromium-headful/neko.yaml /etc/neko/neko.yaml +COPY --from=neko /usr/bin/neko /usr/bin/neko +COPY --from=client /src/dist/ /var/www +COPY --from=xorg-deps /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so +COPY --from=xorg-deps /usr/local/lib/xorg/modules/input/neko_drv.so /usr/lib/xorg/modules/input/neko_drv.so + +COPY kernel-images/images/chromium-headful/image-chromium/ / +COPY kernel-images/images/chromium-headful/start-chromium.sh /images/chromium-headful/start-chromium.sh +RUN chmod +x /images/chromium-headful/start-chromium.sh +COPY kernel-images/images/chromium-headful/wrapper.sh /wrapper.sh +COPY kernel-images/images/chromium-headful/supervisord.conf /etc/supervisor/supervisord.conf +COPY kernel-images/images/chromium-headful/supervisor/services/ /etc/supervisor/conf.d/services/ + +# copy the kernel-images API binary built in the builder stage +COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api + +# ============================================================================ +# DevTools Integration +# ============================================================================ + +# Copy DevTools static files from builder +COPY --from=devtools-builder /workspace/devtools/devtools-frontend/out/Default/gen/front_end /usr/share/nginx/devtools + +# Create DevTools nginx configuration +COPY nginx-devtools.conf /etc/nginx/sites-available/devtools +RUN ln -s /etc/nginx/sites-available/devtools /etc/nginx/sites-enabled/devtools && \ + rm /etc/nginx/sites-enabled/default + +# Add DevTools nginx service to supervisor +COPY supervisor/services/nginx-devtools.conf /etc/supervisor/conf.d/services/nginx-devtools.conf + +# Create nginx temp directories and set permissions +RUN mkdir -p /var/lib/nginx/body \ + /var/lib/nginx/proxy \ + /var/lib/nginx/fastcgi \ + /var/lib/nginx/uwsgi \ + /var/lib/nginx/scgi && \ + chown -R www-data:www-data /var/lib/nginx && \ + chown -R www-data:www-data /usr/share/nginx/devtools + +RUN useradd -m -s /bin/bash kernel + +ENTRYPOINT [ "/wrapper.sh" ] \ No newline at end of file diff --git a/Makefile b/Makefile index 9fdfcde..e0415bc 100644 --- a/Makefile +++ b/Makefile @@ -103,4 +103,62 @@ native-run: ## Run using kernel-images native script directly UKC_TOKEN=dummy-token UKC_METRO=dummy-metro IMAGE=kernel-browser:local NAME=kernel-browser-local ENABLE_WEBRTC=true ./run-docker.sh # Quick development workflow -quick: init build compose-up test ## Quick setup: init + build + run + test \ No newline at end of file +quick: init build compose-up test ## Quick setup: init + build + run + test + +# ============================================================================ +# Extended targets with DevTools frontend +# ============================================================================ + +build-extended: init ## Build extended image with DevTools frontend + @echo "๐Ÿ”จ Building extended kernel-browser with DevTools frontend..." + docker build -f Dockerfile.local-extended -t kernel-browser:extended . + @echo "โœ… Extended build complete" + +run-extended: ## Run extended container with DevTools (interactive) + @echo "๐Ÿš€ Starting extended kernel-browser with DevTools..." + ./run-local-extended.sh + +info-extended: ## Show extended connection information + @echo "" + @echo "๐ŸŒ Extended Service Access Points:" + @echo " WebRTC Client: http://localhost:8080" + @echo " Chrome DevTools: http://localhost:9222/json" + @echo " Recording API: http://localhost:444/api" + @echo " Enhanced DevTools UI: http://localhost:8001" + @echo " DevTools Health: http://localhost:8001/health" + +test-extended: ## Test extended service endpoints including DevTools + @echo "๐Ÿงช Testing extended service endpoints..." + @echo -n "WebRTC Client (8080): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/ || echo "Failed to connect" + @echo "" + @echo -n "Chrome DevTools (9222): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:9222/json/version || echo "Failed to connect" + @echo "" + @echo -n "Recording API (444): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:444/ && echo " (404 is normal - API is running)" || echo "Failed to connect" + @echo "" + @echo -n "DevTools UI (8001): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:8001/ || echo "Failed to connect" + @echo "" + @echo -n "DevTools Health (8001): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:8001/health || echo "Failed to connect" + @echo "" + @echo "๐ŸŽฏ All extended services are ready! Access points:" + @echo " WebRTC Client: http://localhost:8080" + @echo " Chrome DevTools: http://localhost:9222/json" + @echo " Enhanced DevTools UI: http://localhost:8001" + +stop-extended: ## Stop extended container + @echo "๐Ÿ›‘ Stopping extended containers..." + docker stop kernel-browser-extended 2>/dev/null || true + docker rm kernel-browser-extended 2>/dev/null || true + @echo "โœ… Extended containers stopped" + +clean-extended: stop-extended ## Clean up extended containers and images + @echo "๐Ÿงน Cleaning up extended resources..." + docker rmi kernel-browser:extended 2>/dev/null || true + @echo "โœ… Extended cleanup complete" + +# Extended workflow +quick-extended: init build-extended run-extended ## Quick extended setup: init + build + run with DevTools \ No newline at end of file diff --git a/nginx-devtools.conf b/nginx-devtools.conf new file mode 100644 index 0000000..42db1a1 --- /dev/null +++ b/nginx-devtools.conf @@ -0,0 +1,79 @@ +server { + listen 8001; + listen [::]:8001; + server_name localhost; + + # Root directory for DevTools frontend + root /usr/share/nginx/devtools; + index inspector.html devtools_app.html index.html; + + # Compression + gzip on; + gzip_vary on; + gzip_min_length 1024; + gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json application/wasm; + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + + # CORS headers for DevTools + add_header Access-Control-Allow-Origin "*" always; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; + add_header Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" always; + + # Handle OPTIONS requests + if ($request_method = 'OPTIONS') { + return 204; + } + + # Cache control for static assets + location ~* \.(jpg|jpeg|png|gif|ico|css|js|svg|woff|woff2|ttf|eot|avif)$ { + expires 1d; + add_header Cache-Control "public, immutable"; + } + + # Specific handling for WebAssembly files + location ~ \.wasm$ { + add_header Content-Type application/wasm; + } + + # JSON files + location ~ \.json$ { + add_header Content-Type application/json; + } + + # Main location + location / { + try_files $uri $uri/ /index.html; + } + + # Specific paths for DevTools + location /front_end/ { + alias /usr/share/nginx/devtools/; + try_files $uri $uri/ =404; + } + + # Health check for DevTools service + location /health { + access_log off; + add_header Content-Type application/json; + return 200 '{"status": "healthy", "service": "devtools-frontend"}'; + } + + # Error pages + error_page 404 /404.html; + location = /404.html { + internal; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + internal; + } + + # Logging + access_log /var/log/nginx/devtools-access.log; + error_log /var/log/nginx/devtools-error.log warn; +} \ No newline at end of file diff --git a/run-local-extended.sh b/run-local-extended.sh new file mode 100755 index 0000000..dc4cbe0 --- /dev/null +++ b/run-local-extended.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Extended local run wrapper for kernel-images chromium-headful + DevTools +set -e -o pipefail + +echo "๐Ÿš€ Starting kernel-browser (EXTENDED) locally using kernel-images run system..." + +# Ensure we're in the right directory +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +cd "$SCRIPT_DIR" + +# Check if kernel-images submodule exists +if [ ! -d "kernel-images" ] || [ ! -f "kernel-images/images/chromium-headful/run-docker.sh" ]; then + echo "โŒ Error: kernel-images submodule not found or incomplete" + echo " Run: git submodule update --init --recursive" + exit 1 +fi + +# Create local recordings directory +mkdir -p "$SCRIPT_DIR/recordings" + +# Change to kernel-images directory +cd kernel-images/images/chromium-headful + +# Make run script executable +chmod +x run-docker.sh + +# Set environment variables for extended local development +export IMAGE="kernel-browser:extended" +export NAME="kernel-browser-extended" +export ENABLE_WEBRTC="true" +export RUN_AS_ROOT="false" + +# Set dummy UKC variables to bypass cloud requirements (we only need Docker) +export UKC_TOKEN="dummy-token-for-local-run" +export UKC_METRO="dummy-metro-for-local-run" + +# Local-friendly Chrome flags (less restrictive than cloud) + custom DevTools frontend +export CHROMIUM_FLAGS="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --start-maximized --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --custom-devtools-frontend=http://localhost:8001/" + +echo "๐Ÿ”ง Configuration:" +echo " Image: $IMAGE" +echo " Container: $NAME" +echo " WebRTC: $ENABLE_WEBRTC" +echo " DevTools UI: enabled" +echo " Run as root: $RUN_AS_ROOT" +echo " Recordings: $SCRIPT_DIR/recordings" +echo "" + +echo "๐Ÿƒ Starting extended container with kernel-images run system..." + +# Backup original run-docker.sh to modify port mappings +if [ ! -f run-docker.sh.original ]; then + cp run-docker.sh run-docker.sh.original +fi + +# Create modified run script that adds DevTools port mapping +cat run-docker.sh.original | \ +sed 's/docker run -it/docker run -it -p 8001:8001/' > run-docker.sh.extended + +chmod +x run-docker.sh.extended + +# Run using the modified run script with DevTools port +./run-docker.sh.extended + +echo "" +echo "๐ŸŒ Extended service should be accessible at:" +echo " WebRTC Client: http://localhost:8080" +echo " Chrome DevTools: http://localhost:9222" +echo " Recording API: http://localhost:444" +echo " Enhanced DevTools UI: http://localhost:8001" \ No newline at end of file diff --git a/supervisor/services/nginx-devtools.conf b/supervisor/services/nginx-devtools.conf new file mode 100644 index 0000000..442cf08 --- /dev/null +++ b/supervisor/services/nginx-devtools.conf @@ -0,0 +1,15 @@ +[program:nginx-devtools] +command=nginx -g 'daemon off;' +autostart=true +autorestart=true +startretries=3 +user=root +stdout_logfile=/var/log/nginx-devtools-stdout.log +stderr_logfile=/var/log/nginx-devtools-stderr.log +stdout_logfile_maxbytes=10MB +stderr_logfile_maxbytes=10MB +stdout_logfile_backups=3 +stderr_logfile_backups=3 +redirect_stderr=false +killasgroup=true +stopasgroup=true \ No newline at end of file From 0ad4e72cb6bfdd08cf089da8e2f40ca1b9d0fd9c Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Tue, 2 Sep 2025 15:15:59 -0500 Subject: [PATCH 04/19] Custom Devtools working on CloudRun --- Dockerfile.cloudrun | 87 ++++++++++++++++++-- cloudbuild.yaml | 18 ++--- cloudrun-wrapper.sh | 148 ++++++++++++++++++++++++++++++++--- nginx-devtools-cloudrun.conf | 105 +++++++++++++++++++++++++ nginx.conf | 16 +++- service.yaml | 14 ++-- 6 files changed, 355 insertions(+), 33 deletions(-) create mode 100644 nginx-devtools-cloudrun.conf diff --git a/Dockerfile.cloudrun b/Dockerfile.cloudrun index 8c3e812..78776be 100644 --- a/Dockerfile.cloudrun +++ b/Dockerfile.cloudrun @@ -1,5 +1,56 @@ +# DevTools Frontend build stage using browser-operator-core +FROM --platform=linux/amd64 ubuntu:22.04 AS devtools-builder + +# Install required packages for DevTools frontend build +RUN apt-get update && apt-get install -y \ + curl \ + git \ + python3 \ + python3-pip \ + python-is-python3 \ + wget \ + unzip \ + sudo \ + ca-certificates \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js 18.x +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \ + apt-get install -y nodejs && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +# Clone depot_tools +RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +ENV PATH="/workspace/depot_tools:${PATH}" +ENV DEPOT_TOOLS_UPDATE=0 + +# Follow README instructions exactly - fetching code +RUN mkdir devtools +WORKDIR /workspace/devtools +RUN fetch devtools-frontend + +# Build steps +WORKDIR /workspace/devtools/devtools-frontend + +RUN gclient sync +RUN /workspace/depot_tools/ensure_bootstrap + +# Build standard DevTools first +RUN npm run build + +# Add Browser Operator fork and switch to it +RUN git remote add upstream https://github.com/BrowserOperator/browser-operator-core.git +RUN git fetch upstream +RUN git checkout upstream/main + +# Build Browser Operator version +RUN npm run build + # Multi-stage build using kernel-images as base -FROM docker.io/golang:1.25.0 AS server-builder +FROM docker.io/golang:1.23.0 AS server-builder WORKDIR /workspace/server ARG TARGETOS @@ -90,6 +141,12 @@ RUN apt-get update && \ nginx \ # PPA req software-properties-common && \ + # Disable nginx auto-start to prevent conflicts with custom config + systemctl disable nginx || true && \ + systemctl mask nginx || true && \ + # Remove default nginx config to prevent conflicts + rm -f /etc/nginx/sites-enabled/default && \ + rm -f /etc/nginx/nginx.conf && \ # Userland apps sudo add-apt-repository ppa:mozillateam/ppa && \ sudo apt-get install -y --no-install-recommends \ @@ -186,19 +243,39 @@ COPY kernel-images/images/chromium-headful/supervisor/services/ /etc/supervisor/ # Copy the kernel-images API binary COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api -# Cloud Run specific: nginx configuration for port proxying -COPY nginx.conf /etc/nginx/nginx.conf +# ============================================================================ +# DevTools Integration +# ============================================================================ + +# Copy DevTools static files from builder +COPY --from=devtools-builder /workspace/devtools/devtools-frontend/out/Default/gen/front_end /usr/share/nginx/devtools + +# Set permissions for DevTools files +RUN chown -R kernel:kernel /usr/share/nginx/devtools + +# Cloud Run specific: wrapper script only (nginx config is inline) +# DO NOT copy nginx.conf to avoid auto-start conflicts COPY cloudrun-wrapper.sh /cloudrun-wrapper.sh RUN chmod +x /cloudrun-wrapper.sh +# Add essential services for neko WebRTC and Chromium +COPY supervisor/services-cloudrun/dbus.conf /etc/supervisor/conf.d/services-cloudrun/dbus.conf +COPY supervisor/services-cloudrun/xorg.conf /etc/supervisor/conf.d/services-cloudrun/xorg.conf +COPY supervisor/services-cloudrun/neko.conf /etc/supervisor/conf.d/services-cloudrun/neko.conf +COPY supervisor/services-cloudrun/chromium.conf /etc/supervisor/conf.d/services-cloudrun/chromium.conf +COPY supervisor/services-cloudrun/devtools-frontend.conf /etc/supervisor/conf.d/services-cloudrun/devtools-frontend.conf + # Create nginx temp directories for non-root execution RUN mkdir -p /tmp/nginx_client_temp /tmp/nginx_proxy_temp /tmp/nginx_fastcgi_temp \ - /tmp/nginx_uwsgi_temp /tmp/nginx_scgi_temp && \ + /tmp/nginx_uwsgi_temp /tmp/nginx_scgi_temp \ + /tmp/nginx_devtools_client_temp /tmp/nginx_devtools_proxy_temp /tmp/nginx_devtools_fastcgi_temp \ + /tmp/nginx_devtools_uwsgi_temp /tmp/nginx_devtools_scgi_temp && \ chown -R kernel:kernel /tmp/nginx_* # Create supervisor log directories RUN mkdir -p /var/log/supervisord/chromium /var/log/supervisord/neko /var/log/supervisord/xorg \ - /var/log/supervisord/dbus /var/log/supervisord/kernel-images-api /var/log/supervisord/mutter && \ + /var/log/supervisord/dbus /var/log/supervisord/kernel-images-api /var/log/supervisord/mutter \ + /var/log/supervisord/nginx /var/log/supervisord/devtools-frontend && \ chown -R kernel:kernel /var/log/supervisord # Create health check endpoint diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 37dac28..3470f89 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -18,26 +18,26 @@ steps: - '-c' - | echo "Attempting to pull previous image for caching..." - docker pull gcr.io/$PROJECT_ID/kernel-browser:latest || echo "No previous image found for caching" + docker pull us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest || echo "No previous image found for caching" - # Step 3: Build the Docker image with caching (using kernel-cloud Dockerfile) + # Step 3: Build the Docker image with caching (using cloudrun Dockerfile) - name: 'gcr.io/cloud-builders/docker' args: - 'build' - '--file' - - 'Dockerfile.kernel-cloud' + - 'Dockerfile.cloudrun' - '--cache-from' - - 'gcr.io/$PROJECT_ID/kernel-browser:latest' + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' - '--tag' - - 'gcr.io/$PROJECT_ID/kernel-browser:latest' + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' - '.' timeout: '3600s' # Allow 1 hour for build (it's a large image) - # Step 4: Push the image to Google Container Registry + # Step 4: Push the image to Artifact Registry - name: 'gcr.io/cloud-builders/docker' args: - 'push' - - 'gcr.io/$PROJECT_ID/kernel-browser:latest' + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' # Step 5: Update the service.yaml with the correct project ID - name: 'gcr.io/cloud-builders/gcloud' @@ -89,9 +89,9 @@ options: # Allocate disk space for the large build diskSizeGb: 100 -# Images to be pushed to Container Registry +# Images to be pushed to Artifact Registry images: - - 'gcr.io/$PROJECT_ID/kernel-browser:latest' + - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' # Tags for organization diff --git a/cloudrun-wrapper.sh b/cloudrun-wrapper.sh index a62fdf6..8e91d01 100644 --- a/cloudrun-wrapper.sh +++ b/cloudrun-wrapper.sh @@ -11,23 +11,23 @@ export ENABLE_WEBRTC=true export DISPLAY_NUM=1 export HEIGHT=768 export WIDTH=1024 +export NEKO_BIND=:8081 # Port configuration for Cloud Run export PORT=${PORT:-8080} -export CHROMIUM_FLAGS="${CHROMIUM_FLAGS:---user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor}" +export CHROMIUM_FLAGS="${CHROMIUM_FLAGS:---user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor --custom-devtools-frontend=http://localhost:8001/ https://www.google.com}" # Setup directories with proper permissions mkdir -p /tmp/nginx_client_temp /tmp/nginx_proxy_temp /tmp/nginx_fastcgi_temp \ /tmp/nginx_uwsgi_temp /tmp/nginx_scgi_temp \ + /tmp/nginx_devtools_client_temp /tmp/nginx_devtools_proxy_temp /tmp/nginx_devtools_fastcgi_temp \ + /tmp/nginx_devtools_uwsgi_temp /tmp/nginx_devtools_scgi_temp \ /home/kernel/user-data /home/kernel/.config /home/kernel/.cache \ - /tmp/runtime-kernel /var/log/neko /tmp/recordings + /tmp/runtime-kernel /var/log/neko /tmp/recordings \ + /tmp/supervisord /tmp/dbus -# Test nginx configuration -echo "[cloudrun-wrapper] Testing nginx configuration..." -if ! nginx -t; then - echo "[cloudrun-wrapper] ERROR: nginx configuration test failed" - exit 1 -fi +# Skip nginx test - supervisor will handle nginx startup +echo "[cloudrun-wrapper] Skipping nginx test - supervisor manages nginx" # Start supervisor for kernel-images services in background echo "[cloudrun-wrapper] Starting kernel-images services..." @@ -45,6 +45,134 @@ cleanup() { } trap cleanup TERM INT -# Start nginx in foreground (main process for Cloud Run) +# Start nginx proxy on Cloud Run port (proxies directly to services) echo "[cloudrun-wrapper] Starting nginx proxy on port $PORT" -nginx -g "daemon off;" \ No newline at end of file + +# Create nginx config file +cat > /tmp/nginx.conf < Date: Thu, 4 Sep 2025 09:47:52 -0500 Subject: [PATCH 05/19] Local setup --- Dockerfile.local-extended => Dockerfile.local | 0 Makefile | 97 +++++-------------- build-local.sh | 34 ++----- run-local-extended.sh | 71 -------------- run-local.sh | 41 +++++--- 5 files changed, 61 insertions(+), 182 deletions(-) rename Dockerfile.local-extended => Dockerfile.local (100%) delete mode 100755 run-local-extended.sh diff --git a/Dockerfile.local-extended b/Dockerfile.local similarity index 100% rename from Dockerfile.local-extended rename to Dockerfile.local diff --git a/Makefile b/Makefile index e0415bc..24c7cdc 100644 --- a/Makefile +++ b/Makefile @@ -15,13 +15,13 @@ init: ## Initialize submodules (run this first) git submodule update --init --recursive @echo "โœ… Submodules initialized" -build: init ## Build using kernel-images build system - @echo "๐Ÿ”จ Building with kernel-images native build system..." - ./build-local.sh - @echo "โœ… Build complete" +build: init ## Build extended image with DevTools frontend + @echo "๐Ÿ”จ Building extended kernel-browser with DevTools frontend..." + docker build -f Dockerfile.local -t kernel-browser:extended . + @echo "โœ… Extended build complete" -run: ## Run using kernel-images run system (interactive) - @echo "๐Ÿš€ Starting kernel-browser using native run script..." +run: ## Run extended container with DevTools (interactive) + @echo "๐Ÿš€ Starting extended kernel-browser with DevTools..." ./run-local.sh compose-up: build ## Start with docker-compose (background) @@ -40,8 +40,8 @@ dev: compose-dev ## Alias for compose-dev stop: ## Stop all containers @echo "๐Ÿ›‘ Stopping containers..." docker-compose down - docker stop kernel-browser-local 2>/dev/null || true - docker rm kernel-browser-local 2>/dev/null || true + docker stop kernel-browser-extended 2>/dev/null || true + docker rm kernel-browser-extended 2>/dev/null || true @echo "โœ… Containers stopped" restart: ## Restart containers @@ -59,15 +59,16 @@ status: ## Show container status @docker ps --filter name=kernel-browser shell: ## Get shell access to running container - docker exec -it kernel-browser-local bash || docker-compose exec kernel-browser bash + docker exec -it kernel-browser-extended bash || docker-compose exec kernel-browser bash info: ## Show connection information @echo "" @echo "๐ŸŒ Service Access Points:" - @echo " WebRTC Client: http://localhost:8080" - @echo " Chrome DevTools: http://localhost:9222/json" - @echo " Recording API: http://localhost:444/api" - @echo " Health Check: http://localhost:8080/" + @echo " WebRTC Client: http://localhost:8080" + @echo " Chrome DevTools: http://localhost:9222/json" + @echo " Recording API: http://localhost:444/api" + @echo " Enhanced DevTools UI: http://localhost:8001" + @echo " DevTools Health: http://localhost:8001/health" test: ## Test service endpoints @echo "๐Ÿงช Testing service endpoints..." @@ -80,14 +81,21 @@ test: ## Test service endpoints @echo -n "Recording API (444): " @curl -s -o /dev/null -w "%{http_code}" http://localhost:444/ && echo " (404 is normal - API is running)" || echo "Failed to connect" @echo "" + @echo -n "DevTools UI (8001): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:8001/ || echo "Failed to connect" + @echo "" + @echo -n "DevTools Health (8001): " + @curl -s -o /dev/null -w "%{http_code}" http://localhost:8001/health || echo "Failed to connect" + @echo "" @echo "๐ŸŽฏ All services are ready! Access points:" - @echo " WebRTC Client: http://localhost:8080" - @echo " Chrome DevTools: http://localhost:9222/json" + @echo " WebRTC Client: http://localhost:8080" + @echo " Chrome DevTools: http://localhost:9222/json" + @echo " Enhanced DevTools UI: http://localhost:8001" clean: stop ## Clean up everything @echo "๐Ÿงน Cleaning up..." docker-compose down -v 2>/dev/null || true - docker rmi kernel-browser:local 2>/dev/null || true + docker rmi kernel-browser:extended 2>/dev/null || true docker system prune -f rm -rf recordings/* 2>/dev/null || true rm -rf kernel-images/images/chromium-headful/.tmp 2>/dev/null || true @@ -105,60 +113,3 @@ native-run: ## Run using kernel-images native script directly # Quick development workflow quick: init build compose-up test ## Quick setup: init + build + run + test -# ============================================================================ -# Extended targets with DevTools frontend -# ============================================================================ - -build-extended: init ## Build extended image with DevTools frontend - @echo "๐Ÿ”จ Building extended kernel-browser with DevTools frontend..." - docker build -f Dockerfile.local-extended -t kernel-browser:extended . - @echo "โœ… Extended build complete" - -run-extended: ## Run extended container with DevTools (interactive) - @echo "๐Ÿš€ Starting extended kernel-browser with DevTools..." - ./run-local-extended.sh - -info-extended: ## Show extended connection information - @echo "" - @echo "๐ŸŒ Extended Service Access Points:" - @echo " WebRTC Client: http://localhost:8080" - @echo " Chrome DevTools: http://localhost:9222/json" - @echo " Recording API: http://localhost:444/api" - @echo " Enhanced DevTools UI: http://localhost:8001" - @echo " DevTools Health: http://localhost:8001/health" - -test-extended: ## Test extended service endpoints including DevTools - @echo "๐Ÿงช Testing extended service endpoints..." - @echo -n "WebRTC Client (8080): " - @curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/ || echo "Failed to connect" - @echo "" - @echo -n "Chrome DevTools (9222): " - @curl -s -o /dev/null -w "%{http_code}" http://localhost:9222/json/version || echo "Failed to connect" - @echo "" - @echo -n "Recording API (444): " - @curl -s -o /dev/null -w "%{http_code}" http://localhost:444/ && echo " (404 is normal - API is running)" || echo "Failed to connect" - @echo "" - @echo -n "DevTools UI (8001): " - @curl -s -o /dev/null -w "%{http_code}" http://localhost:8001/ || echo "Failed to connect" - @echo "" - @echo -n "DevTools Health (8001): " - @curl -s -o /dev/null -w "%{http_code}" http://localhost:8001/health || echo "Failed to connect" - @echo "" - @echo "๐ŸŽฏ All extended services are ready! Access points:" - @echo " WebRTC Client: http://localhost:8080" - @echo " Chrome DevTools: http://localhost:9222/json" - @echo " Enhanced DevTools UI: http://localhost:8001" - -stop-extended: ## Stop extended container - @echo "๐Ÿ›‘ Stopping extended containers..." - docker stop kernel-browser-extended 2>/dev/null || true - docker rm kernel-browser-extended 2>/dev/null || true - @echo "โœ… Extended containers stopped" - -clean-extended: stop-extended ## Clean up extended containers and images - @echo "๐Ÿงน Cleaning up extended resources..." - docker rmi kernel-browser:extended 2>/dev/null || true - @echo "โœ… Extended cleanup complete" - -# Extended workflow -quick-extended: init build-extended run-extended ## Quick extended setup: init + build + run with DevTools \ No newline at end of file diff --git a/build-local.sh b/build-local.sh index 975196e..b388617 100755 --- a/build-local.sh +++ b/build-local.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Local build wrapper for kernel-images chromium-headful +# Extended local build wrapper for kernel-browser with DevTools set -e -o pipefail -echo "๐Ÿ”จ Building kernel-browser using kernel-images build system..." +echo "๐Ÿ”จ Building extended kernel-browser with DevTools frontend..." # Ensure we're in the right directory SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) @@ -22,29 +22,15 @@ if [ ! -f "kernel-images/images/chromium-headful/build-docker.sh" ]; then exit 1 fi -# Change to kernel-images directory and build using their system -echo "๐Ÿ“ Changing to kernel-images directory..." -cd kernel-images/images/chromium-headful +echo "๐Ÿš€ Starting extended build with Docker..." +echo " Using: Dockerfile.local" +echo " Target image: kernel-browser:extended" -# Make build script executable -chmod +x build-docker.sh +# Build using Docker with extended Dockerfile +docker build -f Dockerfile.local -t kernel-browser:extended . -# Set image name for local use -export IMAGE="kernel-browser:local" -export NAME="kernel-browser-local" - -# Set dummy UKC variables to bypass cloud requirements (we only need Docker) -export UKC_TOKEN="dummy-token-for-local-build" -export UKC_METRO="dummy-metro-for-local-build" - -echo "๐Ÿš€ Starting build with kernel-images build system..." -echo " Image: $IMAGE" -echo " Bypassing UKC requirements for local Docker build..." - -# Run the official build script -./build-docker.sh - -echo "โœ… Build completed successfully!" -echo " Image built: $IMAGE" +echo "โœ… Extended build completed successfully!" +echo " Image built: kernel-browser:extended" +echo " Includes: Chromium + DevTools frontend + WebRTC" echo "" echo "๐Ÿƒ To run locally, use: ./run-local.sh" \ No newline at end of file diff --git a/run-local-extended.sh b/run-local-extended.sh deleted file mode 100755 index dc4cbe0..0000000 --- a/run-local-extended.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -# Extended local run wrapper for kernel-images chromium-headful + DevTools -set -e -o pipefail - -echo "๐Ÿš€ Starting kernel-browser (EXTENDED) locally using kernel-images run system..." - -# Ensure we're in the right directory -SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) -cd "$SCRIPT_DIR" - -# Check if kernel-images submodule exists -if [ ! -d "kernel-images" ] || [ ! -f "kernel-images/images/chromium-headful/run-docker.sh" ]; then - echo "โŒ Error: kernel-images submodule not found or incomplete" - echo " Run: git submodule update --init --recursive" - exit 1 -fi - -# Create local recordings directory -mkdir -p "$SCRIPT_DIR/recordings" - -# Change to kernel-images directory -cd kernel-images/images/chromium-headful - -# Make run script executable -chmod +x run-docker.sh - -# Set environment variables for extended local development -export IMAGE="kernel-browser:extended" -export NAME="kernel-browser-extended" -export ENABLE_WEBRTC="true" -export RUN_AS_ROOT="false" - -# Set dummy UKC variables to bypass cloud requirements (we only need Docker) -export UKC_TOKEN="dummy-token-for-local-run" -export UKC_METRO="dummy-metro-for-local-run" - -# Local-friendly Chrome flags (less restrictive than cloud) + custom DevTools frontend -export CHROMIUM_FLAGS="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --start-maximized --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --custom-devtools-frontend=http://localhost:8001/" - -echo "๐Ÿ”ง Configuration:" -echo " Image: $IMAGE" -echo " Container: $NAME" -echo " WebRTC: $ENABLE_WEBRTC" -echo " DevTools UI: enabled" -echo " Run as root: $RUN_AS_ROOT" -echo " Recordings: $SCRIPT_DIR/recordings" -echo "" - -echo "๐Ÿƒ Starting extended container with kernel-images run system..." - -# Backup original run-docker.sh to modify port mappings -if [ ! -f run-docker.sh.original ]; then - cp run-docker.sh run-docker.sh.original -fi - -# Create modified run script that adds DevTools port mapping -cat run-docker.sh.original | \ -sed 's/docker run -it/docker run -it -p 8001:8001/' > run-docker.sh.extended - -chmod +x run-docker.sh.extended - -# Run using the modified run script with DevTools port -./run-docker.sh.extended - -echo "" -echo "๐ŸŒ Extended service should be accessible at:" -echo " WebRTC Client: http://localhost:8080" -echo " Chrome DevTools: http://localhost:9222" -echo " Recording API: http://localhost:444" -echo " Enhanced DevTools UI: http://localhost:8001" \ No newline at end of file diff --git a/run-local.sh b/run-local.sh index cfab5c4..dc4cbe0 100755 --- a/run-local.sh +++ b/run-local.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Local run wrapper for kernel-images chromium-headful +# Extended local run wrapper for kernel-images chromium-headful + DevTools set -e -o pipefail -echo "๐Ÿš€ Starting kernel-browser locally using kernel-images run system..." +echo "๐Ÿš€ Starting kernel-browser (EXTENDED) locally using kernel-images run system..." # Ensure we're in the right directory SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) @@ -25,9 +25,9 @@ cd kernel-images/images/chromium-headful # Make run script executable chmod +x run-docker.sh -# Set environment variables for local development -export IMAGE="kernel-browser:local" -export NAME="kernel-browser-local" +# Set environment variables for extended local development +export IMAGE="kernel-browser:extended" +export NAME="kernel-browser-extended" export ENABLE_WEBRTC="true" export RUN_AS_ROOT="false" @@ -35,24 +35,37 @@ export RUN_AS_ROOT="false" export UKC_TOKEN="dummy-token-for-local-run" export UKC_METRO="dummy-metro-for-local-run" -# Local-friendly Chrome flags (less restrictive than cloud) -export CHROMIUM_FLAGS="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --start-maximized --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox" +# Local-friendly Chrome flags (less restrictive than cloud) + custom DevTools frontend +export CHROMIUM_FLAGS="--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --start-maximized --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --custom-devtools-frontend=http://localhost:8001/" echo "๐Ÿ”ง Configuration:" echo " Image: $IMAGE" echo " Container: $NAME" echo " WebRTC: $ENABLE_WEBRTC" +echo " DevTools UI: enabled" echo " Run as root: $RUN_AS_ROOT" echo " Recordings: $SCRIPT_DIR/recordings" echo "" -echo "๐Ÿƒ Starting container with kernel-images run system..." +echo "๐Ÿƒ Starting extended container with kernel-images run system..." -# Run using the official run script -./run-docker.sh +# Backup original run-docker.sh to modify port mappings +if [ ! -f run-docker.sh.original ]; then + cp run-docker.sh run-docker.sh.original +fi + +# Create modified run script that adds DevTools port mapping +cat run-docker.sh.original | \ +sed 's/docker run -it/docker run -it -p 8001:8001/' > run-docker.sh.extended + +chmod +x run-docker.sh.extended + +# Run using the modified run script with DevTools port +./run-docker.sh.extended echo "" -echo "๐ŸŒ Service should be accessible at:" -echo " WebRTC Client: http://localhost:8080" -echo " Chrome DevTools: http://localhost:9222" -echo " Recording API: http://localhost:444" \ No newline at end of file +echo "๐ŸŒ Extended service should be accessible at:" +echo " WebRTC Client: http://localhost:8080" +echo " Chrome DevTools: http://localhost:9222" +echo " Recording API: http://localhost:444" +echo " Enhanced DevTools UI: http://localhost:8001" \ No newline at end of file From 25522854f8a828a35bae1e2c9e22368faea36472 Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 17:20:39 -0500 Subject: [PATCH 06/19] Using Twilio + CloudRun to deploy devtools frontend with Chromium --- .env.example | 17 ++ .gitignore | 54 ++++++ DEPLOYMENT.md | 166 +++++++++++++++++ Dockerfile.cloudrun | 10 +- cloudbuild.yaml | 37 +++- cloudrun-wrapper.sh | 69 ++++--- deploy.sh | 149 +++++++++++++++- service-secrets.yaml | 96 ++++++++++ service.yaml | 24 ++- supervisor/services-cloudrun/chromium.conf | 10 ++ supervisor/services-cloudrun/dbus.conf | 10 ++ .../services-cloudrun/devtools-frontend.conf | 11 ++ supervisor/services-cloudrun/neko.conf | 10 ++ supervisor/services-cloudrun/xorg.conf | 10 ++ twilio/README.md | 79 ++++++++ twilio/generate-twilio-credential.js | 42 +++++ twilio/test-twilio-api.sh | 54 ++++++ twilio/test-twilio-node.js | 62 +++++++ twilio/test-twilio-turn.js | 45 +++++ twilio/twilio-credential-updater.sh | 72 ++++++++ twilio/twilio-token-service.js | 168 ++++++++++++++++++ twilio/update-twilio-credentials.sh | 96 ++++++++++ twilio/verify-twilio.js | 84 +++++++++ 23 files changed, 1323 insertions(+), 52 deletions(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 DEPLOYMENT.md create mode 100644 service-secrets.yaml create mode 100644 supervisor/services-cloudrun/chromium.conf create mode 100644 supervisor/services-cloudrun/dbus.conf create mode 100644 supervisor/services-cloudrun/devtools-frontend.conf create mode 100644 supervisor/services-cloudrun/neko.conf create mode 100644 supervisor/services-cloudrun/xorg.conf create mode 100644 twilio/README.md create mode 100644 twilio/generate-twilio-credential.js create mode 100755 twilio/test-twilio-api.sh create mode 100644 twilio/test-twilio-node.js create mode 100644 twilio/test-twilio-turn.js create mode 100644 twilio/twilio-credential-updater.sh create mode 100644 twilio/twilio-token-service.js create mode 100755 twilio/update-twilio-credentials.sh create mode 100644 twilio/verify-twilio.js diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..268baa2 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# Twilio Network Traversal Service Credentials +# Get these from your Twilio Console: +# 1. Go to https://console.twilio.com/ +# 2. Navigate to Account > API Keys & Tokens +# 3. Create a new API Key +# 4. Use the SID as TWILIO_ACCOUNT_SID +# 5. Use the Secret as TWILIO_AUTH_TOKEN +TWILIO_ACCOUNT_SID=SK...your_api_key_sid_here +TWILIO_AUTH_TOKEN=your_api_key_secret_here + +# Optional: Google Cloud Configuration +# If not provided, will use current gcloud config +# PROJECT_ID=your-project-id +# REGION=us-central1 + +# Optional: Service Configuration +# SERVICE_NAME=kernel-browser \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d571dba --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +# Environment variables +.env +.env.local +*.env +!.env.example + +# Node modules +node_modules/ + +# Build outputs +dist/ +build/ +out/ + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# OS files +.DS_Store +Thumbs.db + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo + +# Temporary files +tmp/ +temp/ +*.tmp + +# Python +__pycache__/ +*.py[cod] +*$py.class +.Python +venv/ +env/ + +# Google Cloud +.gcloudignore +gcs-key.json +service-account-key.json + +# Docker +.dockerignore + +# Backup files +*.bak +*.backup \ No newline at end of file diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..b029426 --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,166 @@ +# Kernel Browser - Cloud Run Deployment Guide + +This guide explains how to deploy the Kernel Browser to Google Cloud Run with secure Twilio credential management. + +## Prerequisites + +- Google Cloud SDK (`gcloud`) installed +- Docker installed +- Git installed +- A Google Cloud Project with billing enabled +- Twilio account with API credentials (for WebRTC TURN servers) + +## Quick Start + +### 1. Clone the repository +```bash +git clone +cd browser-web-agent +git submodule update --init --recursive +``` + +### 2. Set up Twilio credentials +```bash +# Copy the example environment file +cp .env.example .env + +# Edit .env and add your Twilio credentials +# Get these from https://console.twilio.com/ > Account > API Keys & Tokens +``` + +Your `.env` file should contain: +``` +TWILIO_ACCOUNT_SID=SK...your_api_key_sid_here +TWILIO_AUTH_TOKEN=your_api_key_secret_here +``` + +### 3. Deploy to Cloud Run +```bash +./deploy.sh +``` + +The script will: +- Load credentials from `.env` +- Create/update secrets in Google Secret Manager +- Build and deploy the container to Cloud Run +- Configure all necessary permissions + +## Deployment Options + +### Using Cloud Build (recommended) +```bash +./deploy.sh +``` + +### Using local Docker build +```bash +./deploy.sh --local +``` + +### Specify project and region +```bash +./deploy.sh --project YOUR_PROJECT_ID --region us-central1 +``` + +## How It Works + +### Credential Management + +1. **Local Development**: Credentials are stored in `.env` file (gitignored) +2. **Secret Manager**: Deploy script automatically creates/updates secrets in Google Secret Manager +3. **Cloud Run**: Service uses `secretKeyRef` to securely access credentials at runtime +4. **Dynamic TURN**: Container fetches fresh TURN credentials from Twilio on startup + +### Security Features + +- Credentials never appear in code or logs +- Secrets are encrypted at rest and in transit +- Service account has minimal required permissions +- Automatic credential rotation support + +### Files Overview + +- `.env.example` - Template for environment variables +- `.env` - Your local credentials (gitignored) +- `deploy.sh` - Main deployment script with Secret Manager integration +- `service-secrets.yaml` - Cloud Run config with secret references +- `service.yaml` - Fallback config (for deployments without secrets) +- `cloudbuild.yaml` - Cloud Build configuration +- `twilio/` - Twilio credential management scripts + +## Updating Credentials + +To update Twilio credentials: + +1. Update `.env` with new credentials +2. Run `./deploy.sh` again +3. Script will update secrets and redeploy + +## Manual Secret Management + +If you need to manage secrets manually: + +```bash +# Create secrets +echo -n "YOUR_SID" | gcloud secrets create twilio-account-sid --data-file=- +echo -n "YOUR_TOKEN" | gcloud secrets create twilio-auth-token --data-file=- + +# Update secrets +echo -n "NEW_SID" | gcloud secrets versions add twilio-account-sid --data-file=- +echo -n "NEW_TOKEN" | gcloud secrets versions add twilio-auth-token --data-file=- + +# Grant access to service account +gcloud secrets add-iam-policy-binding twilio-account-sid \ + --member="serviceAccount:kernel-browser-sa@PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/secretmanager.secretAccessor" +``` + +## Service Endpoints + +After deployment, you'll have access to: + +- **Main Interface**: `https://SERVICE_URL/` +- **WebRTC Client**: `https://SERVICE_URL/` +- **Chrome DevTools**: `https://SERVICE_URL/devtools/` +- **DevTools WebSocket**: `wss://SERVICE_URL/cdp/ws` +- **Recording API**: `https://SERVICE_URL/api` +- **Health Check**: `https://SERVICE_URL/health` + +## Troubleshooting + +### Deployment fails +- Check that all prerequisites are installed +- Ensure billing is enabled on your GCP project +- Verify you have sufficient quota in your region + +### WebRTC not working +- Ensure Twilio credentials are correct +- Check Cloud Run logs: `gcloud run services logs read kernel-browser --region=us-central1` +- Verify TURN servers are accessible from your network + +### Secrets not found +- Run `gcloud secrets list` to verify secrets exist +- Check service account permissions +- Ensure Secret Manager API is enabled + +## Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Client โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ Cloud Run โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ Secret Manager โ”‚ +โ”‚ (Browser) โ”‚ โ”‚ (Container) โ”‚ โ”‚ (Credentials) โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ Twilio API โ”‚ + โ”‚ (TURN Servers) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Support + +For issues or questions: +- Check logs: `gcloud run services logs read kernel-browser --region=us-central1` +- Review service status: `gcloud run services describe kernel-browser --region=us-central1` +- File an issue on GitHub \ No newline at end of file diff --git a/Dockerfile.cloudrun b/Dockerfile.cloudrun index 78776be..91ca6c7 100644 --- a/Dockerfile.cloudrun +++ b/Dockerfile.cloudrun @@ -1,6 +1,9 @@ # DevTools Frontend build stage using browser-operator-core FROM --platform=linux/amd64 ubuntu:22.04 AS devtools-builder +# Cache bust argument to force rebuilds +ARG CACHE_BUST + # Install required packages for DevTools frontend build RUN apt-get update && apt-get install -y \ curl \ @@ -50,7 +53,7 @@ RUN git checkout upstream/main RUN npm run build # Multi-stage build using kernel-images as base -FROM docker.io/golang:1.23.0 AS server-builder +FROM docker.io/golang:1.25.0 AS server-builder WORKDIR /workspace/server ARG TARGETOS @@ -253,10 +256,11 @@ COPY --from=devtools-builder /workspace/devtools/devtools-frontend/out/Default/g # Set permissions for DevTools files RUN chown -R kernel:kernel /usr/share/nginx/devtools -# Cloud Run specific: wrapper script only (nginx config is inline) +# Cloud Run specific: wrapper scripts (nginx config is inline) # DO NOT copy nginx.conf to avoid auto-start conflicts COPY cloudrun-wrapper.sh /cloudrun-wrapper.sh -RUN chmod +x /cloudrun-wrapper.sh +COPY twilio/twilio-credential-updater.sh /twilio-credential-updater.sh +RUN chmod +x /cloudrun-wrapper.sh /twilio-credential-updater.sh # Add essential services for neko WebRTC and Chromium COPY supervisor/services-cloudrun/dbus.conf /etc/supervisor/conf.d/services-cloudrun/dbus.conf diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 3470f89..d178da5 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -28,6 +28,8 @@ steps: - 'Dockerfile.cloudrun' - '--cache-from' - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' + - '--build-arg' + - 'CACHE_BUST=$BUILD_ID' - '--tag' - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' - '.' @@ -39,24 +41,41 @@ steps: - 'push' - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' - # Step 5: Update the service.yaml with the correct project ID + # Step 5: Choose appropriate service.yaml based on secrets availability - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' args: - '-c' - | - sed -i "s/PROJECT_ID/$PROJECT_ID/g" service.yaml - cat service.yaml + # Check if Twilio secrets exist + if gcloud secrets describe twilio-account-sid --project=$PROJECT_ID >/dev/null 2>&1 && \ + gcloud secrets describe twilio-auth-token --project=$PROJECT_ID >/dev/null 2>&1; then + echo "Using service-secrets.yaml with Secret Manager references" + SERVICE_YAML="service-secrets.yaml" + else + echo "Using standard service.yaml (secrets not configured)" + SERVICE_YAML="service.yaml" + fi + + # Update project ID in the chosen service file + sed -i "s/PROJECT_ID/$PROJECT_ID/g" $SERVICE_YAML + + echo "Deploying with: $SERVICE_YAML" + cat $SERVICE_YAML + + # Save the choice for next step + echo $SERVICE_YAML > /workspace/service_choice.txt # Step 6: Deploy to Cloud Run - name: 'gcr.io/cloud-builders/gcloud' + entrypoint: 'bash' args: - - 'run' - - 'services' - - 'replace' - - 'service.yaml' - - '--region=us-central1' - - '--quiet' + - '-c' + - | + SERVICE_YAML=$(cat /workspace/service_choice.txt) + gcloud run services replace $SERVICE_YAML \ + --region=us-central1 \ + --quiet # Step 7: Update traffic to latest revision - name: 'gcr.io/cloud-builders/gcloud' diff --git a/cloudrun-wrapper.sh b/cloudrun-wrapper.sh index 8e91d01..c49c995 100644 --- a/cloudrun-wrapper.sh +++ b/cloudrun-wrapper.sh @@ -13,6 +13,14 @@ export HEIGHT=768 export WIDTH=1024 export NEKO_BIND=:8081 +# Get fresh Twilio TURN credentials if available +if [ -f /twilio-credential-updater.sh ]; then + echo "[cloudrun-wrapper] Getting fresh Twilio TURN credentials..." + source /twilio-credential-updater.sh +else + echo "[cloudrun-wrapper] Twilio updater not found, using credentials from environment" +fi + # Port configuration for Cloud Run export PORT=${PORT:-8080} export CHROMIUM_FLAGS="${CHROMIUM_FLAGS:---user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor --custom-devtools-frontend=http://localhost:8001/ https://www.google.com}" @@ -26,27 +34,8 @@ mkdir -p /tmp/nginx_client_temp /tmp/nginx_proxy_temp /tmp/nginx_fastcgi_temp \ /tmp/runtime-kernel /var/log/neko /tmp/recordings \ /tmp/supervisord /tmp/dbus -# Skip nginx test - supervisor will handle nginx startup -echo "[cloudrun-wrapper] Skipping nginx test - supervisor manages nginx" - -# Start supervisor for kernel-images services in background -echo "[cloudrun-wrapper] Starting kernel-images services..." -supervisord -c /etc/supervisor/supervisord-cloudrun.conf -n & -SUPERVISOR_PID=$! - -# Wait a moment for services to start -sleep 5 - -# Cleanup function -cleanup() { - echo "[cloudrun-wrapper] Cleaning up..." - kill $SUPERVISOR_PID 2>/dev/null || true - supervisorctl -c /etc/supervisor/supervisord-cloudrun.conf stop all 2>/dev/null || true -} -trap cleanup TERM INT - -# Start nginx proxy on Cloud Run port (proxies directly to services) -echo "[cloudrun-wrapper] Starting nginx proxy on port $PORT" +# Start nginx immediately in background to respond to CloudRun health checks +echo "[cloudrun-wrapper] Starting nginx proxy on port $PORT (background)" # Create nginx config file cat > /tmp/nginx.conf < /dev/null 2>&1; then + echo "[cloudrun-wrapper] Neko service is ready" + break + fi + if [ $i -eq 60 ]; then + echo "[cloudrun-wrapper] Warning: Neko service not ready after 60 seconds, starting nginx anyway" + fi + sleep 1 +done + +# Start nginx in foreground (required for Cloud Run) +echo "[cloudrun-wrapper] Starting nginx proxy on port $PORT" exec nginx -g "daemon off;" -c /tmp/nginx.conf \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index 3d7f4b1..50527e1 100755 --- a/deploy.sh +++ b/deploy.sh @@ -34,6 +34,127 @@ info() { echo -e "โ„น๏ธ $1" } +# Load environment variables from .env file +load_env_file() { + if [ -f .env ]; then + info "Loading configuration from .env file..." + # Export variables from .env, ignoring comments and empty lines + set -a + source <(grep -v '^#' .env | grep -v '^$') + set +a + success "Configuration loaded from .env" + elif [ -f .env.example ]; then + warning "No .env file found. Copy .env.example to .env and add your credentials" + info "Run: cp .env.example .env" + fi +} + +# Validate Twilio credentials +validate_twilio_credentials() { + if [ -z "${TWILIO_ACCOUNT_SID:-}" ] || [ -z "${TWILIO_AUTH_TOKEN:-}" ]; then + warning "Twilio credentials not found in environment" + echo "You can either:" + echo " 1. Add them to .env file (recommended)" + echo " 2. Enter them now (temporary)" + echo " 3. Skip (WebRTC may not work properly)" + echo + read -p "Enter choice [1/2/3]: " choice + + case $choice in + 1) + info "Please add credentials to .env file and re-run the script" + exit 0 + ;; + 2) + read -p "Enter Twilio Account SID: " TWILIO_ACCOUNT_SID + read -s -p "Enter Twilio Auth Token: " TWILIO_AUTH_TOKEN + echo + ;; + 3) + warning "Skipping Twilio configuration" + return 1 + ;; + *) + error "Invalid choice" + ;; + esac + fi + + if [ -n "${TWILIO_ACCOUNT_SID:-}" ] && [ -n "${TWILIO_AUTH_TOKEN:-}" ]; then + # Basic validation of credential format + if [[ ! "$TWILIO_ACCOUNT_SID" =~ ^SK[a-f0-9]{32}$ ]]; then + warning "Twilio Account SID format looks incorrect (should start with SK and be 34 chars)" + fi + success "Twilio credentials configured" + return 0 + else + return 1 + fi +} + +# Setup Google Secret Manager secrets +setup_secrets() { + if ! validate_twilio_credentials; then + warning "Skipping Secret Manager setup - no Twilio credentials provided" + return 0 + fi + + info "Setting up Google Secret Manager secrets..." + + # Enable Secret Manager API if not already enabled + info "Enabling Secret Manager API..." + gcloud services enable secretmanager.googleapis.com --project="$PROJECT_ID" --quiet + + # Create or update twilio-account-sid secret + if gcloud secrets describe twilio-account-sid --project="$PROJECT_ID" &>/dev/null; then + echo -n "$TWILIO_ACCOUNT_SID" | gcloud secrets versions add twilio-account-sid \ + --data-file=- \ + --project="$PROJECT_ID" + info "Updated twilio-account-sid secret" + else + echo -n "$TWILIO_ACCOUNT_SID" | gcloud secrets create twilio-account-sid \ + --data-file=- \ + --project="$PROJECT_ID" \ + --replication-policy="automatic" + success "Created twilio-account-sid secret" + fi + + # Create or update twilio-auth-token secret + if gcloud secrets describe twilio-auth-token --project="$PROJECT_ID" &>/dev/null; then + echo -n "$TWILIO_AUTH_TOKEN" | gcloud secrets versions add twilio-auth-token \ + --data-file=- \ + --project="$PROJECT_ID" + info "Updated twilio-auth-token secret" + else + echo -n "$TWILIO_AUTH_TOKEN" | gcloud secrets create twilio-auth-token \ + --data-file=- \ + --project="$PROJECT_ID" \ + --replication-policy="automatic" + success "Created twilio-auth-token secret" + fi + + # Grant access to service account + local sa_email="kernel-browser-sa@${PROJECT_ID}.iam.gserviceaccount.com" + + info "Granting Secret Manager access to service account..." + gcloud secrets add-iam-policy-binding twilio-account-sid \ + --member="serviceAccount:$sa_email" \ + --role="roles/secretmanager.secretAccessor" \ + --project="$PROJECT_ID" \ + --quiet + + gcloud secrets add-iam-policy-binding twilio-auth-token \ + --member="serviceAccount:$sa_email" \ + --role="roles/secretmanager.secretAccessor" \ + --project="$PROJECT_ID" \ + --quiet + + # Set flag to use secrets-enabled service.yaml + export USE_SECRETS=true + + success "Secret Manager configured with Twilio credentials" +} + # Check prerequisites check_prerequisites() { info "Checking prerequisites..." @@ -85,6 +206,7 @@ enable_apis() { "containerregistry.googleapis.com" "compute.googleapis.com" "storage.googleapis.com" + "secretmanager.googleapis.com" ) for api in "${apis[@]}"; do @@ -175,16 +297,29 @@ deploy_local() { info "Deploying to Cloud Run..." - # Update service.yaml with project ID and image - sed -i.bak "s/PROJECT_ID/$PROJECT_ID/g" service.yaml - sed -i.bak "s|gcr.io/PROJECT_ID/kernel-browser:latest|$image_name|g" service.yaml + # Choose appropriate service.yaml based on secrets availability + local service_file="service.yaml" + if [ "${USE_SECRETS:-false}" = "true" ]; then + if gcloud secrets describe twilio-account-sid --project="$PROJECT_ID" &>/dev/null && \ + gcloud secrets describe twilio-auth-token --project="$PROJECT_ID" &>/dev/null; then + service_file="service-secrets.yaml" + info "Using service-secrets.yaml with Secret Manager references" + else + warning "Secrets not found, falling back to standard service.yaml" + fi + fi + + # Update service file with project ID and image + cp "$service_file" "${service_file}.tmp" + sed -i.bak "s/PROJECT_ID/$PROJECT_ID/g" "${service_file}.tmp" + sed -i.bak "s|us-docker.pkg.dev/func-241017/gcr.io/kernel-browser:latest|$image_name|g" "${service_file}.tmp" - gcloud run services replace service.yaml \ + gcloud run services replace "${service_file}.tmp" \ --region="$REGION" \ --project="$PROJECT_ID" - # Restore original service.yaml - mv service.yaml.bak service.yaml + # Clean up temporary files + rm -f "${service_file}.tmp" "${service_file}.tmp.bak" success "Local build and deployment completed" } @@ -255,9 +390,11 @@ main() { done check_prerequisites + load_env_file setup_project enable_apis create_service_account + setup_secrets update_submodules if [ "${LOCAL_BUILD:-false}" = "true" ]; then diff --git a/service-secrets.yaml b/service-secrets.yaml new file mode 100644 index 0000000..34b4b91 --- /dev/null +++ b/service-secrets.yaml @@ -0,0 +1,96 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: kernel-browser + annotations: + run.googleapis.com/ingress: all + run.googleapis.com/ingress-status: all +spec: + template: + metadata: + annotations: + # Use second generation execution environment + run.googleapis.com/execution-environment: gen2 + # Disable CPU throttling for consistent performance + run.googleapis.com/cpu-throttling: "false" + # Increase startup timeout to 10 minutes for complex service startup + run.googleapis.com/timeout: "600" + # Auto-scaling settings + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "2" + spec: + # Allow multiple concurrent requests (browser can handle multiple tabs/requests) + containerConcurrency: 10 + # 1 hour timeout for long browser sessions + timeoutSeconds: 3600 + # Service account for GCP access + serviceAccountName: kernel-browser-sa + containers: + - name: kernel-browser + # This will be set during deployment + image: us-docker.pkg.dev/func-241017/gcr.io/kernel-browser:latest + ports: + - name: http1 + containerPort: 8080 + resources: + limits: + # 2 CPU cores (within quota limits) + cpu: "2" + # 4GiB memory (within quota limits) + memory: "4Gi" + requests: + cpu: "1" + memory: "2Gi" + env: + # Enable WebRTC for live viewing + - name: ENABLE_WEBRTC + value: "true" + # Run as non-root user (Cloud Run requirement) + - name: RUN_AS_ROOT + value: "false" + # Chrome optimizations for Cloud Run + - name: CHROMIUM_FLAGS + value: "--user-data-dir=/home/kernel/user-data --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor" + # Display configuration + - name: DISPLAY_NUM + value: "1" + - name: HEIGHT + value: "768" + - name: WIDTH + value: "1024" + # Twilio API Key credentials from Secret Manager + - name: TWILIO_ACCOUNT_SID + valueFrom: + secretKeyRef: + name: twilio-account-sid + key: latest + - name: TWILIO_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: twilio-auth-token + key: latest + # Dynamic TURN credentials will be generated using above secrets + # The twilio-credential-updater.sh script will use these at startup + - name: NEKO_ICESERVERS + value: 'DYNAMIC' # Placeholder - will be replaced by twilio-credential-updater.sh + # Disable TCP multiplexing (nginx handles port 8080) + - name: NEKO_WEBRTC_TCPMUX + value: "0" + # Optional: Google Cloud Storage bucket for recordings + - name: GCS_BUCKET + value: "kernel-browser-recordings" + # API configuration + - name: KERNEL_IMAGES_API_PORT + value: "10001" + - name: KERNEL_IMAGES_API_FRAME_RATE + value: "10" + - name: KERNEL_IMAGES_API_MAX_SIZE_MB + value: "500" + - name: KERNEL_IMAGES_API_OUTPUT_DIR + value: "/tmp/recordings" + # Force new revision + - name: DEPLOYMENT_VERSION + value: "v12-secret-manager" + traffic: + - percent: 100 + latestRevision: true \ No newline at end of file diff --git a/service.yaml b/service.yaml index 8f59516..f39c167 100644 --- a/service.yaml +++ b/service.yaml @@ -13,8 +13,8 @@ spec: run.googleapis.com/execution-environment: gen2 # Disable CPU throttling for consistent performance run.googleapis.com/cpu-throttling: "false" - # Increase startup timeout - run.googleapis.com/timeout: "3600" + # Increase startup timeout to 10 minutes for complex service startup + run.googleapis.com/timeout: "600" # Auto-scaling settings autoscaling.knative.dev/minScale: "1" autoscaling.knative.dev/maxScale: "2" @@ -58,14 +58,17 @@ spec: value: "768" - name: WIDTH value: "1024" - # ICE servers configuration for WebRTC (includes both STUN and TURN) + # Twilio API Key credentials for dynamic TURN generation + - name: TWILIO_ACCOUNT_SID + value: "SK5346918f48275d6571be927e84cfd6f8" + - name: TWILIO_AUTH_TOKEN + value: "OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo" + # Fresh Twilio TURN credentials (manually generated for now) - name: NEKO_ICESERVERS - value: '[{"urls":["stun:global.stun.twilio.com:3478"]},{"urls":["turn:global.turn.twilio.com:3478?transport=udp"],"username":"464cefa09d5a8b4030b34b3faf15871b5efe0eef8331e9324f3f4f9144158ada","credential":"1Fm/UdpnNFbvfDPBtETUSZ4BhQsi0cubgLBdbScluPs="}]' - # WebRTC configuration - - name: NEKO_WEBRTC_TCPPORT - value: "8081" - - name: NEKO_WEBRTC_UDPPORT - value: "8082" + value: '[{"urls":["turn:global.turn.twilio.com:3478?transport=tcp"],"username":"b88cfa1369190aa9cbc8bfaca683c457476b5d7062aa0a7b184c87db3ade0ff5","credential":"m3oPEt94gQQP+g2yd4R32MuZtCCdw6Rmmuvkp6/Dkd0="},{"urls":["turn:global.turn.twilio.com:443?transport=tcp"],"username":"b88cfa1369190aa9cbc8bfaca683c457476b5d7062aa0a7b184c87db3ade0ff5","credential":"m3oPEt94gQQP+g2yd4R32MuZtCCdw6Rmmuvkp6/Dkd0="}]' + # Disable TCP multiplexing (nginx handles port 8080) + - name: NEKO_WEBRTC_TCPMUX + value: "0" # Optional: Google Cloud Storage bucket for recordings - name: GCS_BUCKET value: "kernel-browser-recordings" @@ -78,6 +81,9 @@ spec: value: "500" - name: KERNEL_IMAGES_API_OUTPUT_DIR value: "/tmp/recordings" + # Force new revision + - name: DEPLOYMENT_VERSION + value: "v11-fresh-twilio-credentials" traffic: - percent: 100 latestRevision: true \ No newline at end of file diff --git a/supervisor/services-cloudrun/chromium.conf b/supervisor/services-cloudrun/chromium.conf new file mode 100644 index 0000000..6fdc548 --- /dev/null +++ b/supervisor/services-cloudrun/chromium.conf @@ -0,0 +1,10 @@ +[program:chromium] +command=/bin/bash -lc 'sleep 3 && DISPLAY=":1" DBUS_SESSION_BUS_ADDRESS="unix:path=/tmp/dbus/session_bus_socket" chromium --remote-debugging-port=9223 --remote-allow-origins=* --user-data-dir=/home/kernel/user-data --password-store=basic --no-first-run --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor --custom-devtools-frontend=http://localhost:8001/ https://www.google.com' +autostart=true +autorestart=true +startsecs=8 +priority=20 +stdout_logfile=/var/log/supervisord/chromium/chromium.log +stdout_logfile_maxbytes=50MB +redirect_stderr=true +environment=HOME="/home/kernel",USER="kernel",DISPLAY=":1",DBUS_SESSION_BUS_ADDRESS="unix:path=/tmp/dbus/session_bus_socket" \ No newline at end of file diff --git a/supervisor/services-cloudrun/dbus.conf b/supervisor/services-cloudrun/dbus.conf new file mode 100644 index 0000000..beb4cda --- /dev/null +++ b/supervisor/services-cloudrun/dbus.conf @@ -0,0 +1,10 @@ +[program:dbus] +command=/bin/bash -lc 'mkdir -p /tmp/dbus && dbus-uuidgen --ensure && dbus-daemon --session --address=unix:path=/tmp/dbus/session_bus_socket --nopidfile --nosyslog --nofork' +autostart=true +autorestart=true +startsecs=2 +priority=1 +stdout_logfile=/var/log/supervisord/dbus/dbus.log +stdout_logfile_maxbytes=50MB +redirect_stderr=true +environment=HOME="/home/kernel",USER="kernel",DBUS_SESSION_BUS_ADDRESS="unix:path=/tmp/dbus/session_bus_socket" \ No newline at end of file diff --git a/supervisor/services-cloudrun/devtools-frontend.conf b/supervisor/services-cloudrun/devtools-frontend.conf new file mode 100644 index 0000000..b0855ed --- /dev/null +++ b/supervisor/services-cloudrun/devtools-frontend.conf @@ -0,0 +1,11 @@ +[program:devtools-frontend] +command=/bin/bash -c 'cd /usr/share/nginx/devtools && python3 -m http.server 8001' +autostart=true +autorestart=true +startsecs=5 +priority=20 +stdout_logfile=/var/log/supervisord/devtools-frontend/devtools-frontend.log +stdout_logfile_maxbytes=50MB +redirect_stderr=true +environment=HOME="/home/kernel",USER="kernel" +user=kernel \ No newline at end of file diff --git a/supervisor/services-cloudrun/neko.conf b/supervisor/services-cloudrun/neko.conf new file mode 100644 index 0000000..969b62d --- /dev/null +++ b/supervisor/services-cloudrun/neko.conf @@ -0,0 +1,10 @@ +[program:neko] +command=/usr/bin/neko serve --server.static /var/www --server.bind 0.0.0.0:8081 +autostart=true +autorestart=true +startsecs=5 +priority=15 +stdout_logfile=/var/log/supervisord/neko/neko.log +stdout_logfile_maxbytes=50MB +redirect_stderr=true +environment=HOME="/home/kernel",USER="kernel",DISPLAY=":1",NEKO_WEBRTC_ICESERVERS_FRONTEND="",NEKO_WEBRTC_ICESERVERS_BACKEND="" \ No newline at end of file diff --git a/supervisor/services-cloudrun/xorg.conf b/supervisor/services-cloudrun/xorg.conf new file mode 100644 index 0000000..243c8f5 --- /dev/null +++ b/supervisor/services-cloudrun/xorg.conf @@ -0,0 +1,10 @@ +[program:xorg] +command=/usr/bin/Xorg :1 -config /etc/neko/xorg.conf -noreset -nolisten tcp +autostart=true +autorestart=true +startsecs=2 +priority=2 +stdout_logfile=/var/log/supervisord/xorg/xorg.log +stdout_logfile_maxbytes=50MB +redirect_stderr=true +environment=HOME="/home/kernel",USER="kernel" \ No newline at end of file diff --git a/twilio/README.md b/twilio/README.md new file mode 100644 index 0000000..3b77d32 --- /dev/null +++ b/twilio/README.md @@ -0,0 +1,79 @@ +# Twilio TURN Server Integration + +This folder contains scripts for integrating Twilio's Network Traversal Service to provide TURN server credentials for WebRTC in Cloud Run. + +## Scripts + +### `twilio-credential-updater.sh` +- **Purpose**: Called by `cloudrun-wrapper.sh` on container startup +- **Function**: Fetches fresh TURN credentials from Twilio API +- **Fallback**: Uses free TURN servers if Twilio fails +- **Environment Variables Required**: + - `TWILIO_ACCOUNT_SID` (API Key SID) + - `TWILIO_AUTH_TOKEN` (API Key Secret) + +### `twilio-token-service.js` +- **Purpose**: Node.js service for TURN credential generation +- **Features**: + - HTTP server mode (`--server` flag) + - One-time credential generation (default) + - Credential caching (1 hour) +- **Dependencies**: Express.js (for server mode) + +### `test-twilio-api.sh` +- **Purpose**: Test Twilio Network Traversal Service API +- **Usage**: `TWILIO_ACCOUNT_SID=xxx TWILIO_AUTH_TOKEN=xxx ./test-twilio-api.sh` +- **Output**: Formatted credentials for `NEKO_ICESERVERS` + +### `test-twilio-node.js` +- **Purpose**: Simple Node.js test for Twilio API +- **Usage**: Node.js version of the API test +- **Dependencies**: Only Node.js built-ins + +### `update-twilio-credentials.sh` +- **Purpose**: Update running Cloud Run service with fresh credentials +- **Usage**: Run periodically to refresh credentials +- **Features**: Direct Cloud Run service update + +## Integration + +The main integration point is in `../cloudrun-wrapper.sh`: + +```bash +# Get fresh Twilio TURN credentials if available +if [ -f /twilio-credential-updater.sh ]; then + echo "[cloudrun-wrapper] Getting fresh Twilio TURN credentials..." + source /twilio-credential-updater.sh +else + echo "[cloudrun-wrapper] Twilio updater not found, using credentials from environment" +fi +``` + +## Credentials Format + +Twilio Network Traversal Service returns credentials in this format: + +```json +{ + "ice_servers": [ + { + "url": "turn:global.turn.twilio.com:3478?transport=tcp", + "username": "long-generated-username", + "credential": "base64-encoded-credential" + } + ], + "ttl": "86400" +} +``` + +These are converted to neko format: + +```json +[ + { + "urls": ["turn:global.turn.twilio.com:3478?transport=tcp"], + "username": "long-generated-username", + "credential": "base64-encoded-credential" + } +] +``` \ No newline at end of file diff --git a/twilio/generate-twilio-credential.js b/twilio/generate-twilio-credential.js new file mode 100644 index 0000000..fcc384d --- /dev/null +++ b/twilio/generate-twilio-credential.js @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +const crypto = require('crypto'); + +// Twilio API credentials +const API_KEY_SID = 'SK5346918f48275d6571be927e84cfd6f8'; +const API_KEY_SECRET = process.env.TWILIO_API_KEY_SECRET || 'YOUR_API_KEY_SECRET_HERE'; + +// Time to live (in seconds) - 24 hours +const ttl = 86400; + +// Calculate expiration timestamp +const unixTimestamp = Math.floor(Date.now() / 1000) + ttl; + +// Create username (timestamp:apiKeySid) +const username = `${unixTimestamp}:${API_KEY_SID}`; + +// Generate password using HMAC-SHA1 +const password = crypto + .createHmac('sha1', API_KEY_SECRET) + .update(username) + .digest('base64'); + +console.log('Twilio TURN Credential Generator'); +console.log('=' .repeat(60)); +console.log('\nConfiguration:'); +console.log(`API Key SID: ${API_KEY_SID}`); +console.log(`API Key Secret: ${API_KEY_SECRET === 'YOUR_API_KEY_SECRET_HERE' ? '[NOT SET - Please provide]' : '[HIDDEN]'}`); +console.log(`TTL: ${ttl} seconds (${ttl/3600} hours)`); +console.log('\nGenerated Credentials:'); +console.log(`Username: ${username}`); +console.log(`Password: ${password}`); +console.log(`\nExpires at: ${new Date(unixTimestamp * 1000).toISOString()}`); + +console.log('\nFor service.yaml, use:'); +console.log(`- name: NEKO_ICESERVERS`); +console.log(` value: '[{"urls": ["turn:global.turn.twilio.com:3478?transport=tcp", "turns:global.turn.twilio.com:5349?transport=tcp"], "username": "${username}", "credential": "${password}"}]'`); + +if (API_KEY_SECRET === 'YOUR_API_KEY_SECRET_HERE') { + console.log('\nโš ๏ธ WARNING: You need to set the actual API Key Secret!'); + console.log('Run with: TWILIO_API_KEY_SECRET=your_actual_secret node generate-twilio-credential.js'); +} \ No newline at end of file diff --git a/twilio/test-twilio-api.sh b/twilio/test-twilio-api.sh new file mode 100755 index 0000000..a07a03d --- /dev/null +++ b/twilio/test-twilio-api.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Test Twilio Network Traversal Service API +# This generates temporary TURN credentials + +ACCOUNT_SID="${TWILIO_ACCOUNT_SID:-YOUR_ACCOUNT_SID}" +AUTH_TOKEN="${TWILIO_AUTH_TOKEN:-YOUR_AUTH_TOKEN}" + +echo "Testing Twilio Network Traversal Service API" +echo "============================================" +echo "Account SID: $ACCOUNT_SID" +echo "" + +# Make API call to get TURN credentials +echo "Requesting TURN credentials from Twilio..." +echo "" + +response=$(curl -s -X POST \ + "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json" \ + -u "${ACCOUNT_SID}:${AUTH_TOKEN}") + +# Check if request was successful +if echo "$response" | grep -q "ice_servers"; then + echo "โœ… Success! Received TURN credentials:" + echo "$response" | python3 -m json.tool + + # Extract and format for service.yaml + echo "" + echo "Formatted for NEKO_ICESERVERS:" + echo "$response" | python3 -c " +import json +import sys +data = json.load(sys.stdin) +servers = [] +for server in data.get('ice_servers', []): + if server.get('url', '').startswith('turn'): + url = server['url'] + if 'transport=' not in url: + url += '?transport=tcp' + servers.append({ + 'urls': [url], + 'username': server.get('username', ''), + 'credential': server.get('credential', '') + }) +print(json.dumps(servers)) +" +else + echo "โŒ Failed to get TURN credentials" + echo "Response: $response" + echo "" + echo "Make sure you have:" + echo "1. Valid Twilio Account SID and Auth Token" + echo "2. Network Traversal Service enabled on your Twilio account" +fi \ No newline at end of file diff --git a/twilio/test-twilio-node.js b/twilio/test-twilio-node.js new file mode 100644 index 0000000..da67c9a --- /dev/null +++ b/twilio/test-twilio-node.js @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +const https = require('https'); + +const ACCOUNT_SID = 'SK5346918f48275d6571be927e84cfd6f8'; +const AUTH_TOKEN = 'OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo'; + +function getTwilioTurnCredentials() { + return new Promise((resolve, reject) => { + console.log('Fetching TURN credentials from Twilio...'); + + const auth = Buffer.from(`${ACCOUNT_SID}:${AUTH_TOKEN}`).toString('base64'); + + const options = { + hostname: 'api.twilio.com', + port: 443, + path: `/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json`, + method: 'POST', + headers: { + 'Authorization': `Basic ${auth}`, + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': 0 + } + }; + + const req = https.request(options, (res) => { + let data = ''; + res.on('data', (chunk) => data += chunk); + res.on('end', () => { + try { + const response = JSON.parse(data); + if (res.statusCode === 201 || res.statusCode === 200) { + console.log('โœ… Success!'); + resolve(response.ice_servers || []); + } else { + reject(new Error(`API error: ${response.message}`)); + } + } catch (error) { + reject(error); + } + }); + }); + + req.on('error', reject); + req.end(); + }); +} + +getTwilioTurnCredentials() + .then(servers => { + const nekoServers = servers + .filter(s => s.url && s.url.startsWith('turn')) + .map(s => ({ + urls: [s.url], + username: s.username, + credential: s.credential + })); + + console.log('\nFormatted for NEKO_ICESERVERS:'); + console.log(JSON.stringify(nekoServers)); + }) + .catch(console.error); \ No newline at end of file diff --git a/twilio/test-twilio-turn.js b/twilio/test-twilio-turn.js new file mode 100644 index 0000000..58782d0 --- /dev/null +++ b/twilio/test-twilio-turn.js @@ -0,0 +1,45 @@ +#!/usr/bin/env node + +// Test Twilio TURN server credentials +const crypto = require('crypto'); + +// Parse the credentials from service.yaml +const username = "1757273052:SK5346918f48275d6571be927e84cfd6f8"; +const credential = "12HiXDndTPnUQZorm6TDDHd9Co8="; + +console.log("Testing Twilio TURN credentials:"); +console.log("Username:", username); +console.log("Credential:", credential); + +// Extract timestamp from username +const parts = username.split(':'); +const timestamp = parseInt(parts[0]); +const apiKeySid = parts[1]; + +console.log("\nParsed values:"); +console.log("Timestamp:", timestamp); +console.log("API Key SID:", apiKeySid); + +// Check if timestamp is valid (not expired) +const now = Math.floor(Date.now() / 1000); +const expiresIn = timestamp - now; + +console.log("\nTimestamp validation:"); +console.log("Current time (Unix):", now); +console.log("Credential timestamp:", timestamp); +console.log("Expires in:", expiresIn, "seconds"); + +if (expiresIn < 0) { + console.log("โŒ Credentials have EXPIRED!"); +} else { + console.log("โœ… Credentials are still valid for", Math.floor(expiresIn / 3600), "hours"); +} + +// To verify the credential, we would need the API Key Secret +// The credential should be: base64(hmac-sha1(username, apiKeySecret)) +console.log("\nNote: To fully verify the credential, we would need the API Key Secret."); +console.log("The credential should be computed as: base64(hmac-sha1(username, apiKeySecret))"); + +// Test with curl (requires actual network test) +console.log("\nTo test the TURN server directly, you can use a tool like 'turnutils_uclient':"); +console.log(`turnutils_uclient -T -p 3478 -u "${username}" -w "${credential}" turn:global.turn.twilio.com`); \ No newline at end of file diff --git a/twilio/twilio-credential-updater.sh b/twilio/twilio-credential-updater.sh new file mode 100644 index 0000000..43fe575 --- /dev/null +++ b/twilio/twilio-credential-updater.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Twilio TURN Credential Updater for Cloud Run +# This script is called from cloudrun-wrapper.sh to get fresh credentials on startup + +set -e + +# Check if we're using dynamic credentials mode (from Secret Manager) +if [ "$NEKO_ICESERVERS" = "DYNAMIC" ]; then + echo "[twilio-updater] Dynamic credentials mode - will fetch fresh TURN credentials" +elif [ -n "$NEKO_ICESERVERS" ] && [ "$NEKO_ICESERVERS" != "DYNAMIC" ]; then + # NEKO_ICESERVERS is already set with actual credentials + echo "[twilio-updater] Using pre-configured TURN credentials" + return 0 2>/dev/null || exit 0 +fi + +# Twilio credentials (passed as environment variables) +ACCOUNT_SID="${TWILIO_ACCOUNT_SID}" +AUTH_TOKEN="${TWILIO_AUTH_TOKEN}" + +if [ -z "$ACCOUNT_SID" ] || [ -z "$AUTH_TOKEN" ]; then + echo "[twilio-updater] Warning: Twilio credentials not set, using fallback TURN servers" + # Export fallback servers + export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' + return 0 2>/dev/null || exit 0 +fi + +echo "[twilio-updater] Fetching fresh TURN credentials from Twilio..." + +# Get TURN credentials from Twilio API +response=$(curl -s -X POST \ + "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json" \ + -u "${ACCOUNT_SID}:${AUTH_TOKEN}" 2>/dev/null) + +# Check if request was successful +if echo "$response" | grep -q "ice_servers"; then + # Format credentials for neko + ice_servers=$(echo "$response" | python3 -c " +import json +import sys +try: + data = json.load(sys.stdin) + servers = [] + for server in data.get('ice_servers', []): + if server.get('url', '').startswith('turn'): + url = server['url'] + if 'transport=' not in url: + url += '?transport=tcp' + servers.append({ + 'urls': [url], + 'username': server.get('username', ''), + 'credential': server.get('credential', '') + }) + print(json.dumps(servers)) +except: + print('[]') +" 2>/dev/null) + + if [ -n "$ice_servers" ] && [ "$ice_servers" != "[]" ]; then + echo "[twilio-updater] Successfully retrieved TURN credentials" + export NEKO_ICESERVERS="$ice_servers" + else + echo "[twilio-updater] Failed to parse TURN credentials, using fallback" + export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' + fi +else + echo "[twilio-updater] Failed to get TURN credentials from Twilio, using fallback" + echo "[twilio-updater] Response: ${response:0:100}..." + export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' +fi + +echo "[twilio-updater] NEKO_ICESERVERS set to: ${NEKO_ICESERVERS:0:100}..." \ No newline at end of file diff --git a/twilio/twilio-token-service.js b/twilio/twilio-token-service.js new file mode 100644 index 0000000..a845e78 --- /dev/null +++ b/twilio/twilio-token-service.js @@ -0,0 +1,168 @@ +#!/usr/bin/env node + +/** + * Twilio Network Traversal Service Token Generator + * Generates short-lived TURN credentials using Twilio's API + */ + +const https = require('https'); +const express = require('express'); + +// Twilio Account credentials (these are different from API Key) +const ACCOUNT_SID = process.env.TWILIO_ACCOUNT_SID || 'YOUR_ACCOUNT_SID'; +const AUTH_TOKEN = process.env.TWILIO_AUTH_TOKEN || 'YOUR_AUTH_TOKEN'; + +// Optional: API Key credentials (if using API keys instead of master credentials) +const API_KEY_SID = process.env.TWILIO_API_KEY_SID || 'SK5346918f48275d6571be927e84cfd6f8'; +const API_KEY_SECRET = process.env.TWILIO_API_KEY_SECRET || 'OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo'; + +// Cache for tokens +let tokenCache = null; +let tokenExpiry = 0; + +/** + * Get TURN credentials from Twilio Network Traversal Service + */ +async function getTwilioTurnCredentials() { + return new Promise((resolve, reject) => { + // Check cache first + if (tokenCache && Date.now() < tokenExpiry) { + console.log('Returning cached TURN credentials'); + return resolve(tokenCache); + } + + console.log('Fetching new TURN credentials from Twilio...'); + + // Twilio API endpoint for Network Traversal Service + const options = { + hostname: 'api.twilio.com', + port: 443, + path: `/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json`, + method: 'POST', + auth: `${ACCOUNT_SID}:${AUTH_TOKEN}`, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': 0 + } + }; + + const req = https.request(options, (res) => { + let data = ''; + + res.on('data', (chunk) => { + data += chunk; + }); + + res.on('end', () => { + try { + const response = JSON.parse(data); + + if (res.statusCode !== 201 && res.statusCode !== 200) { + console.error('Twilio API error:', response); + return reject(new Error(`Twilio API error: ${response.message || 'Unknown error'}`)); + } + + // Parse the ice_servers from response + const iceServers = response.ice_servers || []; + + // Cache for 1 hour (Twilio tokens are typically valid for 24 hours) + tokenCache = iceServers; + tokenExpiry = Date.now() + (60 * 60 * 1000); // 1 hour + + console.log(`Received ${iceServers.length} ICE servers from Twilio`); + resolve(iceServers); + } catch (error) { + reject(new Error(`Failed to parse Twilio response: ${error.message}`)); + } + }); + }); + + req.on('error', (error) => { + reject(new Error(`Twilio API request failed: ${error.message}`)); + }); + + req.end(); + }); +} + +/** + * Format ICE servers for neko + */ +function formatForNeko(twilioIceServers) { + // Twilio returns format: {"url": "...", "username": "...", "credential": "..."} + // Neko expects: {"urls": ["..."], "username": "...", "credential": "..."} + return twilioIceServers.map(server => { + if (server.url) { + // Add TCP transport for TURN servers in Cloud Run + let url = server.url; + if (url.startsWith('turn:') && !url.includes('transport=')) { + url += '?transport=tcp'; + } + + return { + urls: [url], + username: server.username, + credential: server.credential + }; + } + return server; + }).filter(server => { + // Only keep TURN servers for Cloud Run (STUN won't work) + return server.urls && server.urls[0] && server.urls[0].startsWith('turn'); + }); +} + +// Create Express server for health checks and credential endpoint +const app = express(); +const PORT = process.env.PORT || 3000; + +app.get('/health', (req, res) => { + res.json({ status: 'healthy' }); +}); + +app.get('/turn-credentials', async (req, res) => { + try { + const twilioServers = await getTwilioTurnCredentials(); + const nekoServers = formatForNeko(twilioServers); + + res.json({ + iceServers: nekoServers, + ttl: 3600, // 1 hour + expires: new Date(tokenExpiry).toISOString() + }); + } catch (error) { + console.error('Error getting TURN credentials:', error); + res.status(500).json({ error: error.message }); + } +}); + +// Standalone mode - get credentials and output for service.yaml +if (require.main === module) { + if (process.argv.includes('--server')) { + // Start HTTP server + app.listen(PORT, () => { + console.log(`Twilio token service listening on port ${PORT}`); + console.log(`Health check: http://localhost:${PORT}/health`); + console.log(`TURN credentials: http://localhost:${PORT}/turn-credentials`); + }); + } else { + // One-time credential generation + getTwilioTurnCredentials() + .then(twilioServers => { + const nekoServers = formatForNeko(twilioServers); + + console.log('\n=== Twilio TURN Credentials ==='); + console.log('For service.yaml, use:'); + console.log('- name: NEKO_ICESERVERS'); + console.log(` value: '${JSON.stringify(nekoServers)}'`); + console.log('\nCredentials expire in ~24 hours'); + console.log('Raw response:', JSON.stringify(twilioServers, null, 2)); + }) + .catch(error => { + console.error('Failed to get credentials:', error); + process.exit(1); + }); + } +} + +module.exports = { getTwilioTurnCredentials, formatForNeko }; \ No newline at end of file diff --git a/twilio/update-twilio-credentials.sh b/twilio/update-twilio-credentials.sh new file mode 100755 index 0000000..a468345 --- /dev/null +++ b/twilio/update-twilio-credentials.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Update Cloud Run service with fresh Twilio TURN credentials +# This script should be run periodically (e.g., every hour via cron) +# Run from the root directory: ./twilio/update-twilio-credentials.sh + +set -e + +# Configuration +PROJECT_ID="${PROJECT_ID:-func-241017}" +SERVICE_NAME="kernel-browser" +REGION="us-central1" + +# Twilio credentials (set these as environment variables) +TWILIO_ACCOUNT_SID="${TWILIO_ACCOUNT_SID}" +TWILIO_AUTH_TOKEN="${TWILIO_AUTH_TOKEN}" + +if [ -z "$TWILIO_ACCOUNT_SID" ] || [ -z "$TWILIO_AUTH_TOKEN" ]; then + echo "โŒ Error: TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN must be set" + echo " Export them as environment variables:" + echo " export TWILIO_ACCOUNT_SID=your_account_sid" + echo " export TWILIO_AUTH_TOKEN=your_auth_token" + exit 1 +fi + +echo "๐Ÿ”„ Fetching fresh TURN credentials from Twilio..." + +# Get TURN credentials from Twilio API +response=$(curl -s -X POST \ + "https://api.twilio.com/2010-04-01/Accounts/${TWILIO_ACCOUNT_SID}/Tokens.json" \ + -u "${TWILIO_ACCOUNT_SID}:${TWILIO_AUTH_TOKEN}") + +# Check if request was successful +if ! echo "$response" | grep -q "ice_servers"; then + echo "โŒ Failed to get TURN credentials from Twilio" + echo "Response: $response" + exit 1 +fi + +# Format credentials for neko +ice_servers=$(echo "$response" | python3 -c " +import json +import sys +data = json.load(sys.stdin) +servers = [] +for server in data.get('ice_servers', []): + if server.get('url', '').startswith('turn'): + url = server['url'] + if 'transport=' not in url: + url += '?transport=tcp' + servers.append({ + 'urls': [url], + 'username': server.get('username', ''), + 'credential': server.get('credential', '') + }) +print(json.dumps(servers)) +") + +echo "โœ… Received fresh TURN credentials" +echo " ICE Servers: $ice_servers" + +# Update Cloud Run service with new credentials +echo "๐Ÿš€ Updating Cloud Run service..." + +# Create a temporary service.yaml with updated credentials +cat > /tmp/service-update.yaml < { + console.log(`Testing ${useTLS ? 'TLS' : 'TCP'} connection to ${host}:${port}...`); + + const options = { + host: host, + port: port, + rejectUnauthorized: false + }; + + const socket = useTLS ? + tls.connect(options, () => { + console.log(`โœ… TLS connection established to ${host}:${port}`); + socket.end(); + resolve(true); + }) : + net.connect(options, () => { + console.log(`โœ… TCP connection established to ${host}:${port}`); + socket.end(); + resolve(true); + }); + + socket.on('error', (err) => { + console.log(`โŒ Failed to connect: ${err.message}`); + resolve(false); + }); + + socket.setTimeout(5000, () => { + console.log(`โŒ Connection timeout`); + socket.destroy(); + resolve(false); + }); + }); +} + +// Run tests +(async () => { + await testTurnServer('global.turn.twilio.com', 3478, false); + await testTurnServer('global.turn.twilio.com', 5349, true); + + console.log('\n' + '=' .repeat(60)); + console.log('If connections succeed, credentials should work in service.yaml'); + console.log('=' .repeat(60)); +})(); \ No newline at end of file From 432a8630eed4ea02fa6ccb2d37359c748e8f02b4 Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 17:35:14 -0500 Subject: [PATCH 07/19] Removed obsolete cloudrun script --- start-chromium-cloudrun.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 start-chromium-cloudrun.sh diff --git a/start-chromium-cloudrun.sh b/start-chromium-cloudrun.sh deleted file mode 100644 index 8e00e59..0000000 --- a/start-chromium-cloudrun.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -o pipefail -o errexit -o nounset - -# Cloud Run optimized Chromium launcher - no runuser needed since we're already kernel user - -echo "Starting Chromium launcher (Cloud Run mode)" - -# Resolve internal port for the remote debugging interface -INTERNAL_PORT="${INTERNAL_PORT:-9223}" - -# Load additional Chromium flags from env and optional file -CHROMIUM_FLAGS="${CHROMIUM_FLAGS:-}" -if [[ -f /chromium/flags ]]; then - CHROMIUM_FLAGS="$CHROMIUM_FLAGS $(cat /chromium/flags)" -fi -echo "CHROMIUM_FLAGS: $CHROMIUM_FLAGS" - -# Always use display :1 and point DBus to the system bus socket -export DISPLAY=":1" -export DBUS_SESSION_BUS_ADDRESS="unix:path=/tmp/dbus/system_bus_socket" -export XDG_CONFIG_HOME=/home/kernel/.config -export XDG_CACHE_HOME=/home/kernel/.cache -export HOME=/home/kernel - -echo "Running chromium as kernel user (Cloud Run mode)" -exec chromium \ - --remote-debugging-port="$INTERNAL_PORT" \ - --user-data-dir=/home/kernel/user-data \ - --password-store=basic \ - --no-first-run \ - ${CHROMIUM_FLAGS:-} \ No newline at end of file From 59989593abdc170336aed9c93620c422d4c06f30 Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 17:39:59 -0500 Subject: [PATCH 08/19] Fix Makefile --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index 4bc3420..7c9e9eb 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,7 @@ restart: ## Restart containers @$(MAKE) --no-print-directory compose-up logs: ## Show container logs -<<<<<<< HEAD - docker-compose logs -f kernel-browser || docker logs -f kernel-browser-local -======= docker-compose logs -f kernel-browser || docker logs -f kernel-browser-extended ->>>>>>> main status: ## Show container status @echo "Docker Compose Status:" From f908215c266e0114922178028ae74be93c473780 Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 18:12:13 -0500 Subject: [PATCH 09/19] Fix the build --- cloudbuild.yaml | 32 ++++++++++++-------------------- deploy.sh | 2 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index d178da5..a5b1663 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -41,43 +41,35 @@ steps: - 'push' - 'us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest' - # Step 5: Choose appropriate service.yaml based on secrets availability + # Step 5: Deploy to Cloud Run with appropriate service.yaml - name: 'gcr.io/cloud-builders/gcloud' entrypoint: 'bash' args: - '-c' - | - # Check if Twilio secrets exist + # Check if Twilio secrets exist and choose appropriate service file if gcloud secrets describe twilio-account-sid --project=$PROJECT_ID >/dev/null 2>&1 && \ gcloud secrets describe twilio-auth-token --project=$PROJECT_ID >/dev/null 2>&1; then echo "Using service-secrets.yaml with Secret Manager references" - SERVICE_YAML="service-secrets.yaml" + SERVICE_FILE="service-secrets.yaml" else echo "Using standard service.yaml (secrets not configured)" - SERVICE_YAML="service.yaml" + SERVICE_FILE="service.yaml" fi # Update project ID in the chosen service file - sed -i "s/PROJECT_ID/$PROJECT_ID/g" $SERVICE_YAML + cp $$SERVICE_FILE temp-service.yaml + sed -i "s/PROJECT_ID/$PROJECT_ID/g" temp-service.yaml - echo "Deploying with: $SERVICE_YAML" - cat $SERVICE_YAML + echo "Deploying with: $$SERVICE_FILE" + cat temp-service.yaml - # Save the choice for next step - echo $SERVICE_YAML > /workspace/service_choice.txt - - # Step 6: Deploy to Cloud Run - - name: 'gcr.io/cloud-builders/gcloud' - entrypoint: 'bash' - args: - - '-c' - - | - SERVICE_YAML=$(cat /workspace/service_choice.txt) - gcloud run services replace $SERVICE_YAML \ + # Deploy to Cloud Run + gcloud run services replace temp-service.yaml \ --region=us-central1 \ --quiet - # Step 7: Update traffic to latest revision + # Step 6: Update traffic to latest revision - name: 'gcr.io/cloud-builders/gcloud' args: - 'run' @@ -88,7 +80,7 @@ steps: - '--region=us-central1' - '--quiet' - # Step 8: Get the service URL + # Step 7: Get the service URL - name: 'gcr.io/cloud-builders/gcloud' args: - 'run' diff --git a/deploy.sh b/deploy.sh index 50527e1..048bb3f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -40,7 +40,7 @@ load_env_file() { info "Loading configuration from .env file..." # Export variables from .env, ignoring comments and empty lines set -a - source <(grep -v '^#' .env | grep -v '^$') + . .env set +a success "Configuration loaded from .env" elif [ -f .env.example ]; then From 6ff623db22cecefc15f2f8027caf1261d000256f Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 20:28:40 -0500 Subject: [PATCH 10/19] Fixed conlict in the run-local.sh --- run-local.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/run-local.sh b/run-local.sh index d5dae5a..75b8dc0 100755 --- a/run-local.sh +++ b/run-local.sh @@ -50,21 +50,6 @@ echo "" echo "๐Ÿƒ Starting extended container with kernel-images run system..." -<<<<<<< HEAD -# Backup original run-docker.sh to modify port mappings -if [ ! -f run-docker.sh.original ]; then - cp run-docker.sh run-docker.sh.original -fi - -# Create modified run script that adds DevTools port mapping -cat run-docker.sh.original | \ -sed 's/docker run -it/docker run -it -p 8001:8001/' > run-docker.sh.extended - -chmod +x run-docker.sh.extended - -# Run using the modified run script with DevTools port -./run-docker.sh.extended -======= # Execute the kernel-images script setup but override the final docker run command # We'll replicate the essential parts here to avoid the sed hack @@ -120,7 +105,6 @@ fi # Run with our additional DevTools port mapping docker rm -f "$NAME" 2>/dev/null || true docker run -it "${RUN_ARGS[@]}" "$IMAGE" ->>>>>>> main echo "" echo "๐ŸŒ Extended service should be accessible at:" From f7a49cbad8a191250ac3409e2dff89091b7c0f8d Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 20:31:27 -0500 Subject: [PATCH 11/19] Remove test script --- twilio/verify-twilio.js | 84 ----------------------------------------- 1 file changed, 84 deletions(-) delete mode 100644 twilio/verify-twilio.js diff --git a/twilio/verify-twilio.js b/twilio/verify-twilio.js deleted file mode 100644 index 7d91745..0000000 --- a/twilio/verify-twilio.js +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env node - -const crypto = require('crypto'); - -// Your Twilio API credentials -const API_KEY_SID = 'SK5346918f48275d6571be927e84cfd6f8'; -const API_KEY_SECRET = 'OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo'; - -// Generate credentials -const ttl = 86400; // 24 hours -const unixTimestamp = Math.floor(Date.now() / 1000) + ttl; -const username = `${unixTimestamp}:${API_KEY_SID}`; -const password = crypto - .createHmac('sha1', API_KEY_SECRET) - .update(username) - .digest('base64'); - -console.log('Testing Twilio TURN credentials locally'); -console.log('=' .repeat(60)); -console.log('Username:', username); -console.log('Password:', password); -console.log('=' .repeat(60)); - -// Test with curl commands -console.log('\nTest commands to verify TURN server access:\n'); - -// Test STUN binding -console.log('1. Test STUN binding (should work without auth):'); -console.log(`curl -X POST "https://global.turn.twilio.com:5349" --http1.1 -k`); - -// Test with turnutils if available -console.log('\n2. Test with turnutils_uclient (if installed):'); -console.log(`turnutils_uclient -T -p 3478 -u "${username}" -w "${password}" turn:global.turn.twilio.com`); - -// Test with Node.js TURN client -console.log('\n3. Testing connection with Node.js...\n'); - -const net = require('net'); -const tls = require('tls'); - -function testTurnServer(host, port, useTLS = false) { - return new Promise((resolve, reject) => { - console.log(`Testing ${useTLS ? 'TLS' : 'TCP'} connection to ${host}:${port}...`); - - const options = { - host: host, - port: port, - rejectUnauthorized: false - }; - - const socket = useTLS ? - tls.connect(options, () => { - console.log(`โœ… TLS connection established to ${host}:${port}`); - socket.end(); - resolve(true); - }) : - net.connect(options, () => { - console.log(`โœ… TCP connection established to ${host}:${port}`); - socket.end(); - resolve(true); - }); - - socket.on('error', (err) => { - console.log(`โŒ Failed to connect: ${err.message}`); - resolve(false); - }); - - socket.setTimeout(5000, () => { - console.log(`โŒ Connection timeout`); - socket.destroy(); - resolve(false); - }); - }); -} - -// Run tests -(async () => { - await testTurnServer('global.turn.twilio.com', 3478, false); - await testTurnServer('global.turn.twilio.com', 5349, true); - - console.log('\n' + '=' .repeat(60)); - console.log('If connections succeed, credentials should work in service.yaml'); - console.log('=' .repeat(60)); -})(); \ No newline at end of file From 457f8826a2220318c7d4697fdecff1d14ac7a04b Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 20:32:31 -0500 Subject: [PATCH 12/19] Remove test file --- twilio/test-twilio-node.js | 62 -------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 twilio/test-twilio-node.js diff --git a/twilio/test-twilio-node.js b/twilio/test-twilio-node.js deleted file mode 100644 index da67c9a..0000000 --- a/twilio/test-twilio-node.js +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env node - -const https = require('https'); - -const ACCOUNT_SID = 'SK5346918f48275d6571be927e84cfd6f8'; -const AUTH_TOKEN = 'OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo'; - -function getTwilioTurnCredentials() { - return new Promise((resolve, reject) => { - console.log('Fetching TURN credentials from Twilio...'); - - const auth = Buffer.from(`${ACCOUNT_SID}:${AUTH_TOKEN}`).toString('base64'); - - const options = { - hostname: 'api.twilio.com', - port: 443, - path: `/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json`, - method: 'POST', - headers: { - 'Authorization': `Basic ${auth}`, - 'Content-Type': 'application/x-www-form-urlencoded', - 'Content-Length': 0 - } - }; - - const req = https.request(options, (res) => { - let data = ''; - res.on('data', (chunk) => data += chunk); - res.on('end', () => { - try { - const response = JSON.parse(data); - if (res.statusCode === 201 || res.statusCode === 200) { - console.log('โœ… Success!'); - resolve(response.ice_servers || []); - } else { - reject(new Error(`API error: ${response.message}`)); - } - } catch (error) { - reject(error); - } - }); - }); - - req.on('error', reject); - req.end(); - }); -} - -getTwilioTurnCredentials() - .then(servers => { - const nekoServers = servers - .filter(s => s.url && s.url.startsWith('turn')) - .map(s => ({ - urls: [s.url], - username: s.username, - credential: s.credential - })); - - console.log('\nFormatted for NEKO_ICESERVERS:'); - console.log(JSON.stringify(nekoServers)); - }) - .catch(console.error); \ No newline at end of file From fbad4843499158913957d7f4e7cc5d3951d87a22 Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 20:33:47 -0500 Subject: [PATCH 13/19] Remove test files --- twilio/generate-twilio-credential.js | 42 ------- twilio/test-twilio-api.sh | 54 --------- twilio/test-twilio-turn.js | 45 ------- twilio/twilio-token-service.js | 168 --------------------------- 4 files changed, 309 deletions(-) delete mode 100644 twilio/generate-twilio-credential.js delete mode 100755 twilio/test-twilio-api.sh delete mode 100644 twilio/test-twilio-turn.js delete mode 100644 twilio/twilio-token-service.js diff --git a/twilio/generate-twilio-credential.js b/twilio/generate-twilio-credential.js deleted file mode 100644 index fcc384d..0000000 --- a/twilio/generate-twilio-credential.js +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env node - -const crypto = require('crypto'); - -// Twilio API credentials -const API_KEY_SID = 'SK5346918f48275d6571be927e84cfd6f8'; -const API_KEY_SECRET = process.env.TWILIO_API_KEY_SECRET || 'YOUR_API_KEY_SECRET_HERE'; - -// Time to live (in seconds) - 24 hours -const ttl = 86400; - -// Calculate expiration timestamp -const unixTimestamp = Math.floor(Date.now() / 1000) + ttl; - -// Create username (timestamp:apiKeySid) -const username = `${unixTimestamp}:${API_KEY_SID}`; - -// Generate password using HMAC-SHA1 -const password = crypto - .createHmac('sha1', API_KEY_SECRET) - .update(username) - .digest('base64'); - -console.log('Twilio TURN Credential Generator'); -console.log('=' .repeat(60)); -console.log('\nConfiguration:'); -console.log(`API Key SID: ${API_KEY_SID}`); -console.log(`API Key Secret: ${API_KEY_SECRET === 'YOUR_API_KEY_SECRET_HERE' ? '[NOT SET - Please provide]' : '[HIDDEN]'}`); -console.log(`TTL: ${ttl} seconds (${ttl/3600} hours)`); -console.log('\nGenerated Credentials:'); -console.log(`Username: ${username}`); -console.log(`Password: ${password}`); -console.log(`\nExpires at: ${new Date(unixTimestamp * 1000).toISOString()}`); - -console.log('\nFor service.yaml, use:'); -console.log(`- name: NEKO_ICESERVERS`); -console.log(` value: '[{"urls": ["turn:global.turn.twilio.com:3478?transport=tcp", "turns:global.turn.twilio.com:5349?transport=tcp"], "username": "${username}", "credential": "${password}"}]'`); - -if (API_KEY_SECRET === 'YOUR_API_KEY_SECRET_HERE') { - console.log('\nโš ๏ธ WARNING: You need to set the actual API Key Secret!'); - console.log('Run with: TWILIO_API_KEY_SECRET=your_actual_secret node generate-twilio-credential.js'); -} \ No newline at end of file diff --git a/twilio/test-twilio-api.sh b/twilio/test-twilio-api.sh deleted file mode 100755 index a07a03d..0000000 --- a/twilio/test-twilio-api.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# Test Twilio Network Traversal Service API -# This generates temporary TURN credentials - -ACCOUNT_SID="${TWILIO_ACCOUNT_SID:-YOUR_ACCOUNT_SID}" -AUTH_TOKEN="${TWILIO_AUTH_TOKEN:-YOUR_AUTH_TOKEN}" - -echo "Testing Twilio Network Traversal Service API" -echo "============================================" -echo "Account SID: $ACCOUNT_SID" -echo "" - -# Make API call to get TURN credentials -echo "Requesting TURN credentials from Twilio..." -echo "" - -response=$(curl -s -X POST \ - "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json" \ - -u "${ACCOUNT_SID}:${AUTH_TOKEN}") - -# Check if request was successful -if echo "$response" | grep -q "ice_servers"; then - echo "โœ… Success! Received TURN credentials:" - echo "$response" | python3 -m json.tool - - # Extract and format for service.yaml - echo "" - echo "Formatted for NEKO_ICESERVERS:" - echo "$response" | python3 -c " -import json -import sys -data = json.load(sys.stdin) -servers = [] -for server in data.get('ice_servers', []): - if server.get('url', '').startswith('turn'): - url = server['url'] - if 'transport=' not in url: - url += '?transport=tcp' - servers.append({ - 'urls': [url], - 'username': server.get('username', ''), - 'credential': server.get('credential', '') - }) -print(json.dumps(servers)) -" -else - echo "โŒ Failed to get TURN credentials" - echo "Response: $response" - echo "" - echo "Make sure you have:" - echo "1. Valid Twilio Account SID and Auth Token" - echo "2. Network Traversal Service enabled on your Twilio account" -fi \ No newline at end of file diff --git a/twilio/test-twilio-turn.js b/twilio/test-twilio-turn.js deleted file mode 100644 index 58782d0..0000000 --- a/twilio/test-twilio-turn.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node - -// Test Twilio TURN server credentials -const crypto = require('crypto'); - -// Parse the credentials from service.yaml -const username = "1757273052:SK5346918f48275d6571be927e84cfd6f8"; -const credential = "12HiXDndTPnUQZorm6TDDHd9Co8="; - -console.log("Testing Twilio TURN credentials:"); -console.log("Username:", username); -console.log("Credential:", credential); - -// Extract timestamp from username -const parts = username.split(':'); -const timestamp = parseInt(parts[0]); -const apiKeySid = parts[1]; - -console.log("\nParsed values:"); -console.log("Timestamp:", timestamp); -console.log("API Key SID:", apiKeySid); - -// Check if timestamp is valid (not expired) -const now = Math.floor(Date.now() / 1000); -const expiresIn = timestamp - now; - -console.log("\nTimestamp validation:"); -console.log("Current time (Unix):", now); -console.log("Credential timestamp:", timestamp); -console.log("Expires in:", expiresIn, "seconds"); - -if (expiresIn < 0) { - console.log("โŒ Credentials have EXPIRED!"); -} else { - console.log("โœ… Credentials are still valid for", Math.floor(expiresIn / 3600), "hours"); -} - -// To verify the credential, we would need the API Key Secret -// The credential should be: base64(hmac-sha1(username, apiKeySecret)) -console.log("\nNote: To fully verify the credential, we would need the API Key Secret."); -console.log("The credential should be computed as: base64(hmac-sha1(username, apiKeySecret))"); - -// Test with curl (requires actual network test) -console.log("\nTo test the TURN server directly, you can use a tool like 'turnutils_uclient':"); -console.log(`turnutils_uclient -T -p 3478 -u "${username}" -w "${credential}" turn:global.turn.twilio.com`); \ No newline at end of file diff --git a/twilio/twilio-token-service.js b/twilio/twilio-token-service.js deleted file mode 100644 index a845e78..0000000 --- a/twilio/twilio-token-service.js +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env node - -/** - * Twilio Network Traversal Service Token Generator - * Generates short-lived TURN credentials using Twilio's API - */ - -const https = require('https'); -const express = require('express'); - -// Twilio Account credentials (these are different from API Key) -const ACCOUNT_SID = process.env.TWILIO_ACCOUNT_SID || 'YOUR_ACCOUNT_SID'; -const AUTH_TOKEN = process.env.TWILIO_AUTH_TOKEN || 'YOUR_AUTH_TOKEN'; - -// Optional: API Key credentials (if using API keys instead of master credentials) -const API_KEY_SID = process.env.TWILIO_API_KEY_SID || 'SK5346918f48275d6571be927e84cfd6f8'; -const API_KEY_SECRET = process.env.TWILIO_API_KEY_SECRET || 'OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo'; - -// Cache for tokens -let tokenCache = null; -let tokenExpiry = 0; - -/** - * Get TURN credentials from Twilio Network Traversal Service - */ -async function getTwilioTurnCredentials() { - return new Promise((resolve, reject) => { - // Check cache first - if (tokenCache && Date.now() < tokenExpiry) { - console.log('Returning cached TURN credentials'); - return resolve(tokenCache); - } - - console.log('Fetching new TURN credentials from Twilio...'); - - // Twilio API endpoint for Network Traversal Service - const options = { - hostname: 'api.twilio.com', - port: 443, - path: `/2010-04-01/Accounts/${ACCOUNT_SID}/Tokens.json`, - method: 'POST', - auth: `${ACCOUNT_SID}:${AUTH_TOKEN}`, - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Content-Length': 0 - } - }; - - const req = https.request(options, (res) => { - let data = ''; - - res.on('data', (chunk) => { - data += chunk; - }); - - res.on('end', () => { - try { - const response = JSON.parse(data); - - if (res.statusCode !== 201 && res.statusCode !== 200) { - console.error('Twilio API error:', response); - return reject(new Error(`Twilio API error: ${response.message || 'Unknown error'}`)); - } - - // Parse the ice_servers from response - const iceServers = response.ice_servers || []; - - // Cache for 1 hour (Twilio tokens are typically valid for 24 hours) - tokenCache = iceServers; - tokenExpiry = Date.now() + (60 * 60 * 1000); // 1 hour - - console.log(`Received ${iceServers.length} ICE servers from Twilio`); - resolve(iceServers); - } catch (error) { - reject(new Error(`Failed to parse Twilio response: ${error.message}`)); - } - }); - }); - - req.on('error', (error) => { - reject(new Error(`Twilio API request failed: ${error.message}`)); - }); - - req.end(); - }); -} - -/** - * Format ICE servers for neko - */ -function formatForNeko(twilioIceServers) { - // Twilio returns format: {"url": "...", "username": "...", "credential": "..."} - // Neko expects: {"urls": ["..."], "username": "...", "credential": "..."} - return twilioIceServers.map(server => { - if (server.url) { - // Add TCP transport for TURN servers in Cloud Run - let url = server.url; - if (url.startsWith('turn:') && !url.includes('transport=')) { - url += '?transport=tcp'; - } - - return { - urls: [url], - username: server.username, - credential: server.credential - }; - } - return server; - }).filter(server => { - // Only keep TURN servers for Cloud Run (STUN won't work) - return server.urls && server.urls[0] && server.urls[0].startsWith('turn'); - }); -} - -// Create Express server for health checks and credential endpoint -const app = express(); -const PORT = process.env.PORT || 3000; - -app.get('/health', (req, res) => { - res.json({ status: 'healthy' }); -}); - -app.get('/turn-credentials', async (req, res) => { - try { - const twilioServers = await getTwilioTurnCredentials(); - const nekoServers = formatForNeko(twilioServers); - - res.json({ - iceServers: nekoServers, - ttl: 3600, // 1 hour - expires: new Date(tokenExpiry).toISOString() - }); - } catch (error) { - console.error('Error getting TURN credentials:', error); - res.status(500).json({ error: error.message }); - } -}); - -// Standalone mode - get credentials and output for service.yaml -if (require.main === module) { - if (process.argv.includes('--server')) { - // Start HTTP server - app.listen(PORT, () => { - console.log(`Twilio token service listening on port ${PORT}`); - console.log(`Health check: http://localhost:${PORT}/health`); - console.log(`TURN credentials: http://localhost:${PORT}/turn-credentials`); - }); - } else { - // One-time credential generation - getTwilioTurnCredentials() - .then(twilioServers => { - const nekoServers = formatForNeko(twilioServers); - - console.log('\n=== Twilio TURN Credentials ==='); - console.log('For service.yaml, use:'); - console.log('- name: NEKO_ICESERVERS'); - console.log(` value: '${JSON.stringify(nekoServers)}'`); - console.log('\nCredentials expire in ~24 hours'); - console.log('Raw response:', JSON.stringify(twilioServers, null, 2)); - }) - .catch(error => { - console.error('Failed to get credentials:', error); - process.exit(1); - }); - } -} - -module.exports = { getTwilioTurnCredentials, formatForNeko }; \ No newline at end of file From 202b50bb16ba2e3adcb5308a85232aa7a5b211ec Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 20:44:58 -0500 Subject: [PATCH 14/19] Remove hardcoded credentials --- cloudbuild.yaml | 4 ++-- service.yaml | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index a5b1663..e2b34a9 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -58,10 +58,10 @@ steps: fi # Update project ID in the chosen service file - cp $$SERVICE_FILE temp-service.yaml + cp $SERVICE_FILE temp-service.yaml sed -i "s/PROJECT_ID/$PROJECT_ID/g" temp-service.yaml - echo "Deploying with: $$SERVICE_FILE" + echo "Deploying with: $SERVICE_FILE" cat temp-service.yaml # Deploy to Cloud Run diff --git a/service.yaml b/service.yaml index f39c167..8013a07 100644 --- a/service.yaml +++ b/service.yaml @@ -59,13 +59,10 @@ spec: - name: WIDTH value: "1024" # Twilio API Key credentials for dynamic TURN generation - - name: TWILIO_ACCOUNT_SID - value: "SK5346918f48275d6571be927e84cfd6f8" - - name: TWILIO_AUTH_TOKEN - value: "OWJDRGxZZnxUlwOVXbupRs9yhQaylXzo" - # Fresh Twilio TURN credentials (manually generated for now) + # No Twilio credentials in fallback mode - credentials handled by twilio-credential-updater.sh + # Fallback TURN servers (used when Twilio credentials are not configured) - name: NEKO_ICESERVERS - value: '[{"urls":["turn:global.turn.twilio.com:3478?transport=tcp"],"username":"b88cfa1369190aa9cbc8bfaca683c457476b5d7062aa0a7b184c87db3ade0ff5","credential":"m3oPEt94gQQP+g2yd4R32MuZtCCdw6Rmmuvkp6/Dkd0="},{"urls":["turn:global.turn.twilio.com:443?transport=tcp"],"username":"b88cfa1369190aa9cbc8bfaca683c457476b5d7062aa0a7b184c87db3ade0ff5","credential":"m3oPEt94gQQP+g2yd4R32MuZtCCdw6Rmmuvkp6/Dkd0="}]' + value: '[{"urls":["stun:stun.l.google.com:19302"]},{"urls":["turn:openrelay.metered.ca:80"],"username":"openrelayproject","credential":"openrelayproject"}]' # Disable TCP multiplexing (nginx handles port 8080) - name: NEKO_WEBRTC_TCPMUX value: "0" @@ -83,7 +80,7 @@ spec: value: "/tmp/recordings" # Force new revision - name: DEPLOYMENT_VERSION - value: "v11-fresh-twilio-credentials" + value: "v12-fallback-configuration" traffic: - percent: 100 latestRevision: true \ No newline at end of file From 40b559056bb5c22b25de245d28471cbdecf8e71d Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 20:48:49 -0500 Subject: [PATCH 15/19] Fixed cloubuild.yaml --- cloudbuild.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index e2b34a9..4d2bfe2 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -51,17 +51,16 @@ steps: if gcloud secrets describe twilio-account-sid --project=$PROJECT_ID >/dev/null 2>&1 && \ gcloud secrets describe twilio-auth-token --project=$PROJECT_ID >/dev/null 2>&1; then echo "Using service-secrets.yaml with Secret Manager references" - SERVICE_FILE="service-secrets.yaml" + cp service-secrets.yaml temp-service.yaml else echo "Using standard service.yaml (secrets not configured)" - SERVICE_FILE="service.yaml" + cp service.yaml temp-service.yaml fi # Update project ID in the chosen service file - cp $SERVICE_FILE temp-service.yaml sed -i "s/PROJECT_ID/$PROJECT_ID/g" temp-service.yaml - echo "Deploying with: $SERVICE_FILE" + echo "Deploying service configuration:" cat temp-service.yaml # Deploy to Cloud Run From 5f9e3a0cab789ddf2dc831db156e19e03de44a3c Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Mon, 8 Sep 2025 21:17:38 -0500 Subject: [PATCH 16/19] Switching to TCP instead UDP for WebRTC --- service-secrets.yaml | 10 ++++++--- service.yaml | 14 ++++++++----- twilio/twilio-credential-updater.sh | 32 ++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/service-secrets.yaml b/service-secrets.yaml index 34b4b91..f3c983e 100644 --- a/service-secrets.yaml +++ b/service-secrets.yaml @@ -73,9 +73,13 @@ spec: # The twilio-credential-updater.sh script will use these at startup - name: NEKO_ICESERVERS value: 'DYNAMIC' # Placeholder - will be replaced by twilio-credential-updater.sh - # Disable TCP multiplexing (nginx handles port 8080) + # TCP-only WebRTC configuration for Cloud Run - name: NEKO_WEBRTC_TCPMUX - value: "0" + value: "0" # Disable TCP multiplexing (nginx handles port 8080) + - name: NEKO_WEBRTC_ICE_LITE + value: "true" # Use ICE-Lite mode for server + - name: NEKO_WEBRTC_ICE_SERVERS_ONLY_TURN + value: "true" # Only use TURN servers, no STUN # Optional: Google Cloud Storage bucket for recordings - name: GCS_BUCKET value: "kernel-browser-recordings" @@ -90,7 +94,7 @@ spec: value: "/tmp/recordings" # Force new revision - name: DEPLOYMENT_VERSION - value: "v12-secret-manager" + value: "v13-tcp-only-webrtc" traffic: - percent: 100 latestRevision: true \ No newline at end of file diff --git a/service.yaml b/service.yaml index 8013a07..e13d6e9 100644 --- a/service.yaml +++ b/service.yaml @@ -60,12 +60,16 @@ spec: value: "1024" # Twilio API Key credentials for dynamic TURN generation # No Twilio credentials in fallback mode - credentials handled by twilio-credential-updater.sh - # Fallback TURN servers (used when Twilio credentials are not configured) + # TCP-only TURN servers for Cloud Run (no STUN/UDP) - name: NEKO_ICESERVERS - value: '[{"urls":["stun:stun.l.google.com:19302"]},{"urls":["turn:openrelay.metered.ca:80"],"username":"openrelayproject","credential":"openrelayproject"}]' - # Disable TCP multiplexing (nginx handles port 8080) + value: '[{"urls":["turn:openrelay.metered.ca:80?transport=tcp"],"username":"openrelayproject","credential":"openrelayproject"},{"urls":["turns:openrelay.metered.ca:443?transport=tcp"],"username":"openrelayproject","credential":"openrelayproject"}]' + # TCP-only WebRTC configuration for Cloud Run - name: NEKO_WEBRTC_TCPMUX - value: "0" + value: "0" # Disable TCP multiplexing (nginx handles port 8080) + - name: NEKO_WEBRTC_ICE_LITE + value: "true" # Use ICE-Lite mode for server + - name: NEKO_WEBRTC_ICE_SERVERS_ONLY_TURN + value: "true" # Only use TURN servers, no STUN # Optional: Google Cloud Storage bucket for recordings - name: GCS_BUCKET value: "kernel-browser-recordings" @@ -80,7 +84,7 @@ spec: value: "/tmp/recordings" # Force new revision - name: DEPLOYMENT_VERSION - value: "v12-fallback-configuration" + value: "v13-tcp-only-fallback" traffic: - percent: 100 latestRevision: true \ No newline at end of file diff --git a/twilio/twilio-credential-updater.sh b/twilio/twilio-credential-updater.sh index 43fe575..80255ea 100644 --- a/twilio/twilio-credential-updater.sh +++ b/twilio/twilio-credential-updater.sh @@ -19,9 +19,9 @@ ACCOUNT_SID="${TWILIO_ACCOUNT_SID}" AUTH_TOKEN="${TWILIO_AUTH_TOKEN}" if [ -z "$ACCOUNT_SID" ] || [ -z "$AUTH_TOKEN" ]; then - echo "[twilio-updater] Warning: Twilio credentials not set, using fallback TURN servers" - # Export fallback servers - export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' + echo "[twilio-updater] Warning: Twilio credentials not set, using TCP-only fallback TURN servers" + # Export TCP-only fallback servers (no STUN for Cloud Run) + export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}, {"urls": ["turns:openrelay.metered.ca:443?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' return 0 2>/dev/null || exit 0 fi @@ -34,7 +34,7 @@ response=$(curl -s -X POST \ # Check if request was successful if echo "$response" | grep -q "ice_servers"; then - # Format credentials for neko + # Format credentials for neko (TCP-only for Cloud Run) ice_servers=$(echo "$response" | python3 -c " import json import sys @@ -44,13 +44,25 @@ try: for server in data.get('ice_servers', []): if server.get('url', '').startswith('turn'): url = server['url'] - if 'transport=' not in url: - url += '?transport=tcp' + # Force TCP transport for Cloud Run compatibility + if '?transport=' in url: + url = url.split('?transport=')[0] + url += '?transport=tcp' servers.append({ 'urls': [url], 'username': server.get('username', ''), 'credential': server.get('credential', '') }) + + # Also add TLS version for redundancy + tls_url = url.replace('turn:', 'turns:').replace(':3478', ':5349') + servers.append({ + 'urls': [tls_url], + 'username': server.get('username', ''), + 'credential': server.get('credential', '') + }) + + # Remove STUN servers - only use TURN for Cloud Run print(json.dumps(servers)) except: print('[]') @@ -60,13 +72,13 @@ except: echo "[twilio-updater] Successfully retrieved TURN credentials" export NEKO_ICESERVERS="$ice_servers" else - echo "[twilio-updater] Failed to parse TURN credentials, using fallback" - export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' + echo "[twilio-updater] Failed to parse TURN credentials, using TCP-only fallback" + export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}, {"urls": ["turns:openrelay.metered.ca:443?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' fi else - echo "[twilio-updater] Failed to get TURN credentials from Twilio, using fallback" + echo "[twilio-updater] Failed to get TURN credentials from Twilio, using TCP-only fallback" echo "[twilio-updater] Response: ${response:0:100}..." - export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' + export NEKO_ICESERVERS='[{"urls": ["turn:openrelay.metered.ca:80?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}, {"urls": ["turns:openrelay.metered.ca:443?transport=tcp"], "username": "openrelayproject", "credential": "openrelayproject"}]' fi echo "[twilio-updater] NEKO_ICESERVERS set to: ${NEKO_ICESERVERS:0:100}..." \ No newline at end of file From 0a3ab42b650cca8a3ce3004c93d0dba3952d59c2 Mon Sep 17 00:00:00 2001 From: Tyson Thomas Date: Tue, 9 Sep 2025 08:51:06 -0700 Subject: [PATCH 17/19] fix the deployment issue --- service-secrets.yaml | 2 +- service.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/service-secrets.yaml b/service-secrets.yaml index f3c983e..d265334 100644 --- a/service-secrets.yaml +++ b/service-secrets.yaml @@ -28,7 +28,7 @@ spec: containers: - name: kernel-browser # This will be set during deployment - image: us-docker.pkg.dev/func-241017/gcr.io/kernel-browser:latest + image: us-docker.pkg.dev/PROJECT_ID/gcr.io/kernel-browser:latest ports: - name: http1 containerPort: 8080 diff --git a/service.yaml b/service.yaml index e13d6e9..c78c45b 100644 --- a/service.yaml +++ b/service.yaml @@ -28,7 +28,7 @@ spec: containers: - name: kernel-browser # This will be set during deployment - image: us-docker.pkg.dev/func-241017/gcr.io/kernel-browser:latest + image: us-docker.pkg.dev/PROJECT_ID/gcr.io/kernel-browser:latest ports: - name: http1 containerPort: 8080 From ca565e40825e755f91d39217c7b9414bc6a8a668 Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Tue, 9 Sep 2025 13:54:17 -0500 Subject: [PATCH 18/19] Fixed hardcoded project name; removed google as a start page --- deploy.sh | 2 +- supervisor/services-cloudrun/chromium.conf | 2 +- twilio/update-twilio-credentials.sh | 26 ++++++++++++++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/deploy.sh b/deploy.sh index 048bb3f..2996275 100755 --- a/deploy.sh +++ b/deploy.sh @@ -312,7 +312,7 @@ deploy_local() { # Update service file with project ID and image cp "$service_file" "${service_file}.tmp" sed -i.bak "s/PROJECT_ID/$PROJECT_ID/g" "${service_file}.tmp" - sed -i.bak "s|us-docker.pkg.dev/func-241017/gcr.io/kernel-browser:latest|$image_name|g" "${service_file}.tmp" + sed -i.bak "s|us-docker.pkg.dev/$PROJECT_ID/gcr.io/kernel-browser:latest|$image_name|g" "${service_file}.tmp" gcloud run services replace "${service_file}.tmp" \ --region="$REGION" \ diff --git a/supervisor/services-cloudrun/chromium.conf b/supervisor/services-cloudrun/chromium.conf index 6fdc548..d8413f5 100644 --- a/supervisor/services-cloudrun/chromium.conf +++ b/supervisor/services-cloudrun/chromium.conf @@ -1,5 +1,5 @@ [program:chromium] -command=/bin/bash -lc 'sleep 3 && DISPLAY=":1" DBUS_SESSION_BUS_ADDRESS="unix:path=/tmp/dbus/session_bus_socket" chromium --remote-debugging-port=9223 --remote-allow-origins=* --user-data-dir=/home/kernel/user-data --password-store=basic --no-first-run --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor --custom-devtools-frontend=http://localhost:8001/ https://www.google.com' +command=/bin/bash -lc 'sleep 3 && DISPLAY=":1" DBUS_SESSION_BUS_ADDRESS="unix:path=/tmp/dbus/session_bus_socket" chromium --remote-debugging-port=9223 --remote-allow-origins=* --user-data-dir=/home/kernel/user-data --password-store=basic --no-first-run --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --no-sandbox --disable-setuid-sandbox --disable-features=VizDisplayCompositor --custom-devtools-frontend=http://localhost:8001/' autostart=true autorestart=true startsecs=8 diff --git a/twilio/update-twilio-credentials.sh b/twilio/update-twilio-credentials.sh index a468345..3ce4364 100755 --- a/twilio/update-twilio-credentials.sh +++ b/twilio/update-twilio-credentials.sh @@ -6,18 +6,36 @@ set -e +# Load environment variables from .env file if it exists +if [ -f ../.env ]; then + set -a + . ../.env + set +a +elif [ -f .env ]; then + set -a + . .env + set +a +fi + # Configuration -PROJECT_ID="${PROJECT_ID:-func-241017}" +PROJECT_ID="${PROJECT_ID}" SERVICE_NAME="kernel-browser" -REGION="us-central1" +REGION="${REGION:-us-central1}" -# Twilio credentials (set these as environment variables) +# Twilio credentials (from environment or .env file) TWILIO_ACCOUNT_SID="${TWILIO_ACCOUNT_SID}" TWILIO_AUTH_TOKEN="${TWILIO_AUTH_TOKEN}" +if [ -z "$PROJECT_ID" ]; then + echo "โŒ Error: PROJECT_ID must be set" + echo " Set it in your .env file or export as environment variable:" + echo " export PROJECT_ID=your-project-id" + exit 1 +fi + if [ -z "$TWILIO_ACCOUNT_SID" ] || [ -z "$TWILIO_AUTH_TOKEN" ]; then echo "โŒ Error: TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN must be set" - echo " Export them as environment variables:" + echo " Set them in your .env file or export as environment variables:" echo " export TWILIO_ACCOUNT_SID=your_account_sid" echo " export TWILIO_AUTH_TOKEN=your_auth_token" exit 1 From 7c6253de22448b6aa5d63b4b338332c4dd535a8a Mon Sep 17 00:00:00 2001 From: Oleh Luchkiv Date: Tue, 9 Sep 2025 13:54:52 -0500 Subject: [PATCH 19/19] Modified example env file --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 268baa2..2aee412 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,9 @@ TWILIO_ACCOUNT_SID=SK...your_api_key_sid_here TWILIO_AUTH_TOKEN=your_api_key_secret_here -# Optional: Google Cloud Configuration +# Google Cloud Configuration # If not provided, will use current gcloud config -# PROJECT_ID=your-project-id +PROJECT_ID=your-gcp-project-id # REGION=us-central1 # Optional: Service Configuration