feat: harden scheduling/reconciliation observability, add safe VM artifact cleanup, and upgrade client + CI/CD workflows

.github/workflows/ci.yml

@@ -8,9 +8,16 @@ on:
       - master
       - work
 
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
 jobs:
-  unit-tests:
-    name: Unit Tests
+  lint-and-validate:
+    name: Lint And Validate
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -22,16 +29,89 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      - name: Download dependencies
+      - name: Go mod download
         run: go mod download
 
-      - name: Run unit tests
+      - name: Verify gofmt
+        run: |
+          files="$(gofmt -l .)"
+          if [ -n "$files" ]; then
+            echo "Unformatted files:"
+            echo "$files"
+            exit 1
+          fi
+
+      - name: Verify shell scripts syntax
+        run: |
+          if find . -type f -name '*.sh' | grep -q .; then
+            find . -type f -name '*.sh' -print0 | xargs -0 -n1 bash -n
+          fi
+
+      # - name: Run golangci-lint
+      #   uses: golangci/golangci-lint-action@v8
+      #   with:
+      #     version: latest
+      #     args: --timeout=5m
+
+  proto-drift-check:
+    name: Proto Drift Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends protobuf-compiler libprotobuf-dev
+          protoc --version
+
+      - name: Install protobuf generators
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.1
+          echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
+
+      - name: Regenerate proto
+        run: make proto
+
+      - name: Verify generated code is committed
+        run: git diff --exit-code
+
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    needs: [lint-and-validate, proto-drift-check]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Run unit tests with race and coverage
         run: make test-unit
 
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-coverage
+          path: coverage.txt
+          if-no-files-found: error
+
   e2e-tests:
     name: End-to-End Tests
     runs-on: ubuntu-latest
-    needs: unit-tests
+    needs: [unit-tests]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -42,8 +122,57 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      - name: Download dependencies
-        run: go mod download
-
       - name: Run end-to-end tests
         run: make test-e2e
+
+  build-binaries:
+    name: Build Binaries
+    runs-on: ubuntu-latest
+    needs: [unit-tests]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Build agent and client
+        run: |
+          go build -v ./cmd/agent
+          go build -v ./examples/client
+
+  docker-build:
+    name: Docker Build
+    runs-on: ubuntu-latest
+    needs: [unit-tests]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build optimized runtime image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          target: runtime
+          push: false
+          tags: persys/compute-agent:ci-runtime
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build full runtime image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          target: full-runtime
+          push: false
+          tags: persys/compute-agent:ci-full-runtime
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Dockerfile

@@ -1,57 +1,59 @@
-# Build stage
-FROM golang:1.21-alpine AS builder
+# syntax=docker/dockerfile:1.7
 
-# Install build dependencies
-RUN apk add --no-cache git make protobuf protobuf-dev
+ARG GO_VERSION=1.24.4
+ARG VERSION=1.0.0
 
-WORKDIR /build
+FROM golang:${GO_VERSION}-bookworm AS builder
+WORKDIR /src
 
-# Copy go mod files
 COPY go.mod go.sum ./
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
 
-# Copy source code
 COPY . .
 
-# Build the binary
-RUN make build-linux
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -trimpath -ldflags "-s -w -X main.version=${VERSION}" \
+    -o /out/persys-agent ./cmd/agent
 
-# Runtime stage
-FROM ubuntu:22.04
-
-# Install runtime dependencies
-RUN apt-get update && apt-get install -y \
+FROM debian:bookworm-slim AS runtime-base
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
     docker.io \
-    docker-compose \
-    qemu-kvm \
-    libvirt-daemon-system \
+    qemu-utils \
+    genisoimage \
     libvirt-clients \
-    ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
-# Create persys user
-RUN useradd -r -u 1000 -g 0 -m -s /bin/bash persys && \
+RUN useradd -r -u 1000 -g 0 -m -s /usr/sbin/nologin persys && \
     mkdir -p /var/lib/persys /etc/persys/certs && \
     chown -R persys:root /var/lib/persys /etc/persys
 
-# Copy binary from builder
-COPY --from=builder /build/bin/persys-agent-linux-amd64 /usr/local/bin/persys-agent
-RUN chmod +x /usr/local/bin/persys-agent
+COPY --from=builder /out/persys-agent /usr/local/bin/persys-agent
 
-# Set up volumes
 VOLUME ["/var/lib/persys", "/etc/persys"]
+EXPOSE 50051 8080
 
-# Expose gRPC port
-EXPOSE 50051
-
-# Run as persys user
 USER persys
 
-# Set default environment variables
 ENV PERSYS_STATE_PATH=/var/lib/persys/state.db \
     PERSYS_TLS_CERT=/etc/persys/certs/agent.crt \
     PERSYS_TLS_KEY=/etc/persys/certs/agent.key \
     PERSYS_TLS_CA=/etc/persys/certs/ca.crt \
     PERSYS_LOG_LEVEL=info
 
 ENTRYPOINT ["/usr/local/bin/persys-agent"]
+
+# Default optimized runtime image.
+FROM runtime-base AS runtime
+
+# Optional full runtime image with local daemons/tools for all-in-one test environments.
+FROM runtime-base AS full-runtime
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    qemu-kvm \
+    libvirt-daemon-system \
+    && rm -rf /var/lib/apt/lists/*
+USER persys

Makefile

@@ -7,6 +7,8 @@ BUILD_DIR=bin
 DOCKER_IMAGE=persys/compute-agent
 PROTO_DIR=api/proto
 PKG_DIR=pkg/api/v1
+CONTROL_PKG_DIR=pkg/control/v1
+PROTOC_SYSTEM_INCLUDE?=/usr/include
 
 # Go parameters
 GOCMD=go
@@ -27,9 +29,13 @@ deps:
 proto:
 	@echo "==> Generating protobuf code..."
 	@mkdir -p $(PKG_DIR)
-	cd api/proto && protoc --go_out=../../$(PKG_DIR) --go_opt=paths=source_relative \
+	@mkdir -p $(CONTROL_PKG_DIR)
+	cd api/proto && protoc -I. -I../../$(PROTO_DIR) -I$(PROTOC_SYSTEM_INCLUDE) --go_out=../../$(PKG_DIR) --go_opt=paths=source_relative \
 		--go-grpc_out=../../$(PKG_DIR) --go-grpc_opt=paths=source_relative \
 		agent.proto
+	cd api/proto && protoc -I. -I../../$(PROTO_DIR) -I$(PROTOC_SYSTEM_INCLUDE) --go_out=../../$(CONTROL_PKG_DIR) --go_opt=paths=source_relative \
+		--go-grpc_out=../../$(CONTROL_PKG_DIR) --go-grpc_opt=paths=source_relative \
+		control.proto
 
 build:
 	@echo "==> Building $(BINARY_NAME)..."
@@ -54,6 +60,7 @@ clean:
 	$(GOCLEAN)
 	rm -rf $(BUILD_DIR)
 	rm -rf $(PKG_DIR)
+	rm -rf $(CONTROL_PKG_DIR)
 
 test:
 	@echo "==> Running all tests..."

README.md

@@ -141,6 +141,17 @@ The agent is configured via environment variables:
 |----------|---------|-------------|
 | `PERSYS_NODE_ID` | hostname | Unique node identifier |
 | `PERSYS_VERSION` | `dev` | Agent version |
+| `PERSYS_NODE_REGION` | `` | Node region label (ex: `us-east-1`) |
+| `PERSYS_NODE_ENV` | `` | Node environment label (ex: `prod`, `staging`) |
+| `PERSYS_NODE_LABELS` | `` | Extra labels as `key=value,key2=value2` |
+| `PERSYS_AGENT_GRPC_ENDPOINT` | auto-derived | Scheduler-reachable agent endpoint (`host:port`) |
+
+### Scheduler Control Plane
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `PERSYS_SCHEDULER_ADDR` | `127.0.0.1:8085` | Scheduler gRPC control endpoint (`host:port`) |
+| `PERSYS_SCHEDULER_INSECURE` | `false` | Disable TLS for scheduler control client (testing only) |
 
 ## API Reference
 

api/proto/agent.proto

@@ -20,6 +20,9 @@ service AgentService {
 
   // HealthCheck returns agent health status
   rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse);
+
+  // ListActions returns action/task history tracked by the agent since startup
+  rpc ListActions(ListActionsRequest) returns (ListActionsResponse);
 }
 
 message ApplyWorkloadRequest {
@@ -73,6 +76,29 @@ message HealthCheckResponse {
   double disk_utilization = 6;    // Disk utilization percentage for root mount (0-100)
 }
 
+message ListActionsRequest {
+  string workload_id = 1; // optional filter
+  string action_type = 2; // optional filter: apply_workload, delete_workload, ...
+  string status = 3; // optional filter: pending, running, completed, failed
+  int32 limit = 4; // optional max number of actions (0 = all)
+  bool newest_first = 5; // if true sorts by created_at descending
+}
+
+message AgentAction {
+  string task_id = 1;
+  string workload_id = 2;
+  string action_type = 3;
+  string status = 4;
+  string error = 5;
+  int64 created_at = 6;
+  int64 started_at = 7;
+  int64 ended_at = 8;
+}
+
+message ListActionsResponse {
+  repeated AgentAction actions = 1;
+}
+
 enum WorkloadType {
   WORKLOAD_TYPE_UNSPECIFIED = 0;
   WORKLOAD_TYPE_CONTAINER = 1;