feat(remote): support sqlite object uploads

remote/contract.go

@@ -74,6 +74,7 @@ func BaseContract() Contract {
 			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/query", Auth: AuthReader},
 			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/batch-read", Auth: AuthReader},
 			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/ingest", Auth: AuthPublisher},
+			{Method: http.MethodPut, Path: "/v1/apps/:app/archives/:archive/sqlite", Auth: AuthPublisher},
 		},
 		Auth: []AuthSpec{
 			{Name: AuthPublic, Description: "no bearer token required"},

remote/remote.go

@@ -198,6 +198,7 @@ type Status struct {
 	LastIngestAt string          `json:"last_ingest_at,omitempty"`
 	Counts       []control.Count `json:"counts,omitempty"`
 	Capabilities []string        `json:"capabilities,omitempty"`
+	SQLiteObject *SQLiteObject   `json:"sqlite_object,omitempty"`
 	Warnings     []string        `json:"warnings,omitempty"`
 }
 
@@ -254,6 +255,32 @@ type IngestResult struct {
 	Complete     bool   `json:"complete,omitempty"`
 }
 
+type SQLiteUploadRequest struct {
+	Body          io.Reader
+	Size          int64
+	ContentSHA256 string
+	SchemaName    string
+	SchemaVersion int
+	SchemaHash    string
+	SourceSyncAt  string
+}
+
+type SQLiteUploadResult struct {
+	App      string        `json:"app,omitempty"`
+	Archive  string        `json:"archive,omitempty"`
+	Complete bool          `json:"complete,omitempty"`
+	Object   *SQLiteObject `json:"object,omitempty"`
+}
+
+type SQLiteObject struct {
+	Key         string `json:"key,omitempty"`
+	Size        int64  `json:"size,omitempty"`
+	ETag        string `json:"etag,omitempty"`
+	UploadedAt  string `json:"uploaded_at,omitempty"`
+	ContentType string `json:"content_type,omitempty"`
+	SHA256      string `json:"sha256,omitempty"`
+}
+
 type LoginStartRequest struct {
 	PollSecretHash string `json:"pollSecretHash"`
 }
@@ -346,6 +373,22 @@ func (c *Client) Ingest(ctx context.Context, app, archive string, req IngestRequ
 	return out, err
 }
 
+func (c *Client) UploadSQLite(ctx context.Context, app, archive string, upload SQLiteUploadRequest) (SQLiteUploadResult, error) {
+	if upload.Body == nil {
+		return SQLiteUploadResult{}, errors.New("sqlite upload body is required")
+	}
+	headers := http.Header{}
+	headers.Set("content-type", "application/vnd.sqlite3")
+	setHeader(headers, "x-crawl-schema-name", upload.SchemaName)
+	setHeader(headers, "x-crawl-schema-version", intHeader(upload.SchemaVersion))
+	setHeader(headers, "x-crawl-schema-hash", upload.SchemaHash)
+	setHeader(headers, "x-crawl-source-sync-at", upload.SourceSyncAt)
+	setHeader(headers, "x-crawl-content-sha256", upload.ContentSHA256)
+	var out SQLiteUploadResult
+	err := c.doRaw(ctx, http.MethodPut, archivePath(app, archive, "sqlite"), upload.Body, upload.Size, headers, &out, true)
+	return out, err
+}
+
 func (c *Client) StartGitHubLogin(ctx context.Context, pollSecretHash string) (LoginStartResult, error) {
 	var out LoginStartResult
 	err := c.do(ctx, http.MethodPost, "/v1/auth/github/start", LoginStartRequest{PollSecretHash: pollSecretHash}, &out, false)
@@ -395,9 +438,29 @@ func (c *Client) do(ctx context.Context, method, route string, input, output any
 	if err != nil {
 		return err
 	}
+	return c.doRequest(ctx, req, input != nil, output, auth)
+}
+
+func (c *Client) doRaw(ctx context.Context, method, route string, body io.Reader, size int64, headers http.Header, output any, auth bool) error {
+	req, err := http.NewRequestWithContext(ctx, method, c.url(route), body)
+	if err != nil {
+		return err
+	}
+	if size >= 0 {
+		req.ContentLength = size
+	}
+	for name, values := range headers {
+		for _, value := range values {
+			req.Header.Add(name, value)
+		}
+	}
+	return c.doRequest(ctx, req, true, output, auth)
+}
+
+func (c *Client) doRequest(ctx context.Context, req *http.Request, hasBody bool, output any, auth bool) error {
 	req.Header.Set("accept", "application/json")
 	req.Header.Set("user-agent", c.userAgent)
-	if input != nil {
+	if hasBody && req.Header.Get("content-type") == "" {
 		req.Header.Set("content-type", "application/json")
 	}
 	if auth {
@@ -428,6 +491,20 @@ func (c *Client) do(ctx context.Context, method, route string, input, output any
 	return nil
 }
 
+func setHeader(headers http.Header, name, value string) {
+	value = strings.TrimSpace(value)
+	if value != "" {
+		headers.Set(name, value)
+	}
+}
+
+func intHeader(value int) string {
+	if value <= 0 {
+		return ""
+	}
+	return fmt.Sprintf("%d", value)
+}
+
 func (c *Client) url(route string) string {
 	route = "/" + strings.TrimLeft(route, "/")
 	u := *c.endpoint

remote/remote_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -83,6 +84,70 @@ func TestClientQuerySendsBearerAndEscapedArchive(t *testing.T) {
 	}
 }
 
+func TestClientUploadSQLiteSendsRawBodyAndMetadata(t *testing.T) {
+	var sawAuth string
+	var sawPath string
+	var sawContentType string
+	var sawLength int64
+	var sawSHA string
+	var sawSchema string
+	var body string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		sawAuth = r.Header.Get("authorization")
+		sawPath = r.URL.EscapedPath()
+		sawContentType = r.Header.Get("content-type")
+		sawLength = r.ContentLength
+		sawSHA = r.Header.Get("x-crawl-content-sha256")
+		sawSchema = r.Header.Get("x-crawl-schema-name")
+		bytes, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("read body: %v", err)
+		}
+		body = string(bytes)
+		_ = json.NewEncoder(w).Encode(SQLiteUploadResult{
+			App:      "gitcrawl",
+			Archive:  "gitcrawl/openclaw__openclaw",
+			Complete: true,
+			Object:   &SQLiteObject{Key: "gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/current.db", Size: int64(len(bytes)), SHA256: sawSHA},
+		})
+	}))
+	defer server.Close()
+
+	client, err := NewClient(Options{Endpoint: server.URL, TokenProvider: StaticToken("secret"), UserAgent: "test-agent"})
+	if err != nil {
+		t.Fatalf("client: %v", err)
+	}
+	result, err := client.UploadSQLite(context.Background(), "gitcrawl", "gitcrawl/openclaw__openclaw", SQLiteUploadRequest{
+		Body:          strings.NewReader("SQLite bytes"),
+		Size:          int64(len("SQLite bytes")),
+		ContentSHA256: "abc123",
+		SchemaName:    "gitcrawl-cloud-v1",
+		SchemaVersion: 1,
+		SchemaHash:    "gitcrawl-cloud-v1",
+	})
+	if err != nil {
+		t.Fatalf("upload: %v", err)
+	}
+	if sawAuth != "Bearer secret" {
+		t.Fatalf("auth = %q", sawAuth)
+	}
+	if !strings.Contains(sawPath, "gitcrawl%2Fopenclaw__openclaw") || !strings.HasSuffix(sawPath, "/sqlite") {
+		t.Fatalf("path = %q", sawPath)
+	}
+	if sawContentType != "application/vnd.sqlite3" {
+		t.Fatalf("content-type = %q", sawContentType)
+	}
+	if sawLength != int64(len("SQLite bytes")) || body != "SQLite bytes" {
+		t.Fatalf("body len/body = %d/%q", sawLength, body)
+	}
+	if sawSHA != "abc123" || sawSchema != "gitcrawl-cloud-v1" {
+		t.Fatalf("metadata sha/schema = %q/%q", sawSHA, sawSchema)
+	}
+	if result.Object == nil || result.Object.Size != int64(len("SQLite bytes")) {
+		t.Fatalf("result = %#v", result)
+	}
+}
+
 func TestClientErrorDecoding(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusForbidden)