diff --git a/remote/contract.go b/remote/contract.go index 28e0a0a..163d8f6 100644 --- a/remote/contract.go +++ b/remote/contract.go @@ -74,6 +74,7 @@ func BaseContract() Contract { {Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/query", Auth: AuthReader}, {Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/batch-read", Auth: AuthReader}, {Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/ingest", Auth: AuthPublisher}, + {Method: http.MethodPut, Path: "/v1/apps/:app/archives/:archive/sqlite", Auth: AuthPublisher}, }, Auth: []AuthSpec{ {Name: AuthPublic, Description: "no bearer token required"}, diff --git a/remote/remote.go b/remote/remote.go index 412a61e..7ce48af 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -198,6 +198,7 @@ type Status struct { LastIngestAt string `json:"last_ingest_at,omitempty"` Counts []control.Count `json:"counts,omitempty"` Capabilities []string `json:"capabilities,omitempty"` + SQLiteObject *SQLiteObject `json:"sqlite_object,omitempty"` Warnings []string `json:"warnings,omitempty"` } @@ -254,6 +255,32 @@ type IngestResult struct { Complete bool `json:"complete,omitempty"` } +type SQLiteUploadRequest struct { + Body io.Reader + Size int64 + ContentSHA256 string + SchemaName string + SchemaVersion int + SchemaHash string + SourceSyncAt string +} + +type SQLiteUploadResult struct { + App string `json:"app,omitempty"` + Archive string `json:"archive,omitempty"` + Complete bool `json:"complete,omitempty"` + Object *SQLiteObject `json:"object,omitempty"` +} + +type SQLiteObject struct { + Key string `json:"key,omitempty"` + Size int64 `json:"size,omitempty"` + ETag string `json:"etag,omitempty"` + UploadedAt string `json:"uploaded_at,omitempty"` + ContentType string `json:"content_type,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + type LoginStartRequest struct { PollSecretHash string `json:"pollSecretHash"` } @@ -346,6 +373,22 @@ func (c *Client) Ingest(ctx context.Context, app, archive string, req IngestRequ return out, err } +func (c *Client) UploadSQLite(ctx context.Context, app, archive string, upload SQLiteUploadRequest) (SQLiteUploadResult, error) { + if upload.Body == nil { + return SQLiteUploadResult{}, errors.New("sqlite upload body is required") + } + headers := http.Header{} + headers.Set("content-type", "application/vnd.sqlite3") + setHeader(headers, "x-crawl-schema-name", upload.SchemaName) + setHeader(headers, "x-crawl-schema-version", intHeader(upload.SchemaVersion)) + setHeader(headers, "x-crawl-schema-hash", upload.SchemaHash) + setHeader(headers, "x-crawl-source-sync-at", upload.SourceSyncAt) + setHeader(headers, "x-crawl-content-sha256", upload.ContentSHA256) + var out SQLiteUploadResult + err := c.doRaw(ctx, http.MethodPut, archivePath(app, archive, "sqlite"), upload.Body, upload.Size, headers, &out, true) + return out, err +} + func (c *Client) StartGitHubLogin(ctx context.Context, pollSecretHash string) (LoginStartResult, error) { var out LoginStartResult err := c.do(ctx, http.MethodPost, "/v1/auth/github/start", LoginStartRequest{PollSecretHash: pollSecretHash}, &out, false) @@ -395,9 +438,29 @@ func (c *Client) do(ctx context.Context, method, route string, input, output any if err != nil { return err } + return c.doRequest(ctx, req, input != nil, output, auth) +} + +func (c *Client) doRaw(ctx context.Context, method, route string, body io.Reader, size int64, headers http.Header, output any, auth bool) error { + req, err := http.NewRequestWithContext(ctx, method, c.url(route), body) + if err != nil { + return err + } + if size >= 0 { + req.ContentLength = size + } + for name, values := range headers { + for _, value := range values { + req.Header.Add(name, value) + } + } + return c.doRequest(ctx, req, true, output, auth) +} + +func (c *Client) doRequest(ctx context.Context, req *http.Request, hasBody bool, output any, auth bool) error { req.Header.Set("accept", "application/json") req.Header.Set("user-agent", c.userAgent) - if input != nil { + if hasBody && req.Header.Get("content-type") == "" { req.Header.Set("content-type", "application/json") } if auth { @@ -428,6 +491,20 @@ func (c *Client) do(ctx context.Context, method, route string, input, output any return nil } +func setHeader(headers http.Header, name, value string) { + value = strings.TrimSpace(value) + if value != "" { + headers.Set(name, value) + } +} + +func intHeader(value int) string { + if value <= 0 { + return "" + } + return fmt.Sprintf("%d", value) +} + func (c *Client) url(route string) string { route = "/" + strings.TrimLeft(route, "/") u := *c.endpoint diff --git a/remote/remote_test.go b/remote/remote_test.go index 23b7b77..6e0f074 100644 --- a/remote/remote_test.go +++ b/remote/remote_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" "net/http/httptest" "os" @@ -83,6 +84,70 @@ func TestClientQuerySendsBearerAndEscapedArchive(t *testing.T) { } } +func TestClientUploadSQLiteSendsRawBodyAndMetadata(t *testing.T) { + var sawAuth string + var sawPath string + var sawContentType string + var sawLength int64 + var sawSHA string + var sawSchema string + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sawAuth = r.Header.Get("authorization") + sawPath = r.URL.EscapedPath() + sawContentType = r.Header.Get("content-type") + sawLength = r.ContentLength + sawSHA = r.Header.Get("x-crawl-content-sha256") + sawSchema = r.Header.Get("x-crawl-schema-name") + bytes, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + body = string(bytes) + _ = json.NewEncoder(w).Encode(SQLiteUploadResult{ + App: "gitcrawl", + Archive: "gitcrawl/openclaw__openclaw", + Complete: true, + Object: &SQLiteObject{Key: "gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/current.db", Size: int64(len(bytes)), SHA256: sawSHA}, + }) + })) + defer server.Close() + + client, err := NewClient(Options{Endpoint: server.URL, TokenProvider: StaticToken("secret"), UserAgent: "test-agent"}) + if err != nil { + t.Fatalf("client: %v", err) + } + result, err := client.UploadSQLite(context.Background(), "gitcrawl", "gitcrawl/openclaw__openclaw", SQLiteUploadRequest{ + Body: strings.NewReader("SQLite bytes"), + Size: int64(len("SQLite bytes")), + ContentSHA256: "abc123", + SchemaName: "gitcrawl-cloud-v1", + SchemaVersion: 1, + SchemaHash: "gitcrawl-cloud-v1", + }) + if err != nil { + t.Fatalf("upload: %v", err) + } + if sawAuth != "Bearer secret" { + t.Fatalf("auth = %q", sawAuth) + } + if !strings.Contains(sawPath, "gitcrawl%2Fopenclaw__openclaw") || !strings.HasSuffix(sawPath, "/sqlite") { + t.Fatalf("path = %q", sawPath) + } + if sawContentType != "application/vnd.sqlite3" { + t.Fatalf("content-type = %q", sawContentType) + } + if sawLength != int64(len("SQLite bytes")) || body != "SQLite bytes" { + t.Fatalf("body len/body = %d/%q", sawLength, body) + } + if sawSHA != "abc123" || sawSchema != "gitcrawl-cloud-v1" { + t.Fatalf("metadata sha/schema = %q/%q", sawSHA, sawSchema) + } + if result.Object == nil || result.Object.Size != int64(len("SQLite bytes")) { + t.Fatalf("result = %#v", result) + } +} + func TestClientErrorDecoding(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusForbidden)